Enhance overcloud openbao deployment

doc/source/configuration/openbao.rst

@@ -604,3 +604,60 @@ However, end users of OpenStack will not be affected.
 Overcloud migration is HA migration and no downtime is expected.
 
 It is recommended to run ``vault-bao-migration-change-config.yml`` after all Vault deployments have been migrated to OpenBao.
+
+.. _openbao-cluster-fix:
+
+Restoring OpenBao cluster
+=========================
+
+Users can use ``fix-openbao-overcloud.yml`` playbook to restore OpenBao cluster.
+
+.. code-block:: bash
+
+   kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/secret-store/fix-openbao-overcloud.yml
+
+.. warning::
+
+   The playbook ``secret-store-deploy-overcloud.yml`` assumes the first
+   controller to always be the leader node.
+   **DO NOT** run ``secret-store-deploy-overcloud.yml`` alone to fix the
+   cluster.
+   If the leader OpenBao node (First controller by default) fails, the leader
+   role gets transferred to one of the other controllers.
+   Without checking which node became the new leader, there is a risk of having
+   two separate clusters as a result.
+
+The ``fix-openbao-overcloud.yml`` playbook runs two playbooks
+
+1. ``get-current-raft-leader.yml``
+2. ``secret-store-deploy-overcloud.yml``
+
+Users can also follow this procedure to fix the OpenBao cluster manually.
+
+1. Use ``get-current-raft-leader.yml`` playbook to get the index of the leader
+   controller.
+
+   .. code-block:: bash
+
+      kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/secret-store/get-current-raft-leader.yml
+
+   The last task "Display the index of the Raft leader" will show the index of
+   the new leader controller in the controllers ansible group.
+
+   .. code-block:: bash
+
+      TASK [Display the index of the Raft leader] ***********************************
+      Monday 03 August 2026  12:15:02 +0000 (0:00:00.148)       0:00:07.797 *********
+      ok: [controller-01] =>
+          msg: 'raft_leader_index: 2'
+      ok: [controller-02] =>
+          msg: 'raft_leader_index: 2'
+      ok: [controller-03] =>
+          msg: 'raft_leader_index: 2'
+
+2. Run ``secret-store-deploy-overcloud.yml`` playbook with the index of the new
+   leader as an extra variable.
+
+   .. code-block:: bash
+
+      kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/secret-store/secret-store-deploy-overcloud.yml -e raft_leader_index=2

etc/kayobe/ansible/secret-store/fix-openbao-overcloud.yml

@@ -0,0 +1,6 @@
+---
+- name: Get current OpenBao Raft leader
+  import_playbook: get-current-raft-leader.yml
+
+- name: Run deploy playbook with the current raft_leader_index set
+  import_playbook: secret-store-deploy-overcloud.yml

etc/kayobe/ansible/secret-store/get-current-raft-leader.yml

@@ -0,0 +1,68 @@
+---
+- name: Get current Raft leader
+  hosts: controllers
+  vars:
+    secret_store_bind_interface: "{{ internal_net_name | net_interface | replace('-', '_') }}"
+    secret_store_bind_address: "{{ ansible_facts[secret_store_bind_interface].ipv4.address }}"
+    secret_store_api_address: "https://{{ secret_store_bind_address }}:8200"
+  tasks:
+    - name: Set a fact about the virtualenv on the remote system
+      ansible.builtin.set_fact:
+        virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}"
+      when:
+        - ansible_python_interpreter is defined
+        - not ansible_python_interpreter.startswith('/bin/')
+        - not ansible_python_interpreter.startswith('/usr/bin/')
+
+    - name: Ensure Python hvac module is installed
+      ansible.builtin.pip:
+        name: hvac
+        state: present
+        extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}"
+        virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}"
+      become: "{{ virtualenv is not defined }}"
+
+    - name: Include secret store keys
+      ansible.builtin.include_vars:
+        file: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/overcloud-{{ stackhpc_ca_secret_store }}-keys.json"
+        name: secret_store_keys
+
+    - name: Query the HA status
+      ansible.builtin.uri:
+        url: "{{ secret_store_api_address }}/v1/sys/ha-status"
+        headers:
+          X-Vault-Token: "{{ secret_store_keys.root_token }}"
+        follow_redirects: none
+      register: ha_status_query
+      ignore_errors: true
+
+    - name: Query the leader
+      ansible.builtin.uri:
+        url: "{{ secret_store_api_address }}/v1/sys/leader"
+        headers:
+          X-Vault-Token: "{{ secret_store_keys.root_token }}"
+        follow_redirects: none
+      register: leader_query
+      ignore_errors: true
+
+    - name: Determine Raft leader
+      ansible.builtin.group_by:
+        key: raft_leader
+      when:
+        - ha_status_query.json is defined
+        - leader_query.json is defined
+        - ha_status_query.json.nodes | length > 1
+        - leader_query.json.is_self | bool
+
+    - name: Assert that there is only one leader
+      ansible.builtin.assert:
+        that: groups['raft_leader'] | default([]) | length == 1
+        fail_msg: There are multiple OpenBao Raft leader. Check the status of the cluster.
+
+    - name: Set Raft leader index fact
+      ansible.builtin.set_fact:
+        raft_leader_index: "{{ lookup('ansible.utils.index_of', groups['controllers'], 'eq', (groups['raft_leader'] | first)) }}"
+
+    - name: Display the index of the Raft leader
+      ansible.builtin.debug:
+        msg: "raft_leader_index: {{ raft_leader_index }}"

etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml

@@ -23,11 +23,15 @@
   vars:
     secret_store_bind_interface: "{{ internal_net_name | net_interface }}"
     secret_store_bind_address: "{{ internal_net_name | net_ip }}"
-    # This is the IP address of the first controller and therefore the leader within
-    # OpenBao. This could be replaced with the VIP address of the internal network if
+    # This is the index of the controller with Raft leadership.
+    # Default value is 0 (First controller becomes OpenBao raft leader).
+    # The index should follow current leader when restarting/fixing the cluster
+    # to prevent having two separate clusters (Split brain).
+    raft_leader_index: 0
+    # This is the IP address of  the leader within OpenBao.
+    # This could be replaced with the VIP address of the internal network if
     # HAProxy has been configured to load balance the OpenBao API.
-    openbao_raft_leaders:
-      - "{{ internal_net_name | net_ip(inventory_hostname=groups['controllers'][0]) }}"
+    raft_leader_address: "{{ internal_net_name | net_ip(inventory_hostname=groups['controllers'][raft_leader_index | int]) }}"
   tasks:
     - name: Set a fact about the virtualenv on the remote system
       ansible.builtin.set_fact:
@@ -98,6 +102,9 @@
         openbao_registry_password: "{{ overcloud_openbao_registry_password }}"
         openbao_config_dir: /opt/kayobe/openbao
         openbao_cluster_name: overcloud
+        openbao_raft_leaders:
+          - "{{ raft_leader_address }}"
+        openbao_init_address: "{{ raft_leader_address }}"
         openbao_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}"
         openbao_docker_image: "{{ overcloud_openbao_docker_image }}"
         openbao_docker_tag: "{{ overcloud_openbao_docker_tag }}"
@@ -123,9 +130,11 @@
         vault_unseal_token: "{{ secret_store_keys.root_token }}"
         vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}"
         vault_unseal_keys: "{{ secret_store_keys.keys_base64 }}"
+        vault_unseal_timeout: 10
       environment:
         https_proxy: ""
       run_once: true
+      delegate_to: "{{ groups['controllers'][raft_leader_index | int] }}"
 
       # As the first instance is now unsealed the other instances will now need some
       # time to connect before we can proceed.