diff --git a/doc/source/configuration/openbao.rst b/doc/source/configuration/openbao.rst index 0248f8bd69..dae91bf3e2 100644 --- a/doc/source/configuration/openbao.rst +++ b/doc/source/configuration/openbao.rst @@ -604,3 +604,60 @@ However, end users of OpenStack will not be affected. Overcloud migration is HA migration and no downtime is expected. It is recommended to run ``vault-bao-migration-change-config.yml`` after all Vault deployments have been migrated to OpenBao. + +.. _openbao-cluster-fix: + +Restoring OpenBao cluster +========================= + +Users can use ``fix-openbao-overcloud.yml`` playbook to restore OpenBao cluster. + +.. code-block:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/secret-store/fix-openbao-overcloud.yml + +.. warning:: + + The playbook ``secret-store-deploy-overcloud.yml`` assumes the first + controller to always be the leader node. + **DO NOT** run ``secret-store-deploy-overcloud.yml`` alone to fix the + cluster. + If the leader OpenBao node (First controller by default) fails, the leader + role gets transferred to one of the other controllers. + Without checking which node became the new leader, there is a risk of having + two separate clusters as a result. + +The ``fix-openbao-overcloud.yml`` playbook runs two playbooks + +1. ``get-current-raft-leader.yml`` +2. ``secret-store-deploy-overcloud.yml`` + +Users can also follow this procedure to fix the OpenBao cluster manually. + +1. Use ``get-current-raft-leader.yml`` playbook to get the index of the leader + controller. + + .. code-block:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/secret-store/get-current-raft-leader.yml + + The last task "Display the index of the Raft leader" will show the index of + the new leader controller in the controllers ansible group. + + .. code-block:: bash + + TASK [Display the index of the Raft leader] *********************************** + Monday 03 August 2026 12:15:02 +0000 (0:00:00.148) 0:00:07.797 ********* + ok: [controller-01] => + msg: 'raft_leader_index: 2' + ok: [controller-02] => + msg: 'raft_leader_index: 2' + ok: [controller-03] => + msg: 'raft_leader_index: 2' + +2. Run ``secret-store-deploy-overcloud.yml`` playbook with the index of the new + leader as an extra variable. + + .. code-block:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/secret-store/secret-store-deploy-overcloud.yml -e raft_leader_index=2 diff --git a/etc/kayobe/ansible/secret-store/fix-openbao-overcloud.yml b/etc/kayobe/ansible/secret-store/fix-openbao-overcloud.yml new file mode 100644 index 0000000000..91bb59467d --- /dev/null +++ b/etc/kayobe/ansible/secret-store/fix-openbao-overcloud.yml @@ -0,0 +1,6 @@ +--- +- name: Get current OpenBao Raft leader + import_playbook: get-current-raft-leader.yml + +- name: Run deploy playbook with the current raft_leader_index set + import_playbook: secret-store-deploy-overcloud.yml diff --git a/etc/kayobe/ansible/secret-store/get-current-raft-leader.yml b/etc/kayobe/ansible/secret-store/get-current-raft-leader.yml new file mode 100644 index 0000000000..46a19fe518 --- /dev/null +++ b/etc/kayobe/ansible/secret-store/get-current-raft-leader.yml @@ -0,0 +1,68 @@ +--- +- name: Get current Raft leader + hosts: controllers + vars: + secret_store_bind_interface: "{{ internal_net_name | net_interface | replace('-', '_') }}" + secret_store_bind_address: "{{ ansible_facts[secret_store_bind_interface].ipv4.address }}" + secret_store_api_address: "https://{{ secret_store_bind_address }}:8200" + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: present + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Include secret store keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/overcloud-{{ stackhpc_ca_secret_store }}-keys.json" + name: secret_store_keys + + - name: Query the HA status + ansible.builtin.uri: + url: "{{ secret_store_api_address }}/v1/sys/ha-status" + headers: + X-Vault-Token: "{{ secret_store_keys.root_token }}" + follow_redirects: none + register: ha_status_query + ignore_errors: true + + - name: Query the leader + ansible.builtin.uri: + url: "{{ secret_store_api_address }}/v1/sys/leader" + headers: + X-Vault-Token: "{{ secret_store_keys.root_token }}" + follow_redirects: none + register: leader_query + ignore_errors: true + + - name: Determine Raft leader + ansible.builtin.group_by: + key: raft_leader + when: + - ha_status_query.json is defined + - leader_query.json is defined + - ha_status_query.json.nodes | length > 1 + - leader_query.json.is_self | bool + + - name: Assert that there is only one leader + ansible.builtin.assert: + that: groups['raft_leader'] | default([]) | length == 1 + fail_msg: There are multiple OpenBao Raft leader. Check the status of the cluster. + + - name: Set Raft leader index fact + ansible.builtin.set_fact: + raft_leader_index: "{{ lookup('ansible.utils.index_of', groups['controllers'], 'eq', (groups['raft_leader'] | first)) }}" + + - name: Display the index of the Raft leader + ansible.builtin.debug: + msg: "raft_leader_index: {{ raft_leader_index }}" diff --git a/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml b/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml index aa9a200d1f..86cd554337 100644 --- a/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml +++ b/etc/kayobe/ansible/secret-store/secret-store-deploy-overcloud.yml @@ -23,11 +23,15 @@ vars: secret_store_bind_interface: "{{ internal_net_name | net_interface }}" secret_store_bind_address: "{{ internal_net_name | net_ip }}" - # This is the IP address of the first controller and therefore the leader within - # OpenBao. This could be replaced with the VIP address of the internal network if + # This is the index of the controller with Raft leadership. + # Default value is 0 (First controller becomes OpenBao raft leader). + # The index should follow current leader when restarting/fixing the cluster + # to prevent having two separate clusters (Split brain). + raft_leader_index: 0 + # This is the IP address of the leader within OpenBao. + # This could be replaced with the VIP address of the internal network if # HAProxy has been configured to load balance the OpenBao API. - openbao_raft_leaders: - - "{{ internal_net_name | net_ip(inventory_hostname=groups['controllers'][0]) }}" + raft_leader_address: "{{ internal_net_name | net_ip(inventory_hostname=groups['controllers'][raft_leader_index | int]) }}" tasks: - name: Set a fact about the virtualenv on the remote system ansible.builtin.set_fact: @@ -98,6 +102,9 @@ openbao_registry_password: "{{ overcloud_openbao_registry_password }}" openbao_config_dir: /opt/kayobe/openbao openbao_cluster_name: overcloud + openbao_raft_leaders: + - "{{ raft_leader_address }}" + openbao_init_address: "{{ raft_leader_address }}" openbao_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" openbao_docker_image: "{{ overcloud_openbao_docker_image }}" openbao_docker_tag: "{{ overcloud_openbao_docker_tag }}" @@ -123,9 +130,11 @@ vault_unseal_token: "{{ secret_store_keys.root_token }}" vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" vault_unseal_keys: "{{ secret_store_keys.keys_base64 }}" + vault_unseal_timeout: 10 environment: https_proxy: "" run_once: true + delegate_to: "{{ groups['controllers'][raft_leader_index | int] }}" # As the first instance is now unsealed the other instances will now need some # time to connect before we can proceed.