Add baremetal ABI support for NODES_PLATFORM=baremetal

01_install_requirements.sh

@@ -27,6 +27,23 @@ if [ -z "${METAL3_DEV_ENV:-}" ]; then
 
 # TODO: remove once Python fixed OpenSSL regression
   sed -i '/name: "{{ packages.centos.common.packages }}"/a\      exclude: "openssl*"' vm-setup/roles/packages_installation/tasks/main.yml
+
+  # Go tarball defaults hardcode linux-amd64; use GOARCH passed as extra var.
+  # Upstream fix: https://github.com/metal3-io/metal3-dev-env/pull/1694
+  GO_DEFAULTS="vm-setup/roles/packages_installation/defaults/main.yml"
+  if grep -q 'linux-amd64' "${GO_DEFAULTS}"; then
+    sed -i 's/go_tarball: "go{{ go_version }}.linux-amd64.tar.gz"/go_tarball: "go{{ go_version }}.linux-{{ GOARCH | default('\''amd64'\'') }}.tar.gz"/' \
+      "${GO_DEFAULTS}"
+  fi
+
+  # RHEL 10: genisoimage replaced by xorriso (provides mkisofs),
+  # python3-bcrypt not in base repos (soft dep of passlib, not required).
+  if [[ "${DISTRO}" =~ ^(rhel10|centos10) ]]; then
+    PKG_DEFAULTS="vm-setup/roles/packages_installation/defaults/main.yml"
+    sed -i '/^      - genisoimage$/d' "${PKG_DEFAULTS}"
+    sed -i '/^      - python3-bcrypt$/d' "${PKG_DEFAULTS}"
+  fi
+
   popd
 fi
 
@@ -115,8 +132,16 @@ case $DISTRO in
     sudo ln -s /usr/bin/python3 /usr/bin/python || true
     PYTHON_DEVEL="python3-devel"
     ;;
+  "rhel10"|"centos10")
+    sudo dnf -y install python3-pip
+    if sudo subscription-manager identity > /dev/null 2>&1; then
+      sudo subscription-manager repos --enable "codeready-builder-for-rhel-10-$(arch)-rpms" || true
+    fi
+    sudo ln -s /usr/bin/python3 /usr/bin/python || true
+    PYTHON_DEVEL="python3-devel"
+    ;;
   *)
-    echo -n "CentOS 9 or RHEL 9 required (el8 is no longer supported due to glibc requirements)"
+    echo -n "CentOS 9+, RHEL 9+, or compatible required (el8 is no longer supported due to glibc requirements)"
     exit 1
     ;;
 esac

02_configure_host.sh

@@ -86,6 +86,20 @@ if [[ "${NODES_PLATFORM}" == "baremetal" ]] && [ -n "$NODES_FILE" ] ; then
     set -x
 fi
 
+if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then
+    sudo systemctl enable --now firewalld
+    configure_chronyd
+
+    sudo firewall-cmd --zone=public --add-port=80/tcp --add-port=53/tcp --add-port=53/udp \
+        --add-port=8000/tcp --add-port="${LOCAL_REGISTRY_PORT}"/tcp \
+        --add-port="${INSTALLER_PROXY_PORT}"/tcp --add-port="${AGENT_BOOT_SERVER_PORT}"/tcp \
+        --add-port=3260/tcp
+
+    switch_to_internal_dns
+
+    exit 0
+fi
+
 # TODO - move this to metal3-dev-env.
 # This is to address the following error:
 #   "msg": "internal error: Check the host setup: enabling IPv6 forwarding with RA routes without accept_ra set to 2 is likely to cause routes loss. Interfaces to look at: eno2"

agent/05_agent_configure.sh

@@ -182,6 +182,51 @@ function get_static_ips_and_macs() {
     fi
 }
 
+function get_baremetal_ips_and_macs() {
+    AGENT_NODES_IPS=()
+    AGENT_NODES_IPSV6=()
+    AGENT_NODES_MACS=()
+    AGENT_NODES_HOSTNAMES=()
+    AGENT_MASTER_HOSTNAMES=()
+    AGENT_EXTRA_WORKERS_IPS=()
+    AGENT_EXTRA_WORKERS_IPSV6=()
+    AGENT_EXTRA_WORKERS_MACS=()
+    AGENT_EXTRA_WORKERS_HOSTNAMES=()
+
+    IFS=',' read -ra ips <<< "${BAREMETAL_IPS}"
+    local total_nodes
+    total_nodes=$(jq '.nodes | length' "$NODES_FILE")
+
+    local -a data_macs=()
+    if [[ -n "${BAREMETAL_MACS:-}" ]]; then
+        IFS=',' read -ra data_macs <<< "${BAREMETAL_MACS}"
+    fi
+
+    for (( i=0; i < total_nodes; i++ )); do
+        local mac hostname ip
+        if [[ ${#data_macs[@]} -gt 0 ]]; then
+            mac="${data_macs[$i]}"
+        else
+            mac=$(node_val "$i" "ports[0].address")
+        fi
+        hostname=$(node_val "$i" "name")
+        ip="${ips[$i]}"
+
+        add_ip_host_entry "$ip" "$hostname"
+
+        if (( i < NUM_MASTERS )); then
+            AGENT_NODES_IPS+=("$ip")
+            AGENT_NODES_MACS+=("$mac")
+            AGENT_NODES_HOSTNAMES+=("$hostname")
+            AGENT_MASTER_HOSTNAMES+=("$hostname")
+        else
+            AGENT_EXTRA_WORKERS_IPS+=("$ip")
+            AGENT_EXTRA_WORKERS_MACS+=("$mac")
+            AGENT_EXTRA_WORKERS_HOSTNAMES+=("$hostname")
+        fi
+    done
+}
+
 function generate_extra_cluster_manifests() {
 
   mkdir -p "${EXTRA_MANIFESTS_PATH}"
@@ -653,7 +698,15 @@ write_pull_secret
 # This is temporary and will go away when https://github.com/nmstate/nmstate is used
 sudo yum install -y nmstate
 
-get_static_ips_and_macs
+if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then
+    # When 02_configure_host.sh is skipped (no libvirt), the NM dnsmasq
+    # plugin hasn't been activated. Enable it so configure_dnsmasq entries
+    # actually resolve.
+    switch_to_internal_dns
+    get_baremetal_ips_and_macs
+else
+    get_static_ips_and_macs
+fi
 
 get_nodes_bmc_info
 
@@ -700,8 +753,15 @@ else
   # For ISO_NO_REGISTRY mode with workers, add api-int to libvirt DNS
   # Workers need to resolve api-int during bootstrap before joining the cluster
   if [[ "${NUM_MASTERS}" -gt "1" ]]; then
-   add_dns_entry "${API_VIPS%"${VIPS_SEPARATOR}"*}" "api-int"
+    if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then
+      add_dnsmasq_multi_entry "apiintvip" "${API_VIPS}"
+      sudo systemctl reload NetworkManager
+    else
+      add_dns_entry "${API_VIPS%"${VIPS_SEPARATOR}"*}" "api-int"
+    fi
   fi
 fi
 
-enable_isolated_baremetal_network
+if [[ "${NODES_PLATFORM}" != "baremetal" ]]; then
+    enable_isolated_baremetal_network
+fi

agent/06_agent_create_cluster.sh

@@ -163,6 +163,111 @@ function attach_agent_iso() {
 
 }
 
+function redfish_curl() {
+    local user="$1" password="$2" url="$3"
+    shift 3
+    curl -s -u "${user}":"${password}" -k -H 'Content-Type: application/json' "$url" "$@"
+}
+
+function mount_agent_iso_baremetal() {
+    local iso_url="${BAREMETAL_ISO_SERVER}"
+    local total_nodes
+    total_nodes=$(jq '.nodes | length' "$NODES_FILE")
+
+    for (( i=0; i < total_nodes; i++ )); do
+        local address user password scheme system systemurl bmc_base
+        address=$(jq -r ".nodes[$i].driver_info.address" "$NODES_FILE")
+        user=$(jq -r ".nodes[$i].driver_info.username" "$NODES_FILE")
+        password=$(jq -r ".nodes[$i].driver_info.password" "$NODES_FILE")
+        local node_name
+        node_name=$(jq -r ".nodes[$i].name" "$NODES_FILE")
+
+        if [[ ! $address =~ ^(redfish.*://)(.*)$ ]]; then
+            echo "ERROR: Unsupported BMC protocol for node $i ($address) — Redfish required"
+            exit 1
+        fi
+        scheme="https://"
+        system="${BASH_REMATCH[2]}"
+        if [[ ${BASH_REMATCH[1]} =~ http: ]]; then
+            scheme="http://"
+        fi
+        systemurl="${scheme}${system}"
+        bmc_base="${scheme}${system%%/*}"
+
+        local manager_id vmedia_uri cd_slot
+        manager_id=$(redfish_curl "$user" "$password" "${bmc_base}/redfish/v1/Managers" | jq -r '.Members[0]."@odata.id"')
+        vmedia_uri="${bmc_base}${manager_id}/VirtualMedia"
+        cd_slot=$(redfish_curl "$user" "$password" "$vmedia_uri" | jq -r '.Members[] | select(."@odata.id" | test("[Cc][Dd]|[Dd][Vv][Dd]|2")) | ."@odata.id"' | head -1)
+
+        if [[ -z "$cd_slot" ]]; then
+            cd_slot=$(redfish_curl "$user" "$password" "$vmedia_uri" | jq -r '.Members[1]."@odata.id"')
+        fi
+
+        local current_image
+        current_image=$(redfish_curl "$user" "$password" "${bmc_base}${cd_slot}" | jq -r '.Inserted // false')
+        if [[ "$current_image" == "true" ]]; then
+            echo "Ejecting existing VirtualMedia from ${node_name}..."
+            redfish_curl "$user" "$password" "${bmc_base}${cd_slot}/Actions/VirtualMedia.EjectMedia" \
+                -d '{}'
+            sleep 2
+        fi
+
+        echo "Mounting ISO on ${node_name} via ${cd_slot}..."
+        redfish_curl "$user" "$password" "${bmc_base}${cd_slot}/Actions/VirtualMedia.InsertMedia" \
+            -d "{\"Image\": \"${iso_url}\", \"Inserted\": true, \"WriteProtected\": true}"
+
+        local inserted
+        inserted=$(redfish_curl "$user" "$password" "${bmc_base}${cd_slot}" | jq -r '.Inserted')
+        if [[ "$inserted" != "true" ]]; then
+            echo "WARNING: VirtualMedia reports Inserted=${inserted} for ${node_name}"
+        fi
+
+        redfish_curl "$user" "$password" "$systemurl" -X PATCH \
+            -d '{"Boot": {"BootSourceOverrideTarget": "Cd", "BootSourceOverrideEnabled": "Once"}}'
+
+        redfish_curl "$user" "$password" "$systemurl/Actions/ComputerSystem.Reset" \
+            -d '{"ResetType": "ForceRestart"}'
+
+        echo "Node ${node_name} booting from ISO"
+    done
+}
+
+function eject_agent_iso_baremetal() {
+    local total_nodes
+    total_nodes=$(jq '.nodes | length' "$NODES_FILE")
+
+    for (( i=0; i < total_nodes; i++ )); do
+        local address user password scheme system bmc_base
+        address=$(jq -r ".nodes[$i].driver_info.address" "$NODES_FILE")
+        user=$(jq -r ".nodes[$i].driver_info.username" "$NODES_FILE")
+        password=$(jq -r ".nodes[$i].driver_info.password" "$NODES_FILE")
+        local node_name
+        node_name=$(jq -r ".nodes[$i].name" "$NODES_FILE")
+
+        if [[ ! $address =~ ^(redfish.*://)(.*)$ ]]; then
+            continue
+        fi
+        scheme="https://"
+        system="${BASH_REMATCH[2]}"
+        if [[ ${BASH_REMATCH[1]} =~ http: ]]; then
+            scheme="http://"
+        fi
+        bmc_base="${scheme}${system%%/*}"
+
+        local manager_id vmedia_uri cd_slot
+        manager_id=$(redfish_curl "$user" "$password" "${bmc_base}/redfish/v1/Managers" | jq -r '.Members[0]."@odata.id"')
+        vmedia_uri="${bmc_base}${manager_id}/VirtualMedia"
+        cd_slot=$(redfish_curl "$user" "$password" "$vmedia_uri" | jq -r '.Members[] | select(."@odata.id" | test("[Cc][Dd]|[Dd][Vv][Dd]|2")) | ."@odata.id"' | head -1)
+
+        if [[ -z "$cd_slot" ]]; then
+            cd_slot=$(redfish_curl "$user" "$password" "$vmedia_uri" | jq -r '.Members[1]."@odata.id"')
+        fi
+
+        echo "Ejecting ISO from ${node_name}..."
+        redfish_curl "$user" "$password" "${bmc_base}${cd_slot}/Actions/VirtualMedia.EjectMedia" -d '{}'
+    done
+}
+
 function attach_appliance_diskimage() {
     set_file_acl
 
@@ -305,6 +410,10 @@ function wait_for_hosts_installed() {
 }
 
 function get_node0_ip() {
+  if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then
+    echo "${BAREMETAL_IPS%%,*}"
+    return
+  fi
   # shellcheck disable=SC2059
   node0_name=$(printf "${MASTER_HOSTNAME_FORMAT}" 0)
   node0_ip=$(sudo virsh net-dumpxml ostestbm | xmllint --xpath "string(//dns[*]/host/hostname[. = '${node0_name}']/../@ip)" -)
@@ -617,16 +726,29 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in
       fi
     fi
 
-    attach_agent_iso master "$NUM_MASTERS"
-    attach_agent_iso worker "$NUM_WORKERS"
-    attach_agent_iso arbiter "$NUM_ARBITERS"
-
-    if [[ "${ARCH}" == "aarch64" ]]; then
-        wait_for_hosts_installed || echo "WARNING: proceeding with CDROM eject despite host status issues"
-        eject_agent_iso master "$NUM_MASTERS"
-        eject_agent_iso worker "$NUM_WORKERS"
-        eject_agent_iso arbiter "$NUM_ARBITERS"
-        echo "aarch64: CDROM media ejected from all VMs"
+    if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then
+        # Stage the ISO where BAREMETAL_ISO_SERVER can serve it. The ISO is
+        # generated under OCP_DIR (relative to dev-scripts), but the HTTP
+        # server root may be elsewhere (e.g. WORKING_DIR under nginx).
+        iso_file="${OCP_DIR}/agent.$(uname -m).iso"
+        if [[ -f "$iso_file" ]]; then
+            serve_dir="${WORKING_DIR}/${CLUSTER_NAME}"
+            mkdir -p "$serve_dir"
+            ln -sf "$(realpath "$iso_file")" "${serve_dir}/agent.$(uname -m).iso"
+        fi
+        mount_agent_iso_baremetal
+    else
+        attach_agent_iso master "$NUM_MASTERS"
+        attach_agent_iso worker "$NUM_WORKERS"
+        attach_agent_iso arbiter "$NUM_ARBITERS"
+
+        if [[ "${ARCH}" == "aarch64" ]]; then
+            wait_for_hosts_installed || echo "WARNING: proceeding with CDROM eject despite host status issues"
+            eject_agent_iso master "$NUM_MASTERS"
+            eject_agent_iso worker "$NUM_WORKERS"
+            eject_agent_iso arbiter "$NUM_ARBITERS"
+            echo "aarch64: CDROM media ejected from all VMs"
+        fi
     fi
 
     ;;
@@ -759,6 +881,10 @@ fi
 
 wait_for_cluster_ready
 
+if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then
+    eject_agent_iso_baremetal
+fi
+
 if [ ! -z "${AGENT_DEPLOY_MCE}" ]; then
   mce_complete_deployment
 fi

agent/common.sh

@@ -100,6 +100,10 @@ if [ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ] ; then
 fi
 
 function getRendezvousIP() {
+    if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then
+        echo "${BAREMETAL_IPS%%,*}"
+        return
+    fi
     node_zero_mac_address=$(sudo virsh domiflist "${AGENT_RENDEZVOUS_NODE_HOSTNAME}" | awk '$3 == "ostestbm" {print $5}')
     # ip neigh can return multiple addresses (IPv4, link-local IPv6, global IPv6)
     # on separate lines. Filter to a single appropriate address.

config_example.sh

@@ -480,6 +480,37 @@ set -x
 #
 #export NODES_PLATFORM=baremetal
 
+# --- Baremetal ABI (Agent-Based Installation) variables ---
+# These are used when NODES_PLATFORM=baremetal with the agent installer (make agent).
+# All BAREMETAL_* vars except BAREMETAL_ISO_SERVER are auto-generated by
+# the adoption tool (make baremetal-adopt) into config_baremetal_fencing.sh.
+
+# BAREMETAL_IPS -
+# Comma-separated node IPs, order matches NODES_FILE entries.
+# Required when NODES_PLATFORM=baremetal.
+#
+#export BAREMETAL_IPS="10.1.155.10,10.1.155.11"
+
+# BAREMETAL_API_VIP -
+# API virtual IP for dnsmasq DNS on the provisioning host.
+# Required when NODES_PLATFORM=baremetal.
+#
+#export BAREMETAL_API_VIP="10.1.155.100"
+
+# BAREMETAL_INGRESS_VIP -
+# Ingress virtual IP. Defaults to BAREMETAL_API_VIP if unset
+# (common for 2-node TNF where API and ingress share a VIP).
+#
+#export BAREMETAL_INGRESS_VIP="10.1.155.101"
+
+# BAREMETAL_ISO_SERVER -
+# Full HTTP URL where the agent ISO is staged for Redfish VirtualMedia boot.
+# This is the only baremetal var NOT auto-generated by adoption — the user
+# must provide an HTTP server reachable from the BMCs.
+# Required when NODES_PLATFORM=baremetal with agent installer.
+#
+#export BAREMETAL_ISO_SERVER="http://10.1.235.49:8080/agent.x86_64.iso"
+
 # ENABLE_WORKLOAD_PARTITIONING -
 # Set to any non zero length string value to enable workload partitioning in the install config.
 #

network.sh

@@ -337,6 +337,10 @@ function set_api_and_ingress_vip() {
   if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then
       get_vips
       configure_dnsmasq "${API_VIPS}" "${INGRESS_VIPS}"
+  elif [ "${NODES_PLATFORM}" == "baremetal" ] && [ -n "${BAREMETAL_API_VIP:-}" ]; then
+      API_VIPS="${BAREMETAL_API_VIP}"
+      INGRESS_VIPS="${BAREMETAL_INGRESS_VIP:-${BAREMETAL_API_VIP}}"
+      configure_dnsmasq "${API_VIPS}" "${INGRESS_VIPS}"
   else
       # Specific for users *NOT* using devscript with KVM (virsh) for deploy. (Reads: baremetal)
       if [[ -z "${EXTERNAL_SUBNET_V4}" ]]; then