OPNET-787: Add optional top-of-rack BGP speaker for the baremetal network

02_configure_host.sh

@@ -512,3 +512,8 @@ if [ "${PERSISTENT_IMAGEREG}" == true ] ; then
     sudo systemctl start nfs-server
     sudo exportfs -a
 fi
+
+# Optionally run a top-of-rack BGP speaker on the baremetal network
+if [[ -n "${ENABLE_BGP_TOR:-}" ]]; then
+    bgp/configure_bgp_tor.sh
+fi

bgp/cleanup_bgp_tor.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+bgp_dir="$(dirname "$(readlink -f "$0")")"
+# shellcheck disable=SC1091
+source "${bgp_dir}/../common.sh"
+
+# Tears down the optional top-of-rack BGP speaker deployed by
+# configure_bgp_tor.sh. Tolerant of a missing container / firewall rule so
+# it can run unconditionally from host_cleanup.sh.
+
+BGP_TOR_NAME="bgp-tor"
+BGP_TOR_DIR="${WORKING_DIR}/bgp-tor"
+
+sudo podman rm -f "${BGP_TOR_NAME}" || true
+
+sudo firewall-cmd --zone=libvirt --permanent --remove-port=179/tcp || true
+sudo firewall-cmd --zone=libvirt --remove-port=179/tcp || true
+
+rm -rf "${BGP_TOR_DIR}"

bgp/configure_bgp_tor.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+bgp_dir="$(dirname "$(readlink -f "$0")")"
+# shellcheck disable=SC1091
+source "${bgp_dir}/../common.sh"
+# shellcheck disable=SC1091
+source "${bgp_dir}/../network.sh"
+
+# Deploys an FRR container on the host network acting as a top-of-rack BGP
+# speaker for the baremetal network. Cluster nodes (e.g. BGP-based VIP
+# management, enhancement openshift/enhancements#1982) peer with it via
+# dynamic neighbors; learned routes are installed into the host kernel by
+# zebra, so the hypervisor reaches advertised VIPs over the BGP paths.
+
+BGP_TOR_NAME="bgp-tor"
+BGP_TOR_DIR="${WORKING_DIR}/bgp-tor"
+
+mkdir -p "${BGP_TOR_DIR}"
+
+if [[ -n "${EXTERNAL_SUBNET_V4:-}" ]]; then
+    ROUTER_ID="$(nth_ip "${EXTERNAL_SUBNET_V4}" 1)"
+else
+    # BGP router IDs are always in IPv4 dotted-quad format; use a fixed
+    # documentation-range ID for IPv6-only deployments.
+    ROUTER_ID="192.0.2.1"
+fi
+
+# Accept dynamic BGP sessions from anywhere on the external subnet(s) and
+# only activate the address families a peer could exist on.
+LISTEN_RANGES=""
+ADDRESS_FAMILIES=""
+if [[ -n "${EXTERNAL_SUBNET_V4:-}" ]]; then
+    LISTEN_RANGES+=" bgp listen range ${EXTERNAL_SUBNET_V4} peer-group CLUSTER
+"
+    ADDRESS_FAMILIES+=" !
+ address-family ipv4 unicast
+  neighbor CLUSTER activate
+ exit-address-family
+"
+fi
+if [[ -n "${EXTERNAL_SUBNET_V6:-}" ]]; then
+    LISTEN_RANGES+=" bgp listen range ${EXTERNAL_SUBNET_V6} peer-group CLUSTER
+"
+    ADDRESS_FAMILIES+=" !
+ address-family ipv6 unicast
+  neighbor CLUSTER activate
+ exit-address-family
+"
+fi
+
+# "no bgp ebgp-requires-policy" relaxes RFC 8212; without it FRR 8+ refuses
+# to exchange routes with eBGP peers unless explicit policies are configured.
+cat > "${BGP_TOR_DIR}/frr.conf" <<EOF
+frr defaults traditional
+hostname ${BGP_TOR_NAME}
+log stdout informational
+!
+router bgp ${BGP_TOR_ASN}
+ bgp router-id ${ROUTER_ID}
+ bgp log-neighbor-changes
+ no bgp ebgp-requires-policy
+ neighbor CLUSTER peer-group
+ neighbor CLUSTER remote-as ${BGP_CLUSTER_ASN}
+${LISTEN_RANGES}${ADDRESS_FAMILIES}!
+EOF
+
+cat > "${BGP_TOR_DIR}/daemons" <<EOF
+zebra=yes
+bgpd=yes
+ospfd=no
+ospf6d=no
+ripd=no
+ripngd=no
+isisd=no
+pimd=no
+ldpd=no
+nhrpd=no
+eigrpd=no
+babeld=no
+sharpd=no
+pbrd=no
+bfdd=no
+fabricd=no
+vrrpd=no
+pathd=no
+
+vtysh_enable=yes
+zebra_options="  -A 127.0.0.1 -s 90000000"
+bgpd_options="   -A 127.0.0.1"
+EOF
+
+sudo firewall-cmd --zone=libvirt --permanent --add-port=179/tcp
+sudo firewall-cmd --zone=libvirt --add-port=179/tcp
+
+sudo podman run -d --replace --name "${BGP_TOR_NAME}" --net host --privileged \
+    -v "${BGP_TOR_DIR}/frr.conf:/etc/frr/frr.conf:z" \
+    -v "${BGP_TOR_DIR}/daemons:/etc/frr/daemons:z" \
+    "${BGP_TOR_IMAGE}"
+
+echo "BGP ToR speaker running: AS ${BGP_TOR_ASN}, router-id ${ROUTER_ID}, listening on port 179"

common.sh

@@ -708,3 +708,10 @@ if [ "${OPENSHIFT_CI}" == true ] ; then
 fi
 
 export ENABLE_CAPI_E2E=${ENABLE_CAPI_E2E:-}
+
+# Optional top-of-rack BGP speaker on the baremetal network
+# (see config_example.sh)
+export ENABLE_BGP_TOR=${ENABLE_BGP_TOR:-}
+export BGP_TOR_ASN=${BGP_TOR_ASN:-64513}
+export BGP_CLUSTER_ASN=${BGP_CLUSTER_ASN:-64512}
+export BGP_TOR_IMAGE=${BGP_TOR_IMAGE:-quay.io/frrouting/frr:9.1.0}

config_example.sh

@@ -222,6 +222,25 @@ set -x
 #export METALLB_IMAGE_BASE=
 #export METALLB_IMAGE_TAG=
 
+# ENABLE_BGP_TOR -
+# Deploy an FRR container on the host network acting as a top-of-rack BGP
+# speaker for the baremetal network. Cluster nodes can peer with it (e.g.
+# BGP-based VIP management, enhancement 1982) via dynamic neighbors; routes
+# learned from the cluster are installed on the host, so VIPs advertised
+# over BGP are reachable from the hypervisor.
+# The speaker listens on the whole external subnet; set the ASNs to match
+# your cluster's install-config platform.baremetal.bgpVIPConfig.
+# Default is unset.
+#
+#export ENABLE_BGP_TOR=true
+#
+# ASN of the ToR speaker (default 64513) and expected cluster ASN (64512):
+#export BGP_TOR_ASN=64513
+#export BGP_CLUSTER_ASN=64512
+#
+# FRR container image:
+#export BGP_TOR_IMAGE=quay.io/frrouting/frr:9.1.0
+
 # PERSISTENT_IMAGEREG
 # Default: false
 # Enables dev-scripts to setup and use nfs on the host as persistent storage

host_cleanup.sh

@@ -83,3 +83,6 @@ sudo pkill -f oc.*proxy
 
 # Remove image-reg nfsshare
 sudo rm -rf /etc/exports.d/dev-scripts.exports /opt/dev-scripts/nfsshare
+
+# Remove the optional top-of-rack BGP speaker
+bgp/cleanup_bgp_tor.sh || true