diff --git a/Cargo.lock b/Cargo.lock index 8e20b2ae33d..4d3de02078d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13701,6 +13701,7 @@ dependencies = [ "daft", "dpd-client 0.1.0 (git+https://github.com/oxidecomputer/dendrite?rev=187aee7de2e50f907099ea06c04aac96c3455665)", "dropshot 0.17.0", + "either", "futures", "gateway-client", "gateway-messages", @@ -13714,7 +13715,9 @@ dependencies = [ "omicron-test-utils", "omicron-uuid-kinds", "omicron-workspace-hack", + "oxnet", "proptest", + "rdb-types", "reqwest 0.13.2", "serde_json", "sled-agent-types", diff --git a/sled-agent/scrimlet-reconcilers/Cargo.toml b/sled-agent/scrimlet-reconcilers/Cargo.toml index 34e6992aba6..8e0c9ef4e06 100644 --- a/sled-agent/scrimlet-reconcilers/Cargo.toml +++ b/sled-agent/scrimlet-reconcilers/Cargo.toml @@ -11,6 +11,7 @@ workspace = true chrono.workspace = true daft.workspace = true dpd-client.workspace = true +either.workspace = true futures.workspace = true gateway-client.workspace = true gateway-types.workspace = true @@ -19,6 +20,8 @@ macaddr.workspace = true mg-admin-client.workspace = true omicron-common.workspace = true omicron-uuid-kinds.workspace = true +oxnet.workspace = true +rdb-types.workspace = true reqwest.workspace = true sled-agent-types.workspace = true slog.workspace = true diff --git a/sled-agent/scrimlet-reconcilers/src/lib.rs b/sled-agent/scrimlet-reconcilers/src/lib.rs index ecee043e8ee..a8acb314f26 100644 --- a/sled-agent/scrimlet-reconcilers/src/lib.rs +++ b/sled-agent/scrimlet-reconcilers/src/lib.rs @@ -59,6 +59,7 @@ pub use handle::ScrimletReconcilers; pub use handle::ScrimletReconcilersMode; pub use handle::SledAgentNetworkingInfo; pub use mgd_reconciler::MgdReconcilerStatus; +pub use mgd_reconciler::MgdStaticRouteReconcilerStatus; pub use status::DetermineSwitchSlotStatus; pub use status::ReconcilerActivationReason; pub use status::ReconcilerCurrentStatus; diff --git a/sled-agent/scrimlet-reconcilers/src/mgd_reconciler.rs b/sled-agent/scrimlet-reconcilers/src/mgd_reconciler.rs index fac5d4f0dca..e25a68de725 100644 --- a/sled-agent/scrimlet-reconcilers/src/mgd_reconciler.rs +++ b/sled-agent/scrimlet-reconcilers/src/mgd_reconciler.rs @@ -13,25 +13,31 @@ use sled_agent_types::system_networking::SystemNetworkingConfig; use slog::Logger; use std::time::Duration; +mod static_route_reconciler; + +pub use static_route_reconciler::MgdStaticRouteReconcilerStatus; + #[derive(Debug, Clone)] pub struct MgdReconcilerStatus { - pub todo_status: (), + pub static_routes_status: MgdStaticRouteReconcilerStatus, } impl slog::KV for MgdReconcilerStatus { fn serialize( &self, - _record: &slog::Record<'_>, + record: &slog::Record<'_>, serializer: &mut dyn slog::Serializer, ) -> slog::Result { - serializer.emit_str("mgd-reconciler".into(), "not yet implemented") + let Self { static_routes_status } = self; + static_routes_status.serialize(record, serializer)?; + Ok(()) } } #[derive(Debug)] pub(crate) struct MgdReconciler { - _client: Client, - _switch_slot: ThisSledSwitchSlot, + client: Client, + switch_slot: ThisSledSwitchSlot, } impl Reconciler for MgdReconciler { @@ -45,14 +51,22 @@ impl Reconciler for MgdReconciler { switch_slot: ThisSledSwitchSlot, parent_log: &Logger, ) -> Self { - Self { _client: mode.mgd_client(parent_log), _switch_slot: switch_slot } + Self { client: mode.mgd_client(parent_log), switch_slot } } async fn do_reconciliation( &mut self, - _system_networking_config: &SystemNetworkingConfig, - _log: &Logger, + system_networking_config: &SystemNetworkingConfig, + log: &Logger, ) -> Self::Status { - MgdReconcilerStatus { todo_status: () } + let static_routes_status = static_route_reconciler::reconcile( + &self.client, + &system_networking_config.rack_network_config, + self.switch_slot, + log, + ) + .await; + + MgdReconcilerStatus { static_routes_status } } } diff --git a/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/static_route_reconciler.rs b/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/static_route_reconciler.rs new file mode 100644 index 00000000000..0045b05f9c0 --- /dev/null +++ b/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/static_route_reconciler.rs @@ -0,0 +1,579 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Reconciliation of static routes within mgd. + +use crate::switch_zone_slot::ThisSledSwitchSlot; +use daft::BTreeSetDiff; +use daft::Diffable; +use either::Either; +use mg_admin_client::Client; +use mg_admin_client::types::AddStaticRoute4Request as MgdAddStaticRoute4Request; +use mg_admin_client::types::AddStaticRoute6Request as MgdAddStaticRoute6Request; +use mg_admin_client::types::DeleteStaticRoute4Request as MgdDeleteStaticRoute4Request; +use mg_admin_client::types::DeleteStaticRoute6Request as MgdDeleteStaticRoute6Request; +use mg_admin_client::types::Path as MgdPath; +use mg_admin_client::types::StaticRoute4 as MgdStaticRoute4; +use mg_admin_client::types::StaticRoute4List as MgdStaticRoute4List; +use mg_admin_client::types::StaticRoute6 as MgdStaticRoute6; +use mg_admin_client::types::StaticRoute6List as MgdStaticRoute6List; +use oxnet::IpNet; +use oxnet::IpNetParseError; +use oxnet::Ipv4Net; +use oxnet::Ipv6Net; +use rdb_types::Prefix4 as MgdPrefix4; +use rdb_types::Prefix6 as MgdPrefix6; +use sled_agent_types::early_networking::RackNetworkConfig; +use sled_agent_types::early_networking::RouteConfig; +use slog::Logger; +use slog::info; +use slog::warn; +use slog_error_chain::InlineErrorChain; +use std::collections::BTreeSet; +use std::collections::HashMap; +use std::iter; +use std::net::IpAddr; +use std::net::Ipv4Addr; +use std::net::Ipv6Addr; + +// This is the default RIB Priority used for static routes. This mirrors +// the const defined in maghemite in rdb/src/lib.rs. +const DEFAULT_RIB_PRIORITY_STATIC: u8 = 1; + +type MgdClientError = mg_admin_client::Error; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MgdStaticRouteReconcilerStatus { + /// Reconciliation was skipped because we couldn't fetch the current set of + /// static routes from MGD. + FailedReadingStaticRoutes(String), + + /// Reconciliation was skipped because we couldn't determine a plan for + /// changes to make. + /// + /// This should never happen - it indicates there's some faulty data + /// somewhere (either coming from mgd or in the rack network config). + FailedGeneratingPlan(String), + + /// Reconciliation completed successfully. + /// + /// mgd operations are performed in bulk, so each item here contains the + /// count of items involved. + Success { + unchanged: usize, + deleted_v4: usize, + deleted_v6: usize, + added_v4: usize, + added_v6: usize, + }, + + /// Reconciliation completed with at least one failure. + /// + /// mgd operations are performed in bulk, so each item here contains the + /// count of items applied on success. + PartialSuccess { + unchanged: usize, + delete_v4_result: Result, + delete_v6_result: Result, + add_v4_result: Result, + add_v6_result: Result, + }, +} + +impl slog::KV for MgdStaticRouteReconcilerStatus { + fn serialize( + &self, + _record: &slog::Record<'_>, + serializer: &mut dyn slog::Serializer, + ) -> slog::Result { + let skipped_key = "static-routes-reconciler-skipped"; + match self { + Self::FailedReadingStaticRoutes(reason) => { + serializer.emit_str(skipped_key.into(), reason) + } + Self::FailedGeneratingPlan(reason) => { + serializer.emit_str(skipped_key.into(), reason) + } + Self::Success { + unchanged, + deleted_v4, + deleted_v6, + added_v4, + added_v6, + } => { + serializer + .emit_usize("static-routes-unchanged".into(), *unchanged)?; + for (key, items) in [ + ("static-routes-delete-v4", deleted_v4), + ("static-routes-delete-v6", deleted_v6), + ("static-routes-add-v4", added_v4), + ("static-routes-add-v6", added_v6), + ] { + serializer.emit_arguments( + key.into(), + &format_args!("success ({items} routes affected)"), + )?; + } + Ok(()) + } + Self::PartialSuccess { + unchanged, + delete_v4_result, + delete_v6_result, + add_v4_result, + add_v6_result, + } => { + serializer + .emit_usize("static-routes-unchanged".into(), *unchanged)?; + for (key, result) in [ + ("static-routes-delete-v4", delete_v4_result), + ("static-routes-delete-v6", delete_v6_result), + ("static-routes-add-v4", add_v4_result), + ("static-routes-add-v6", add_v6_result), + ] { + match result { + Ok(items) => serializer.emit_arguments( + key.into(), + &format_args!("success ({items} routes affected)"), + )?, + Err(err) => { + serializer.emit_str(key.into(), &err)?; + } + } + } + Ok(()) + } + } + } +} + +pub(super) async fn reconcile( + client: &Client, + desired_config: &RackNetworkConfig, + our_switch_slot: ThisSledSwitchSlot, + log: &Logger, +) -> MgdStaticRouteReconcilerStatus { + let current_routes = match MgdCurrentRoutes::fetch(client).await { + Ok(routes) => routes, + Err(err) => { + return MgdStaticRouteReconcilerStatus::FailedReadingStaticRoutes( + format!( + "failed to read current static routes from mgd: {}", + InlineErrorChain::new(&err) + ), + ); + } + }; + + let plan = match ReconciliationPlan::new( + current_routes, + desired_config, + our_switch_slot, + log, + ) { + Ok(plan) => plan, + Err(err) => { + // Ensure `err` is actually a string; if it changes to a proper + // error type, we need to use `InlineErrorChain` here instead. + let err: &str = &err; + return MgdStaticRouteReconcilerStatus::FailedGeneratingPlan( + format!( + "failed to generate plan to apply static routes: {err}", + ), + ); + } + }; + + apply_plan(client, plan, log).await +} + +/// Apply the contents of `plan` to mgd via `client`. +async fn apply_plan( + client: &Client, + plan: ReconciliationPlan, + log: &Logger, +) -> MgdStaticRouteReconcilerStatus { + let ReconciliationPlan { unchanged_count, to_delete, to_add } = plan; + + // Delete before adding in case there are any conflicting routes for new + // routes we want to add. + // + // TODO-correctness If either of our deletes _fail_, should we still attempt + // to add? For now we do. + let (delete_v4_result, delete_v6_result) = { + // Assemble all the v4 and v6 route deletes into two requests. + let mut delete_v4_req = MgdDeleteStaticRoute4Request { + routes: MgdStaticRoute4List { list: Vec::new() }, + }; + let mut delete_v6_req = MgdDeleteStaticRoute6Request { + routes: MgdStaticRoute6List { list: Vec::new() }, + }; + for route in to_delete { + match route.into_mgd_static_route() { + MgdStaticRoute::V4(r) => delete_v4_req.routes.list.push(r), + MgdStaticRoute::V6(r) => delete_v6_req.routes.list.push(r), + } + } + ( + apply_bulk_operation_if_needed( + "deleting static v4 routes", + delete_v4_req.routes.list.len(), + || client.static_remove_v4_route(&delete_v4_req), + log, + ) + .await, + apply_bulk_operation_if_needed( + "deleting static v6 routes", + delete_v6_req.routes.list.len(), + || client.static_remove_v6_route(&delete_v6_req), + log, + ) + .await, + ) + }; + + let (add_v4_result, add_v6_result) = { + // Do the same for all route additions. + let mut add_v4_req = MgdAddStaticRoute4Request { + routes: MgdStaticRoute4List { list: Vec::new() }, + }; + let mut add_v6_req = MgdAddStaticRoute6Request { + routes: MgdStaticRoute6List { list: Vec::new() }, + }; + for route in to_add { + match route.into_mgd_static_route() { + MgdStaticRoute::V4(r) => add_v4_req.routes.list.push(r), + MgdStaticRoute::V6(r) => add_v6_req.routes.list.push(r), + } + } + ( + apply_bulk_operation_if_needed( + "adding static v4 routes", + add_v4_req.routes.list.len(), + || client.static_add_v4_route(&add_v4_req), + log, + ) + .await, + apply_bulk_operation_if_needed( + "adding static v6 routes", + add_v6_req.routes.list.len(), + || client.static_add_v6_route(&add_v6_req), + log, + ) + .await, + ) + }; + + match (&add_v4_result, &add_v6_result, &delete_v4_result, &delete_v6_result) + { + (Ok(added_v4), Ok(added_v6), Ok(deleted_v4), Ok(deleted_v6)) => { + MgdStaticRouteReconcilerStatus::Success { + unchanged: unchanged_count, + deleted_v4: *deleted_v4, + deleted_v6: *deleted_v6, + added_v4: *added_v4, + added_v6: *added_v6, + } + } + _ => MgdStaticRouteReconcilerStatus::PartialSuccess { + unchanged: unchanged_count, + delete_v4_result, + delete_v6_result, + add_v4_result, + add_v6_result, + }, + } +} + +// Helper to optionally perform `op`. +// +// If `nitems` is 0, `op` is never called, and we return +// `MgdStaticRouteBulkOperationResult::SkippedNoItems`. +// +// If `nitems` is not 0, we await the future returned by `op` and return either +// success or failure according to its returned value. +async fn apply_bulk_operation_if_needed( + description: &str, + nitems: usize, + op: F, + log: &Logger, +) -> Result +where + F: FnOnce() -> Fut, + Fut: Future>, +{ + if nitems == 0 { + return Ok(0); + } + + match op().await { + Ok(_) => { + info!(log, "{description} succeeded"; "num-routes" => %nitems); + Ok(nitems) + } + Err(err) => { + let err = InlineErrorChain::new(&err); + warn!(log, "{description} failed"; &err); + Err(format!("{description} failed: {err}")) + } + } +} + +#[derive(Debug, PartialEq)] +struct ReconciliationPlan { + // Count of routes that remained unchanged in this reconciliation. + unchanged_count: usize, + + // Routes to delete. + to_delete: BTreeSet, + + // Routes to add. + to_add: BTreeSet, +} + +impl ReconciliationPlan { + fn new( + mgd_current_routes: MgdCurrentRoutes, + config: &RackNetworkConfig, + our_switch_slot: ThisSledSwitchSlot, + log: &Logger, + ) -> Result { + // Convert current routes into diffable form. + let mgd_current_routes = match mgd_current_routes.try_into_diffable() { + Ok(routes) => routes, + Err(err) => { + return Err(format!( + "invalid route fetched from mgd: {}", + InlineErrorChain::new(&err) + )); + } + }; + + // Convert desired config into diffable form. + let desired_routes = config + .ports + .iter() + .filter(|port| port.switch == our_switch_slot) + .flat_map(|port| port.routes.iter()) + .map(DiffableStaticRoute::try_from) + .collect::, _>>()?; + + let BTreeSetDiff { common, added, removed } = + mgd_current_routes.diff(&desired_routes); + + let unchanged_count = common.len(); + let to_delete = removed.into_iter().copied().collect::>(); + let to_add = added.into_iter().copied().collect::>(); + + info!( + log, + "generated mgd static route reconciliation plan"; + "routes-unchanged" => unchanged_count, + "routes-to-delete" => to_delete.len(), + "routes-to-add" => to_add.len(), + ); + + Ok(Self { unchanged_count, to_delete, to_add }) + } +} + +#[derive(Debug, Default)] +struct MgdCurrentRoutes { + v4: HashMap>, + v6: HashMap>, +} + +impl MgdCurrentRoutes { + async fn fetch(client: &Client) -> Result { + let v4 = client.static_list_v4_routes().await?.into_inner(); + let v6 = client.static_list_v6_routes().await?.into_inner(); + + Ok(Self { v4, v6 }) + } + + fn try_into_diffable( + self, + ) -> Result, BadMgdRoute> { + let v4_routes = self.v4.into_iter().flat_map(|(prefix, paths)| { + DiffableStaticRoute::try_from_v4(prefix, paths) + }); + let v6_routes = self.v6.into_iter().flat_map(|(prefix, paths)| { + DiffableStaticRoute::try_from_v6(prefix, paths) + }); + + v4_routes.chain(v6_routes).collect() + } +} + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, daft::Diffable, +)] +#[cfg_attr(test, derive(test_strategy::Arbitrary))] +struct DiffableStaticRoute { + description: DiffableStaticRouteDescription, + vlan_id: Option, + priority: u8, +} + +impl TryFrom<&'_ RouteConfig> for DiffableStaticRoute { + type Error = String; + + fn try_from(route: &'_ RouteConfig) -> Result { + let description = match (route.nexthop, route.destination) { + (IpAddr::V4(nexthop), IpNet::V4(prefix)) => { + DiffableStaticRouteDescription::V4 { nexthop, prefix } + } + (IpAddr::V6(nexthop), IpNet::V6(prefix)) => { + DiffableStaticRouteDescription::V6 { nexthop, prefix } + } + (nexthop, prefix) => { + return Err(format!( + "rack network config route has mixed IP families \ + for nexthop and prefix: {nexthop}, {prefix}" + )); + } + }; + + Ok(Self { + description, + vlan_id: route.vlan_id, + // TODO The rack network config uses `None` as a sentinel for "the + // default priority". This isn't what we want long-term; see + // https://github.com/oxidecomputer/maghemite/issues/646#issuecomment-3948331208. + priority: route.rib_priority.unwrap_or(DEFAULT_RIB_PRIORITY_STATIC), + }) + } +} + +#[derive(Debug, thiserror::Error)] +enum BadMgdRoute { + #[error("could not parse {family} route prefix `{prefix}`")] + ParsePrefix { + family: &'static str, + prefix: String, + #[source] + err: IpNetParseError, + }, + #[error( + "expected {family} nexthop in prefix `{prefix}`, but got {nexthop}" + )] + BadNexthopFamily { family: &'static str, prefix: String, nexthop: String }, +} + +impl DiffableStaticRoute { + fn into_mgd_static_route(self) -> MgdStaticRoute { + match self.description { + DiffableStaticRouteDescription::V4 { nexthop, prefix } => { + MgdStaticRoute::V4(MgdStaticRoute4 { + nexthop, + prefix: MgdPrefix4 { + value: prefix.addr(), + length: prefix.width(), + }, + rib_priority: self.priority, + vlan_id: self.vlan_id, + }) + } + DiffableStaticRouteDescription::V6 { nexthop, prefix } => { + MgdStaticRoute::V6(MgdStaticRoute6 { + nexthop, + prefix: MgdPrefix6 { + value: prefix.addr(), + length: prefix.width(), + }, + rib_priority: self.priority, + vlan_id: self.vlan_id, + }) + } + } + } + + fn try_from_v4( + prefix: String, + paths: Vec, + ) -> impl Iterator> { + let prefix = match prefix.parse::() { + Ok(prefix) => prefix, + Err(err) => { + return Either::Left(iter::once(Err( + BadMgdRoute::ParsePrefix { family: "ipv4", prefix, err }, + ))); + } + }; + + Either::Right(paths.into_iter().map(move |path| { + let nexthop = match path.nexthop { + IpAddr::V4(ip) => ip, + IpAddr::V6(ip) => { + return Err(BadMgdRoute::BadNexthopFamily { + family: "ipv4", + prefix: prefix.to_string(), + nexthop: ip.to_string(), + }); + } + }; + + let description = + DiffableStaticRouteDescription::V4 { nexthop, prefix }; + + Ok(Self { + description, + vlan_id: path.vlan_id, + priority: path.rib_priority, + }) + })) + } + + fn try_from_v6( + prefix: String, + paths: Vec, + ) -> impl Iterator> { + let prefix = match prefix.parse::() { + Ok(prefix) => prefix, + Err(err) => { + return Either::Left(iter::once(Err( + BadMgdRoute::ParsePrefix { family: "ipv6", prefix, err }, + ))); + } + }; + + Either::Right(paths.into_iter().map(move |path| { + let nexthop = match path.nexthop { + IpAddr::V6(ip) => ip, + IpAddr::V4(ip) => { + return Err(BadMgdRoute::BadNexthopFamily { + family: "ipv6", + prefix: prefix.to_string(), + nexthop: ip.to_string(), + }); + } + }; + + let description = + DiffableStaticRouteDescription::V6 { nexthop, prefix }; + + Ok(Self { + description, + vlan_id: path.vlan_id, + priority: path.rib_priority, + }) + })) + } +} + +enum MgdStaticRoute { + V4(MgdStaticRoute4), + V6(MgdStaticRoute6), +} + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, daft::Diffable, +)] +enum DiffableStaticRouteDescription { + V4 { nexthop: Ipv4Addr, prefix: Ipv4Net }, + V6 { nexthop: Ipv6Addr, prefix: Ipv6Net }, +} + +#[cfg(test)] +mod tests; diff --git a/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/static_route_reconciler/tests.rs b/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/static_route_reconciler/tests.rs new file mode 100644 index 00000000000..dc4786b6e32 --- /dev/null +++ b/sled-agent/scrimlet-reconcilers/src/mgd_reconciler/static_route_reconciler/tests.rs @@ -0,0 +1,1205 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::*; +use crate::switch_zone_slot::ThisSledSwitchSlot; +use gateway_messages::SpPort; +use mg_admin_client::types::Path as MgdPath; +use omicron_test_utils::dev; +use proptest::prelude::Arbitrary; +use proptest::prelude::Just; +use proptest::prelude::any; +use proptest::prelude::proptest as proptest_macro; +use proptest::strategy::Strategy; +use sled_agent_types::early_networking::LinkSpeed; +use sled_agent_types::early_networking::PortConfig; +use sled_agent_types::early_networking::RackNetworkConfig; +use sled_agent_types::early_networking::RouteConfig; +use sled_agent_types::early_networking::SwitchSlot; +use sled_agent_types::early_networking::UplinkIpNet; +use std::collections::BTreeMap; +use test_strategy::proptest; +use tokio::task::block_in_place; + +impl Arbitrary for DiffableStaticRouteDescription { + type Parameters = (); + type Strategy = proptest::prelude::BoxedStrategy; + + fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy { + // mgd enforces the same requirements as `UplinkIpNet` for prefixes, so + // we'll lean on its `Arbitrary` implementation to avoid invalid IPs + // like loopback and ipv4-mapped ipv6 addrs + // + // We have to do some work here to generate a matching v4-v4 or v6-v6 + // nexthop address; that address currently has no requirements (although + // it may eventually: + // ), so we + // generate any arbitrary nexthop that matches the protocol family. + any::() + .prop_flat_map(|prefix| { + let prefix = IpNet::from(prefix); + + let (prefix, nexthop_strategy) = match prefix { + IpNet::V4(prefix) => { + let prefix = Ipv4Net::new( + prefix.addr() & prefix.mask_addr(), + prefix.width(), + ) + .expect("still valid after applying mask"); + + ( + IpNet::from(prefix), + any::().prop_map(IpAddr::from).boxed(), + ) + } + IpNet::V6(prefix) => { + let prefix = Ipv6Net::new( + prefix.addr() & prefix.mask_addr(), + prefix.width(), + ) + .expect("still valid after applying mask"); + + ( + IpNet::from(prefix), + any::().prop_map(IpAddr::from).boxed(), + ) + } + }; + (Just(prefix), nexthop_strategy) + }) + .prop_map(|(prefix, nexthop)| match (prefix, nexthop) { + (IpNet::V4(prefix), IpAddr::V4(nexthop)) => { + Self::V4 { nexthop, prefix } + } + (IpNet::V6(prefix), IpAddr::V6(nexthop)) => { + Self::V6 { nexthop, prefix } + } + (IpNet::V4(_), IpAddr::V6(_)) + | (IpNet::V6(_), IpAddr::V4(_)) => { + unreachable!("invalid v4/v6 combo in Arbitrary impl") + } + }) + .boxed() + } +} + +/// Build a minimal `PortConfig` on the given switch with the given routes. +fn port_config(switch: SwitchSlot, routes: Vec) -> PortConfig { + PortConfig { + routes, + addresses: Vec::new(), + switch, + port: "qsfp0".to_string(), + uplink_port_speed: LinkSpeed::Speed100G, + uplink_port_fec: None, + bgp_peers: Vec::new(), + autoneg: false, + lldp: None, + tx_eq: None, + } +} + +fn rack_config(ports: Vec) -> RackNetworkConfig { + RackNetworkConfig { + rack_subnet: "fd00::/48".parse().unwrap(), + infra_ip_first: "10.0.0.1".parse().unwrap(), + infra_ip_last: "10.0.0.100".parse().unwrap(), + ports, + bgp: Vec::new(), + bfd: Vec::new(), + } +} + +fn v4_route( + destination: &str, + nexthop: &str, + vlan_id: Option, + rib_priority: Option, +) -> RouteConfig { + RouteConfig { + destination: destination.parse().unwrap(), + nexthop: nexthop.parse().unwrap(), + vlan_id, + rib_priority, + } +} + +fn v6_route( + destination: &str, + nexthop: &str, + vlan_id: Option, + rib_priority: Option, +) -> RouteConfig { + RouteConfig { + destination: destination.parse().unwrap(), + nexthop: nexthop.parse().unwrap(), + vlan_id, + rib_priority, + } +} + +fn mgd_path(nexthop: &str, rib_priority: u8, vlan_id: Option) -> MgdPath { + MgdPath { + nexthop: nexthop.parse().unwrap(), + rib_priority, + vlan_id, + shutdown: false, + bgp: None, + nexthop_interface: None, + } +} + +fn mgd_routes( + v4: Vec<(&str, Vec)>, + v6: Vec<(&str, Vec)>, +) -> MgdCurrentRoutes { + MgdCurrentRoutes { + v4: v4.into_iter().map(|(k, v)| (k.to_string(), v)).collect(), + v6: v6.into_iter().map(|(k, v)| (k.to_string(), v)).collect(), + } +} + +#[test] +fn plan_all_unchanged() { + let logctx = dev::test_setup_log("plan_all_unchanged"); + let log = &logctx.log; + + // Desired: one v4 route on Switch0. + let config = rack_config(vec![port_config( + SwitchSlot::Switch0, + vec![v4_route("10.0.0.0/24", "10.0.0.1", None, None)], + )]); + + // mgd has the same route (with default priority filled in). + let current = mgd_routes( + vec![("10.0.0.0/24", vec![mgd_path("10.0.0.1", 1, None)])], + vec![], + ); + + let plan = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + assert_eq!(plan.unchanged_count, 1); + assert!(plan.to_delete.is_empty()); + assert!(plan.to_add.is_empty()); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_add_all() { + let logctx = dev::test_setup_log("plan_add_all"); + let log = &logctx.log; + + // Desired: two routes (one v4, one v6). + let config = rack_config(vec![port_config( + SwitchSlot::Switch0, + vec![ + v4_route("10.0.0.0/24", "10.0.0.1", None, Some(5)), + v6_route("2001:db8::/64", "2001:db8::1", None, Some(3)), + ], + )]); + + // mgd is empty. + let current = mgd_routes(vec![], vec![]); + + let plan = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + assert_eq!(plan.unchanged_count, 0); + assert!(plan.to_delete.is_empty()); + assert_eq!(plan.to_add.len(), 2); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_delete_all() { + let logctx = dev::test_setup_log("plan_delete_all"); + let log = &logctx.log; + + // Desired: no routes. + let config = rack_config(vec![]); + + // mgd has two routes. + let current = mgd_routes( + vec![("10.0.0.0/24", vec![mgd_path("10.0.0.1", 1, None)])], + vec![("2001:db8::/64", vec![mgd_path("2001:db8::1", 1, None)])], + ); + + let plan = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + assert_eq!(plan.unchanged_count, 0); + assert_eq!(plan.to_delete.len(), 2); + assert!(plan.to_add.is_empty()); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_mix() { + let logctx = dev::test_setup_log("plan_mix"); + let log = &logctx.log; + + // Desired: keep 10.0.0.0/24, add 10.1.0.0/24, remove 10.2.0.0/24. + let config = rack_config(vec![port_config( + SwitchSlot::Switch0, + vec![ + v4_route("10.0.0.0/24", "10.0.0.1", None, None), + v4_route("10.1.0.0/24", "10.1.0.1", None, None), + ], + )]); + + let current = mgd_routes( + vec![ + ("10.0.0.0/24", vec![mgd_path("10.0.0.1", 1, None)]), + ("10.2.0.0/24", vec![mgd_path("10.2.0.1", 1, None)]), + ], + vec![], + ); + + let plan = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + assert_eq!(plan.unchanged_count, 1); + assert_eq!(plan.to_delete.len(), 1); + assert_eq!(plan.to_add.len(), 1); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_filters_other_switch_slot() { + let logctx = dev::test_setup_log("plan_filters_other_switch_slot"); + let log = &logctx.log; + + // Desired config has routes on both Switch0 and Switch1. Our reconciler is + // Switch0 (TEST_FAKE), so Switch1 routes should be ignored entirely. + let config = rack_config(vec![ + port_config( + SwitchSlot::Switch0, + vec![v4_route("10.0.0.0/24", "10.0.0.1", None, None)], + ), + port_config( + SwitchSlot::Switch1, + vec![v4_route("10.1.0.0/24", "10.1.0.1", None, None)], + ), + ]); + + // mgd is empty. + let current = mgd_routes(vec![], vec![]); + + let plan = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + // Only the Switch0 route should be added. + assert_eq!(plan.unchanged_count, 0); + assert!(plan.to_delete.is_empty()); + assert_eq!(plan.to_add.len(), 1); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_default_rib_priority() { + let logctx = dev::test_setup_log("plan_default_rib_priority"); + let log = &logctx.log; + + // Desired route has rib_priority = None (should default to 1). + let config = rack_config(vec![port_config( + SwitchSlot::Switch0, + vec![v4_route("10.0.0.0/24", "10.0.0.1", None, None)], + )]); + + // mgd has the same route with priority 1 (the default). + let current = mgd_routes( + vec![( + "10.0.0.0/24", + vec![mgd_path("10.0.0.1", DEFAULT_RIB_PRIORITY_STATIC, None)], + )], + vec![], + ); + + let plan = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + // Should be unchanged because None defaults to 1. + assert_eq!(plan.unchanged_count, 1); + assert!(plan.to_delete.is_empty()); + assert!(plan.to_add.is_empty()); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_priority_change_is_delete_plus_add() { + let logctx = dev::test_setup_log("plan_priority_change_is_delete_plus_add"); + let log = &logctx.log; + + // Desired: priority 5 for a route. + let config = rack_config(vec![port_config( + SwitchSlot::Switch0, + vec![v4_route("10.0.0.0/24", "10.0.0.1", None, Some(5))], + )]); + + // mgd has the same route but with priority 1. + let current = mgd_routes( + vec![("10.0.0.0/24", vec![mgd_path("10.0.0.1", 1, None)])], + vec![], + ); + + let plan = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + // Different priority means old route deleted, new route added. + assert_eq!(plan.unchanged_count, 0); + assert_eq!(plan.to_delete.len(), 1); + assert_eq!(plan.to_add.len(), 1); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_vlan_id_matters() { + let logctx = dev::test_setup_log("plan_vlan_id_matters"); + let log = &logctx.log; + + // Desired: route with vlan_id = Some(100). + let config = rack_config(vec![port_config( + SwitchSlot::Switch0, + vec![v4_route("10.0.0.0/24", "10.0.0.1", Some(100), None)], + )]); + + // mgd has the same route but with no vlan_id. + let current = mgd_routes( + vec![("10.0.0.0/24", vec![mgd_path("10.0.0.1", 1, None)])], + vec![], + ); + + let plan = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + // Different vlan_id: delete the old, add the new. + assert_eq!(plan.unchanged_count, 0); + assert_eq!(plan.to_delete.len(), 1); + assert_eq!(plan.to_add.len(), 1); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_multiple_nexthops_per_prefix() { + let logctx = dev::test_setup_log("plan_multiple_nexthops_per_prefix"); + let log = &logctx.log; + + // Desired: two nexthops for the same prefix. + let config = rack_config(vec![port_config( + SwitchSlot::Switch0, + vec![ + v4_route("10.0.0.0/24", "10.0.0.1", None, None), + v4_route("10.0.0.0/24", "10.0.0.2", None, None), + ], + )]); + + // mgd has one of them. + let current = mgd_routes( + vec![("10.0.0.0/24", vec![mgd_path("10.0.0.1", 1, None)])], + vec![], + ); + + let plan = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + assert_eq!(plan.unchanged_count, 1); + assert!(plan.to_delete.is_empty()); + assert_eq!(plan.to_add.len(), 1); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_rejects_bad_mgd_prefix() { + let logctx = dev::test_setup_log("plan_rejects_bad_mgd_prefix"); + let log = &logctx.log; + + let config = rack_config(vec![]); + + // mgd returns an unparseable prefix. + let current = mgd_routes( + vec![("not-a-prefix", vec![mgd_path("10.0.0.1", 1, None)])], + vec![], + ); + + let err = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect_err("plan should fail with bad prefix"); + + assert!( + err.contains("invalid route fetched from mgd"), + "unexpected error: {err}", + ); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_rejects_mixed_ip_families_in_config() { + let logctx = + dev::test_setup_log("plan_rejects_mixed_ip_families_in_config"); + let log = &logctx.log; + + // A route with a v4 nexthop and a v6 destination. + let config = rack_config(vec![port_config( + SwitchSlot::Switch0, + vec![RouteConfig { + destination: "2001:db8::/64".parse().unwrap(), + nexthop: "10.0.0.1".parse().unwrap(), + vlan_id: None, + rib_priority: None, + }], + )]); + + let current = mgd_routes(vec![], vec![]); + + let err = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect_err("plan should fail with mixed families"); + + assert!(err.contains("mixed IP families"), "unexpected error: {err}",); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_rejects_wrong_nexthop_family_from_mgd() { + let logctx = + dev::test_setup_log("plan_rejects_wrong_nexthop_family_from_mgd"); + let log = &logctx.log; + + let config = rack_config(vec![]); + + // mgd has a v4 prefix but a v6 nexthop. + let current = mgd_routes( + vec![("10.0.0.0/24", vec![mgd_path("2001:db8::1", 1, None)])], + vec![], + ); + + let err = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect_err("plan should fail with wrong nexthop family"); + + assert!( + err.contains("invalid route fetched from mgd"), + "unexpected error: {err}", + ); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_both_empty() { + let logctx = dev::test_setup_log("plan_both_empty"); + let log = &logctx.log; + + let config = rack_config(vec![]); + let current = mgd_routes(vec![], vec![]); + + let plan = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + assert_eq!(plan.unchanged_count, 0); + assert!(plan.to_delete.is_empty()); + assert!(plan.to_add.is_empty()); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_v6_routes() { + let logctx = dev::test_setup_log("plan_v6_routes"); + let log = &logctx.log; + + // Desired: one v6 route. + let config = rack_config(vec![port_config( + SwitchSlot::Switch0, + vec![v6_route("2001:db8::/64", "2001:db8::1", None, Some(2))], + )]); + + // mgd has a different v6 route. + let current = mgd_routes( + vec![], + vec![("fd00::/48", vec![mgd_path("fd00::1", 1, None)])], + ); + + let plan = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + assert_eq!(plan.unchanged_count, 0); + assert_eq!(plan.to_delete.len(), 1); + assert_eq!(plan.to_add.len(), 1); + + logctx.cleanup_successful(); +} + +#[test] +fn plan_routes_from_multiple_ports() { + let logctx = dev::test_setup_log("plan_routes_from_multiple_ports"); + let log = &logctx.log; + + // Desired: two ports on our switch, each with a route. + let config = rack_config(vec![ + port_config( + SwitchSlot::Switch0, + vec![v4_route("10.0.0.0/24", "10.0.0.1", None, None)], + ), + port_config( + SwitchSlot::Switch0, + vec![v4_route("10.1.0.0/24", "10.1.0.1", None, None)], + ), + ]); + + // mgd is empty. + let current = mgd_routes(vec![], vec![]); + + let plan = ReconciliationPlan::new( + current, + &config, + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + // Both routes from both ports should be added. + assert_eq!(plan.unchanged_count, 0); + assert!(plan.to_delete.is_empty()); + assert_eq!(plan.to_add.len(), 2); + + logctx.cleanup_successful(); +} + +#[derive(Debug, Clone, Copy, test_strategy::Arbitrary)] +enum StaticRouteTestInput { + // route only exists in mgd + MgdOnly, + + // route only exists in desired config for switch 0 or switch 1 + DesiredConfigOnly(SwitchSlot), + + // route exists in mgd and our desired switch 0 config + MgdAndSwitch0, + + // route exists in mgd and our desired switch 0 config, but we want to + // change the priority + // + // The `u8` here is the new priority we want in the desired config; the + // prior mgd priority will come from our parent `StaticRouteTestValue`. They + // may randomly be the same in some proptest runs. + MgdAndSwitch0NewPriority(u8), +} + +// mgd's API for describing static routes is looser than it should be; we'll +// generate a map using keys of the tuple (prefix, nexthop, vlan_id). This is +// more consistent with what mgd does internally: +// . +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, test_strategy::Arbitrary, +)] +struct StaticRouteTestKey { + description: DiffableStaticRouteDescription, + vlan_id: Option, +} + +#[derive(Debug, Clone, Copy, test_strategy::Arbitrary)] +struct StaticRouteTestValue { + input: StaticRouteTestInput, + priority: u8, +} + +impl StaticRouteTestKey { + fn to_route(&self, priority: u8) -> DiffableStaticRoute { + DiffableStaticRoute { + description: self.description, + vlan_id: self.vlan_id, + priority, + } + } +} + +#[derive(Debug, Clone, test_strategy::Arbitrary)] +struct TestInput { + routes: BTreeMap, +} + +impl From<&'_ DiffableStaticRoute> for RouteConfig { + fn from(value: &'_ DiffableStaticRoute) -> Self { + let destination = match value.description { + DiffableStaticRouteDescription::V4 { prefix, .. } => { + IpNet::V4(prefix) + } + DiffableStaticRouteDescription::V6 { prefix, .. } => { + IpNet::V6(prefix) + } + }; + let nexthop = match value.description { + DiffableStaticRouteDescription::V4 { nexthop, .. } => { + IpAddr::V4(nexthop) + } + DiffableStaticRouteDescription::V6 { nexthop, .. } => { + IpAddr::V6(nexthop) + } + }; + Self { + destination, + nexthop, + vlan_id: value.vlan_id, + rib_priority: Some(value.priority), + } + } +} + +impl TestInput { + fn initial_mgd_routes(&self) -> MgdCurrentRoutes { + Self::routes_from_iter( + self.routes + .iter() + .filter(|(_, val)| match val.input { + StaticRouteTestInput::MgdOnly + | StaticRouteTestInput::MgdAndSwitch0 + | StaticRouteTestInput::MgdAndSwitch0NewPriority(_) => true, + StaticRouteTestInput::DesiredConfigOnly(_) => false, + }) + .map(|(key, val)| key.to_route(val.priority)), + ) + } + + fn post_reconciliation_mgd_routes(&self) -> MgdCurrentRoutes { + Self::routes_from_iter(self.routes.iter().filter_map(|(key, val)| { + match val.input { + StaticRouteTestInput::DesiredConfigOnly( + SwitchSlot::Switch1, + ) + | StaticRouteTestInput::MgdOnly => None, + + StaticRouteTestInput::DesiredConfigOnly( + SwitchSlot::Switch0, + ) + | StaticRouteTestInput::MgdAndSwitch0 => { + Some(key.to_route(val.priority)) + } + StaticRouteTestInput::MgdAndSwitch0NewPriority(priority) => { + Some(key.to_route(priority)) + } + } + })) + } + + fn routes_from_iter( + mgd_routes: impl Iterator, + ) -> MgdCurrentRoutes { + let mut v4: HashMap> = HashMap::new(); + let mut v6: HashMap> = HashMap::new(); + for route in mgd_routes { + match route.description { + DiffableStaticRouteDescription::V4 { nexthop, prefix } => { + v4.entry(prefix.to_string()).or_default().push(MgdPath { + bgp: None, + nexthop: IpAddr::V4(nexthop), + nexthop_interface: None, + rib_priority: route.priority, + shutdown: false, + vlan_id: route.vlan_id, + }); + } + DiffableStaticRouteDescription::V6 { nexthop, prefix } => { + v6.entry(prefix.to_string()).or_default().push(MgdPath { + bgp: None, + nexthop: IpAddr::V6(nexthop), + nexthop_interface: None, + rib_priority: route.priority, + shutdown: false, + vlan_id: route.vlan_id, + }); + } + } + } + + MgdCurrentRoutes { v4, v6 } + } + + fn desired_rack_config(&self) -> RackNetworkConfig { + let switch0 = + self.routes.iter().filter_map(|(key, val)| match val.input { + StaticRouteTestInput::MgdOnly + | StaticRouteTestInput::DesiredConfigOnly( + SwitchSlot::Switch1, + ) => None, + StaticRouteTestInput::MgdAndSwitch0 + | StaticRouteTestInput::DesiredConfigOnly( + SwitchSlot::Switch0, + ) => Some(RouteConfig::from(&key.to_route(val.priority))), + StaticRouteTestInput::MgdAndSwitch0NewPriority(priority) => { + Some(RouteConfig::from(&key.to_route(priority))) + } + }); + let switch1 = + self.routes.iter().filter_map(|(key, val)| match val.input { + StaticRouteTestInput::MgdOnly + | StaticRouteTestInput::MgdAndSwitch0 + | StaticRouteTestInput::MgdAndSwitch0NewPriority(_) + | StaticRouteTestInput::DesiredConfigOnly( + SwitchSlot::Switch0, + ) => None, + StaticRouteTestInput::DesiredConfigOnly( + SwitchSlot::Switch1, + ) => Some(RouteConfig::from(&key.to_route(val.priority))), + }); + rack_config(vec![ + port_config(SwitchSlot::Switch0, switch0.collect()), + port_config(SwitchSlot::Switch1, switch1.collect()), + ]) + } + + fn expected_unchanged_count(&self) -> usize { + self.routes + .values() + .filter(|val| match val.input { + StaticRouteTestInput::MgdOnly + | StaticRouteTestInput::DesiredConfigOnly(_) => false, + StaticRouteTestInput::MgdAndSwitch0 => true, + StaticRouteTestInput::MgdAndSwitch0NewPriority(priority) => { + val.priority == priority + } + }) + .count() + } + + fn expected_to_add(&self) -> BTreeSet { + self.routes + .iter() + .filter_map(|(key, val)| match val.input { + StaticRouteTestInput::MgdOnly + | StaticRouteTestInput::MgdAndSwitch0 + | StaticRouteTestInput::DesiredConfigOnly( + SwitchSlot::Switch1, + ) => None, + StaticRouteTestInput::DesiredConfigOnly( + SwitchSlot::Switch0, + ) => Some(key.to_route(val.priority)), + StaticRouteTestInput::MgdAndSwitch0NewPriority(priority) => { + if priority != val.priority { + Some(key.to_route(priority)) + } else { + None + } + } + }) + .collect() + } + + fn expected_to_delete(&self) -> BTreeSet { + self.routes + .iter() + .filter_map(|(key, val)| match val.input { + StaticRouteTestInput::MgdAndSwitch0 + | StaticRouteTestInput::DesiredConfigOnly(_) => None, + StaticRouteTestInput::MgdOnly => { + Some(key.to_route(val.priority)) + } + StaticRouteTestInput::MgdAndSwitch0NewPriority(priority) => { + if priority != val.priority { + Some(key.to_route(val.priority)) + } else { + None + } + } + }) + .collect() + } + + fn expected_reconciliation_status(&self) -> MgdStaticRouteReconcilerStatus { + let mut unchanged = 0; + let mut deleted_v4 = 0; + let mut deleted_v6 = 0; + let mut added_v4 = 0; + let mut added_v6 = 0; + + for (key, val) in &self.routes { + match val.input { + StaticRouteTestInput::MgdOnly => match key.description { + DiffableStaticRouteDescription::V4 { .. } => { + deleted_v4 += 1; + } + DiffableStaticRouteDescription::V6 { .. } => { + deleted_v6 += 1; + } + }, + StaticRouteTestInput::DesiredConfigOnly( + SwitchSlot::Switch0, + ) => match key.description { + DiffableStaticRouteDescription::V4 { .. } => { + added_v4 += 1; + } + DiffableStaticRouteDescription::V6 { .. } => { + added_v6 += 1; + } + }, + StaticRouteTestInput::MgdAndSwitch0NewPriority(priority) => { + if priority == val.priority { + unchanged += 1; + } else { + match key.description { + DiffableStaticRouteDescription::V4 { .. } => { + deleted_v4 += 1; + added_v4 += 1; + } + DiffableStaticRouteDescription::V6 { .. } => { + deleted_v6 += 1; + added_v6 += 1; + } + } + } + } + StaticRouteTestInput::DesiredConfigOnly( + SwitchSlot::Switch1, + ) => (), + StaticRouteTestInput::MgdAndSwitch0 => { + unchanged += 1; + } + } + } + + MgdStaticRouteReconcilerStatus::Success { + unchanged, + deleted_v4, + deleted_v6, + added_v4, + added_v6, + } + } +} + +#[proptest] +fn proptest_plan(input: TestInput) { + let logctx = dev::test_setup_log("proptest_plan"); + let log = &logctx.log; + + let plan = ReconciliationPlan::new( + input.initial_mgd_routes(), + &input.desired_rack_config(), + ThisSledSwitchSlot::TEST_FAKE, + log, + ) + .expect("plan should succeed"); + + let ReconciliationPlan { unchanged_count, to_delete, to_add } = plan; + + assert_eq!(unchanged_count, input.expected_unchanged_count()); + assert_eq!(to_delete, input.expected_to_delete(), "incorrect to_delete"); + assert_eq!(to_add, input.expected_to_add(), "incorrect to_add"); + + logctx.cleanup_successful(); +} + +struct MgdStaticRouteLists { + v4: MgdStaticRoute4List, + v6: MgdStaticRoute6List, +} + +impl From for MgdStaticRouteLists { + fn from(value: MgdCurrentRoutes) -> Self { + Self { + v4: MgdStaticRoute4List { + list: value + .v4 + .into_iter() + .flat_map(|(prefix, paths)| { + let prefix = prefix + .parse::() + .expect("valid MgdPrefix4"); + paths.into_iter().map(move |path| { + let nexthop = match path.nexthop { + IpAddr::V4(ip) => ip, + IpAddr::V6(_) => { + panic!("invalid path: v4 with v6 nexthop") + } + }; + MgdStaticRoute4 { + nexthop, + prefix, + rib_priority: path.rib_priority, + vlan_id: path.vlan_id, + } + }) + }) + .collect(), + }, + v6: MgdStaticRoute6List { + list: value + .v6 + .into_iter() + .flat_map(|(prefix, paths)| { + let prefix = prefix + .parse::() + .expect("valid MgdPrefix6"); + paths.into_iter().map(move |path| { + let nexthop = match path.nexthop { + IpAddr::V6(ip) => ip, + IpAddr::V4(_) => { + panic!("invalid path: v6 with v4 nexthop") + } + }; + MgdStaticRoute6 { + nexthop, + prefix, + rib_priority: path.rib_priority, + vlan_id: path.vlan_id, + } + }) + }) + .collect(), + }, + } + } +} + +async fn remove_all_current_routes( + client: &Client, + current_routes: MgdCurrentRoutes, +) -> Result<(), MgdClientError> { + let MgdStaticRouteLists { v4, v6 } = current_routes.into(); + + client + .static_remove_v4_route(&MgdDeleteStaticRoute4Request { routes: v4 }) + .await + .expect("removed v4 routes"); + client + .static_remove_v6_route(&MgdDeleteStaticRoute6Request { routes: v6 }) + .await + .expect("removed v6 routes"); + + Ok(()) +} + +async fn create_initial_routes( + client: &Client, + initial_routes: MgdCurrentRoutes, +) -> Result<(), MgdClientError> { + let MgdStaticRouteLists { v4, v6 } = initial_routes.into(); + + client + .static_add_v4_route(&MgdAddStaticRoute4Request { routes: v4 }) + .await + .expect("added v4 routes"); + client + .static_add_v6_route(&MgdAddStaticRoute6Request { routes: v6 }) + .await + .expect("added v6 routes"); + + Ok(()) +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +struct ComparableMgdPath<'a> { + nexthop: &'a IpAddr, + nexthop_interface: &'a Option, + rib_priority: &'a u8, + shutdown: &'a bool, + vlan_id: &'a Option, +} + +impl<'a> From<&'a MgdPath> for ComparableMgdPath<'a> { + fn from(value: &'a MgdPath) -> Self { + let MgdPath { + bgp, + nexthop, + nexthop_interface, + rib_priority, + shutdown, + vlan_id, + } = value; + assert!(bgp.is_none(), "static route tests never use BGP"); + + Self { nexthop, nexthop_interface, rib_priority, shutdown, vlan_id } + } +} + +fn assert_routes_eq( + actual: MgdCurrentRoutes, + expected: MgdCurrentRoutes, + description: &str, +) { + // convert to BTree{Map,Set} for nicer output from assert_eq!() on failure + #[derive(Debug, PartialEq, Eq)] + struct BTreeMgdCurrentRoutes<'a> { + v4: BTreeMap<&'a str, BTreeSet>>, + v6: BTreeMap<&'a str, BTreeSet>>, + } + + impl<'a> From<&'a MgdCurrentRoutes> for BTreeMgdCurrentRoutes<'a> { + fn from(routes: &'a MgdCurrentRoutes) -> Self { + fn convert_map<'b>( + input: &'b HashMap>, + ) -> BTreeMap<&'b str, BTreeSet>> + { + let mut output: BTreeMap<_, BTreeSet<_>> = BTreeMap::new(); + for (k, paths) in input { + let paths = paths.iter().map(ComparableMgdPath::from); + output.insert(k.as_str(), paths.collect()); + } + output + } + + Self { v4: convert_map(&routes.v4), v6: convert_map(&routes.v6) } + } + } + + let actual = BTreeMgdCurrentRoutes::from(&actual); + let expected = BTreeMgdCurrentRoutes::from(&expected); + eprintln!("--- checking {description} ---"); + assert_eq!(actual, expected); +} + +#[tokio::test(flavor = "multi_thread")] +async fn proptest_full_reconciliation() { + let logctx = dev::test_setup_log("proptest_full_reconciliation"); + let mgsctx = gateway_test_utils::setup::test_setup( + "proptest_full_reconciliation", + SpPort::One, + ) + .await; + let mut mgdctx = + dev::maghemite::MgdInstance::start(0, mgsctx.address().into()) + .await + .expect("started mgd"); + let client = Client::new( + &format!("http://{}", mgdctx.address()), + logctx.log.clone(), + ); + let rt = tokio::runtime::Handle::current(); + + let one_test_invocation = async |input: TestInput| { + // Clear all routes from a previous proptest invocation. + let current_routes = + MgdCurrentRoutes::fetch(&client).await.expect("fetched all routes"); + remove_all_current_routes(&client, current_routes) + .await + .expect("removed all preexisting routes"); + let current_routes = + MgdCurrentRoutes::fetch(&client).await.expect("fetched all routes"); + assert_routes_eq( + current_routes, + MgdCurrentRoutes::default(), + "cleared all routes", + ); + + // Apply all initial settings. + create_initial_routes(&client, input.initial_mgd_routes()) + .await + .expect("created initial routes"); + let current_routes = + MgdCurrentRoutes::fetch(&client).await.expect("fetched all routes"); + assert_routes_eq( + current_routes, + input.initial_mgd_routes(), + "initial setup mismatch", + ); + + // Perform reconciliation. + let status = reconcile( + &client, + &input.desired_rack_config(), + ThisSledSwitchSlot::TEST_FAKE, + &logctx.log, + ) + .await; + + // Check reported status and mgd state + assert_eq!(status, input.expected_reconciliation_status()); + let current_routes = + MgdCurrentRoutes::fetch(&client).await.expect("fetched all routes"); + assert_routes_eq( + current_routes, + input.post_reconciliation_mgd_routes(), + "post-reconciliation mismatch", + ); + }; + + proptest_macro!(|(input: TestInput)| { + // Do a little dance to call our async `one_test_invocation` within the + // non-async `proptest_macro!()` context. + block_in_place(|| rt.block_on(one_test_invocation(input))); + }); + + mgdctx.cleanup().await.expect("mgd cleanup succeeded"); + mgsctx.teardown().await; + logctx.cleanup_successful(); +} diff --git a/sled-agent/types/versions/src/initial/early_networking.rs b/sled-agent/types/versions/src/initial/early_networking.rs index c709a238074..760e87eeec8 100644 --- a/sled-agent/types/versions/src/initial/early_networking.rs +++ b/sled-agent/types/versions/src/initial/early_networking.rs @@ -299,6 +299,7 @@ pub struct PortConfig { strum::EnumIter, )] #[serde(rename_all = "snake_case")] +#[cfg_attr(any(test, feature = "testing"), derive(test_strategy::Arbitrary))] pub enum SwitchSlot { /// Switch in upper slot Switch0, diff --git a/test-utils/src/dev/maghemite.rs b/test-utils/src/dev/maghemite.rs index eaae1af8cd4..988c3a42375 100644 --- a/test-utils/src/dev/maghemite.rs +++ b/test-utils/src/dev/maghemite.rs @@ -4,7 +4,7 @@ //! Tools for managing Maghemite during development -use std::net::SocketAddr; +use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::Duration; @@ -88,6 +88,10 @@ impl MgdInstance { Ok(Self { port, args, child, data_dir: Some(temp_dir) }) } + pub fn address(&self) -> SocketAddrV6 { + SocketAddrV6::new(Ipv6Addr::LOCALHOST, self.port, 0, 0) + } + pub async fn cleanup(&mut self) -> Result<(), anyhow::Error> { if let Some(mut child) = self.child.take() { child.start_kill().context("Sending SIGKILL to child")?;