From 3ab1b09b494ab816c9a376e34274074917fd5e2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20K=C3=A4stner?= Date: Sun, 17 May 2026 16:56:57 +0200 Subject: [PATCH] Implement EVPN Fabric underlay p2p link reconciliation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add numbered underlay addressing support to the Fabric controller. When spec.underlay.addressing.ipPrefixPoolRef is set, the controller allocates /31 prefix Claims per fabric link and assigns host addresses to both interface endpoints. Also refactor the test suite: - Remove helper functions, inline all resource creation - Harmonize GenerateName conventions (spine-, leaf-, etc.) - Simplify IPv4 assignment in unnumbered/numbered phases - Add V(1) info log for interfaces missing neighbor label - Add TODO comments for future SSA migration on patch calls Signed-off-by: Felix Kästner --- internal/controller/evpn/fabric_controller.go | 194 ++++++++ .../controller/evpn/fabric_controller_test.go | 442 ++++++++++++++---- internal/controller/evpn/suite_test.go | 12 + 3 files changed, 554 insertions(+), 94 deletions(-) diff --git a/internal/controller/evpn/fabric_controller.go b/internal/controller/evpn/fabric_controller.go index 35b81b435..664fa3bb6 100644 --- a/internal/controller/evpn/fabric_controller.go +++ b/internal/controller/evpn/fabric_controller.go @@ -4,8 +4,10 @@ package evpn import ( + "cmp" "context" "fmt" + "net/netip" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -14,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -53,6 +56,7 @@ type FabricReconciler struct { // +kubebuilder:rbac:groups=networking.metal.ironcore.dev,resources=interfaces,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=pool.networking.metal.ironcore.dev,resources=claims,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=pool.networking.metal.ironcore.dev,resources=ipaddresspools,verbs=get;list;watch +// +kubebuilder:rbac:groups=pool.networking.metal.ironcore.dev,resources=ipprefixpools,verbs=get;list;watch // +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch // Reconcile is part of the main kubernetes reconciliation loop which aims to @@ -167,6 +171,13 @@ func (r *FabricReconciler) SetupWithManager(mgr ctrl.Manager) error { handler.EnqueueRequestsFromMapFunc(r.devicesToFabrics), builder.WithPredicates(predicate.LabelChangedPredicate{}), ). + // Re-reconcile when an Interface's labels change so that interfaces + // newly matching the spec.underlay.interfaceSelector are enrolled into the fabric. + Watches( + &v1alpha1.Interface{}, + handler.EnqueueRequestsFromMapFunc(r.interfacesToFabrics), + builder.WithPredicates(predicate.LabelChangedPredicate{}), + ). WithEventFilter(filter). Named("evpn-fabric"). Complete(r) @@ -181,6 +192,7 @@ func (r *FabricReconciler) reconcile(ctx context.Context, fabric *evpnv1alpha1.F r.reconcileSystemLoopbacks, r.reconcileVTEPLoopbacks, r.reconcileAnycastRPLoopbacks, + r.reconcileUnderlayLinks, } for _, phase := range phases { res, err := phase(ctx, fabric) @@ -374,6 +386,155 @@ func (r *FabricReconciler) reconcileLoopbackInterface(ctx context.Context, fabri return nil } +// reconcileUnderlayLinks patches pre-existing Interface resources matched by +// spec.underlay.interfaceSelector with MTU 9216 and IPv4 configuration. +// For unnumbered addressing, interfaces borrow the IPv4 address from their device's lo0. +// For numbered addressing, one /31 prefix Claim is allocated per link pair (identified by +// PhysicalInterfaceNeighborLabel); both ends derive their host address from that prefix. +func (r *FabricReconciler) reconcileUnderlayLinks(ctx context.Context, fabric *evpnv1alpha1.Fabric) (ctrl.Result, error) { + intfSelector, err := metav1.LabelSelectorAsSelector(&fabric.Spec.Underlay.InterfaceSelector) + if err != nil { + return ctrl.Result{}, fmt.Errorf("invalid underlay interfaceSelector: %w", err) + } + interfaces := &v1alpha1.InterfaceList{} + if err := r.List(ctx, interfaces, client.InNamespace(fabric.Namespace), client.MatchingLabelsSelector{Selector: intfSelector}); err != nil { + return ctrl.Result{}, fmt.Errorf("listing underlay interfaces: %w", err) + } + deviceSelector, err := metav1.LabelSelectorAsSelector(&fabric.Spec.DeviceSelector) + if err != nil { + return ctrl.Result{}, fmt.Errorf("invalid deviceSelector: %w", err) + } + devices := &v1alpha1.DeviceList{} + if err := r.List(ctx, devices, client.InNamespace(fabric.Namespace), client.MatchingLabelsSelector{Selector: deviceSelector}); err != nil { + return ctrl.Result{}, fmt.Errorf("listing devices: %w", err) + } + deviceSet := sets.New[string]() + for i := range devices.Items { + deviceSet.Insert(devices.Items[i].Name) + } + for i := range interfaces.Items { + intf := &interfaces.Items[i] + if intf.Spec.Type != v1alpha1.InterfaceTypePhysical { + return ctrl.Result{}, fmt.Errorf("interface %s has type %s, expected %s", intf.Name, intf.Spec.Type, v1alpha1.InterfaceTypePhysical) + } + if !deviceSet.Has(intf.Spec.DeviceRef.Name) { + return ctrl.Result{}, fmt.Errorf("interface %s references device %s which is not part of the fabric", intf.Name, intf.Spec.DeviceRef.Name) + } + var err error + switch { + case fabric.Spec.Underlay.Addressing.Unnumbered: + err = r.reconcileUnderlayInterfaceUnnumbered(ctx, fabric, intf) + case fabric.Spec.Underlay.Addressing.IPPrefixPoolRef != nil: + err = r.reconcileUnderlayInterfaceNumbered(ctx, fabric, intf) + } + if err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil +} + +// reconcileUnderlayInterfaceUnnumbered patches the interface to borrow the IPv4 address +// from the device's lo0 interface created by reconcileSystemLoopbacks. +func (r *FabricReconciler) reconcileUnderlayInterfaceUnnumbered(ctx context.Context, fabric *evpnv1alpha1.Fabric, intf *v1alpha1.Interface) error { + orig := intf.DeepCopy() + intf.Spec.MTU = 9216 + intf.Spec.IPv4 = &v1alpha1.InterfaceIPv4{ + Unnumbered: &v1alpha1.InterfaceIPv4Unnumbered{ + InterfaceRef: v1alpha1.LocalObjectReference{ + Name: fmt.Sprintf("%s-%s-lo0", fabric.Name, intf.Spec.DeviceRef.Name), + }, + }, + } + if equality.Semantic.DeepEqual(orig.Spec, intf.Spec) { + return nil + } + // TODO: switch to server-side apply with field ownership once applyconfiguration generation is available + if err := r.Patch(ctx, intf, client.MergeFrom(orig)); err != nil { + return fmt.Errorf("patching underlay interface %s: %w", intf.Name, err) + } + return nil +} + +// reconcileUnderlayInterfaceNumbered allocates a /31 prefix per link pair via a Claim and +// assigns a host address to this interface. The peer is identified by PhysicalInterfaceNeighborLabel; +// if absent the interface is skipped (link not yet complete). The claim name is derived from +// the two interface names sorted alphabetically so both ends resolve to the same Claim. +// The lexicographically first interface receives addr 0 of the /31; the second receives addr 1. +func (r *FabricReconciler) reconcileUnderlayInterfaceNumbered(ctx context.Context, fabric *evpnv1alpha1.Fabric, intf *v1alpha1.Interface) error { + peerName, ok := intf.Labels[v1alpha1.PhysicalInterfaceNeighborLabel] + if !ok { + ctrl.LoggerFrom(ctx).V(1).Info("Skipping interface without neighbor label", "interface", intf.Name) + return nil + } + + // Stable claim name: sort the two interface names so both ends agree. + a, b := intf.Name, peerName + claimName := fmt.Sprintf("%s-%s-%s-p2p", fabric.Name, min(a, b), max(a, b)) + + claim, err := r.reconcileUnderlayPrefixClaim(ctx, fabric, claimName) + if err != nil { + return err + } + + cond := conditions.Get(claim, poolv1alpha1.AllocatedCondition) + if cond == nil || cond.Status != metav1.ConditionTrue || claim.Status.Value == "" { + return nil + } + + prefix, err := v1alpha1.ParsePrefix(claim.Status.Value) + if err != nil { + return reconcile.TerminalError(fmt.Errorf("parsing allocated prefix %q for claim %s: %w", claim.Status.Value, claimName, err)) + } + + // Assign addr 0 to the lex-first interface, addr 1 to the second. + addr := prefix.Addr() + if cmp.Compare(intf.Name, peerName) > 0 { + addr = addr.Next() + } + + orig := intf.DeepCopy() + intf.Spec.MTU = 9216 + intf.Spec.IPv4 = &v1alpha1.InterfaceIPv4{ + Addresses: []v1alpha1.IPPrefix{{Prefix: netip.PrefixFrom(addr, prefix.Bits())}}, + } + if equality.Semantic.DeepEqual(orig.Spec, intf.Spec) { + return nil + } + // TODO: switch to server-side apply with field ownership once applyconfiguration generation is available + if err := r.Patch(ctx, intf, client.MergeFrom(orig)); err != nil { + return fmt.Errorf("patching underlay interface %s: %w", intf.Name, err) + } + return nil +} + +// reconcileUnderlayPrefixClaim ensures a Claim against an IPPrefixPool exists for the given link. +func (r *FabricReconciler) reconcileUnderlayPrefixClaim(ctx context.Context, fabric *evpnv1alpha1.Fabric, claimName string) (*poolv1alpha1.Claim, error) { + claim := &poolv1alpha1.Claim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: fabric.Namespace, + }, + } + res, err := controllerutil.CreateOrPatch(ctx, r.Client, claim, func() error { + claim.Spec = poolv1alpha1.ClaimSpec{ + PoolRef: v1alpha1.TypedLocalObjectReference{ + APIVersion: poolv1alpha1.GroupVersion.String(), + Kind: "IPPrefixPool", + Name: fabric.Spec.Underlay.Addressing.IPPrefixPoolRef.Name, + }, + } + return controllerutil.SetControllerReference(fabric, claim, r.Scheme) + }) + if err != nil { + return nil, fmt.Errorf("reconciling prefix claim %s: %w", claimName, err) + } + if res == controllerutil.OperationResultCreated { + r.Recorder.Eventf(fabric, nil, "Normal", "ClaimCreated", "Reconcile", "Created underlay prefix claim %s", claimName) + } + return claim, nil +} + // devicesToFabrics is a [handler.MapFunc] that enqueues all Fabrics whose // spec.deviceSelector matches the labels of the changed Device. func (r *FabricReconciler) devicesToFabrics(ctx context.Context, obj client.Object) []ctrl.Request { @@ -406,3 +567,36 @@ func (r *FabricReconciler) devicesToFabrics(ctx context.Context, obj client.Obje } return requests } + +// interfacesToFabrics is a [handler.MapFunc] that enqueues all Fabrics whose +// spec.underlay.interfaceSelector matches the labels of the changed Interface. +func (r *FabricReconciler) interfacesToFabrics(ctx context.Context, obj client.Object) []ctrl.Request { + intf, ok := obj.(*v1alpha1.Interface) + if !ok { + panic(fmt.Sprintf("Expected an Interface but got a %T", obj)) + } + + log := ctrl.LoggerFrom(ctx) + + fabricList := &evpnv1alpha1.FabricList{} + if err := r.List(ctx, fabricList, client.InNamespace(intf.Namespace)); err != nil { + log.Error(err, "Failed to list Fabrics") + return nil + } + + var requests []ctrl.Request + for _, fabric := range fabricList.Items { + selector, err := metav1.LabelSelectorAsSelector(&fabric.Spec.Underlay.InterfaceSelector) + if err != nil { + log.Error(err, "Failed to parse underlay interfaceSelector", "fabric", fabric.Name) + continue + } + if selector.Matches(labels.Set(intf.Labels)) { + log.V(2).Info("Enqueuing Fabric for reconciliation", "fabric", fabric.Name) + requests = append(requests, ctrl.Request{ + NamespacedName: client.ObjectKeyFromObject(&fabric), + }) + } + } + return requests +} diff --git a/internal/controller/evpn/fabric_controller_test.go b/internal/controller/evpn/fabric_controller_test.go index 2e232dcec..fdf7505ed 100644 --- a/internal/controller/evpn/fabric_controller_test.go +++ b/internal/controller/evpn/fabric_controller_test.go @@ -35,19 +35,20 @@ import ( // lo2 (anycast VTEP) — leaf-1, leaf-2 // lo100 (anycast RP) — 1 Claim shared across spine-1, spine-2 var _ = Describe("Fabric Controller", func() { - Context("When reconciling a resource", func() { + Context("When reconciling with unnumbered underlay addressing", func() { var ( - pool *poolv1alpha1.IPAddressPool - spine1 *corev1alpha1.Device - spine2 *corev1alpha1.Device - leaf1 *corev1alpha1.Device - leaf2 *corev1alpha1.Device - fabric *evpnv1alpha1.Fabric + loopbackPool *poolv1alpha1.IPAddressPool + spine1 *corev1alpha1.Device + spine2 *corev1alpha1.Device + leaf1 *corev1alpha1.Device + leaf2 *corev1alpha1.Device + spineIntf *corev1alpha1.Interface + leafIntf *corev1alpha1.Interface ) BeforeEach(func() { By("Creating an IPAddressPool for loopback allocation") - pool = &poolv1alpha1.IPAddressPool{ + loopbackPool = &poolv1alpha1.IPAddressPool{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "loopback-pool-", Namespace: metav1.NamespaceDefault, @@ -56,74 +57,109 @@ var _ = Describe("Fabric Controller", func() { Prefixes: []corev1alpha1.IPPrefix{corev1alpha1.MustParsePrefix("10.0.0.0/24")}, }, } - Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + Expect(k8sClient.Create(ctx, loopbackPool)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, loopbackPool))).To(Succeed()) + }) By("Creating spine-1 (route reflector, rendezvous point)") spine1 = &corev1alpha1.Device{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: "spine-", + GenerateName: "spine1-", Namespace: metav1.NamespaceDefault, - Labels: map[string]string{ - "topology.kubernetes.io/zone": "test-zone", - "role": "spine", - }, - }, - Spec: corev1alpha1.DeviceSpec{ - Endpoint: corev1alpha1.Endpoint{Address: "192.168.0.1:9339"}, + Labels: map[string]string{"topology.kubernetes.io/zone": "test-zone", "role": "spine"}, }, + Spec: corev1alpha1.DeviceSpec{Endpoint: corev1alpha1.Endpoint{Address: "192.168.0.1:9339"}}, } Expect(k8sClient.Create(ctx, spine1)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, spine1))).To(Succeed()) + }) By("Creating spine-2 (route reflector, rendezvous point)") spine2 = &corev1alpha1.Device{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: "spine-", + GenerateName: "spine2-", Namespace: metav1.NamespaceDefault, - Labels: map[string]string{ - "topology.kubernetes.io/zone": "test-zone", - "role": "spine", - }, - }, - Spec: corev1alpha1.DeviceSpec{ - Endpoint: corev1alpha1.Endpoint{Address: "192.168.0.2:9339"}, + Labels: map[string]string{"topology.kubernetes.io/zone": "test-zone", "role": "spine"}, }, + Spec: corev1alpha1.DeviceSpec{Endpoint: corev1alpha1.Endpoint{Address: "192.168.0.2:9339"}}, } Expect(k8sClient.Create(ctx, spine2)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, spine2))).To(Succeed()) + }) By("Creating leaf-1 (VTEP)") leaf1 = &corev1alpha1.Device{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: "leaf-", + GenerateName: "leaf1-", Namespace: metav1.NamespaceDefault, - Labels: map[string]string{ - "topology.kubernetes.io/zone": "test-zone", - "role": "leaf", - }, - }, - Spec: corev1alpha1.DeviceSpec{ - Endpoint: corev1alpha1.Endpoint{Address: "192.168.1.1:9339"}, + Labels: map[string]string{"topology.kubernetes.io/zone": "test-zone", "role": "leaf"}, }, + Spec: corev1alpha1.DeviceSpec{Endpoint: corev1alpha1.Endpoint{Address: "192.168.1.1:9339"}}, } Expect(k8sClient.Create(ctx, leaf1)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, leaf1))).To(Succeed()) + }) By("Creating leaf-2 (VTEP)") leaf2 = &corev1alpha1.Device{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: "leaf-", + GenerateName: "leaf2-", Namespace: metav1.NamespaceDefault, - Labels: map[string]string{ - "topology.kubernetes.io/zone": "test-zone", - "role": "leaf", - }, - }, - Spec: corev1alpha1.DeviceSpec{ - Endpoint: corev1alpha1.Endpoint{Address: "192.168.1.2:9339"}, + Labels: map[string]string{"topology.kubernetes.io/zone": "test-zone", "role": "leaf"}, }, + Spec: corev1alpha1.DeviceSpec{Endpoint: corev1alpha1.Endpoint{Address: "192.168.1.2:9339"}}, } Expect(k8sClient.Create(ctx, leaf2)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, leaf2))).To(Succeed()) + }) - By("Creating the Fabric resource") - fabric = &evpnv1alpha1.Fabric{ + By("Creating a fabric-facing Interface on spine-1") + spineIntf = &corev1alpha1.Interface{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: spine1.Name + "-eth0-", + Namespace: metav1.NamespaceDefault, + Labels: map[string]string{"role": "fabric"}, + }, + Spec: corev1alpha1.InterfaceSpec{ + DeviceRef: corev1alpha1.LocalObjectReference{Name: spine1.Name}, + Name: "eth0", + Type: corev1alpha1.InterfaceTypePhysical, + AdminState: corev1alpha1.AdminStateUp, + }, + } + Expect(k8sClient.Create(ctx, spineIntf)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, spineIntf))).To(Succeed()) + }) + + By("Creating a fabric-facing Interface on leaf-1") + leafIntf = &corev1alpha1.Interface{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: leaf1.Name + "-eth0-", + Namespace: metav1.NamespaceDefault, + Labels: map[string]string{"role": "fabric"}, + }, + Spec: corev1alpha1.InterfaceSpec{ + DeviceRef: corev1alpha1.LocalObjectReference{Name: leaf1.Name}, + Name: "eth0", + Type: corev1alpha1.InterfaceTypePhysical, + AdminState: corev1alpha1.AdminStateUp, + }, + } + Expect(k8sClient.Create(ctx, leafIntf)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, leafIntf))).To(Succeed()) + }) + }) + + It("Should create lo0 Claims for all fabric devices, lo1/lo2 Claims for VTEP devices, and one lo100 Claim per RP group", func() { + By("Creating the Fabric resource with unnumbered addressing") + fabric := &evpnv1alpha1.Fabric{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "fabric-", Namespace: metav1.NamespaceDefault, @@ -133,16 +169,12 @@ var _ = Describe("Fabric Controller", func() { MatchLabels: map[string]string{"topology.kubernetes.io/zone": "test-zone"}, }, Loopbacks: evpnv1alpha1.FabricLoopbacksSpec{ - IPAddressPoolRef: corev1alpha1.LocalObjectReference{Name: pool.Name}, + IPAddressPoolRef: corev1alpha1.LocalObjectReference{Name: loopbackPool.Name}, }, Underlay: evpnv1alpha1.FabricUnderlaySpec{ - Protocol: evpnv1alpha1.UnderlayProtocolOSPF, - InterfaceSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{"role": "fabric"}, - }, - Addressing: evpnv1alpha1.FabricUnderlayAddressingSpec{ - Unnumbered: true, - }, + Protocol: evpnv1alpha1.UnderlayProtocolOSPF, + InterfaceSelector: metav1.LabelSelector{MatchLabels: map[string]string{"role": "fabric"}}, + Addressing: evpnv1alpha1.FabricUnderlayAddressingSpec{Unnumbered: true}, }, Overlay: evpnv1alpha1.FabricOverlaySpec{ Protocol: evpnv1alpha1.OverlayProtocolIBGP, @@ -176,36 +208,20 @@ var _ = Describe("Fabric Controller", func() { }, } Expect(k8sClient.Create(ctx, fabric)).To(Succeed()) - }) - - AfterEach(func() { - By("Deleting the Fabric resource") - Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, fabric))).To(Succeed()) - - By("Deleting the Device resources") - for _, d := range []*corev1alpha1.Device{spine1, spine2, leaf1, leaf2} { - Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, d))).To(Succeed()) - } - - By("Deleting the IPAddressPool resource") - Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, pool))).To(Succeed()) - - By("Deleting all Claims created for the Fabric") - Expect(k8sClient.DeleteAllOf(ctx, &poolv1alpha1.Claim{}, client.InNamespace(metav1.NamespaceDefault))).To(Succeed()) - - By("Verifying all Claims are deleted") - Eventually(func(g Gomega) { - list := &poolv1alpha1.ClaimList{} - g.Expect(k8sClient.List(ctx, list, client.InNamespace(metav1.NamespaceDefault))).To(Succeed()) - g.Expect(list.Items).To(BeEmpty()) - }).Should(Succeed()) - }) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, fabric))).To(Succeed()) + Expect(k8sClient.DeleteAllOf(ctx, &poolv1alpha1.Claim{}, client.InNamespace(metav1.NamespaceDefault))).To(Succeed()) + Eventually(func(g Gomega) { + list := &poolv1alpha1.ClaimList{} + g.Expect(k8sClient.List(ctx, list, client.InNamespace(metav1.NamespaceDefault))).To(Succeed()) + g.Expect(list.Items).To(BeEmpty()) + }).Should(Succeed()) + }) - It("Should create lo0 Claims for all fabric devices, lo1/lo2 Claims for VTEP devices, and one lo100 Claim per RP group", func() { By("Verifying the controller adds a finalizer") Eventually(func(g Gomega) { f := &evpnv1alpha1.Fabric{} - g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name, Namespace: metav1.NamespaceDefault}, f)).To(Succeed()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fabric), f)).To(Succeed()) g.Expect(controllerutil.ContainsFinalizer(f, evpnv1alpha1.FinalizerName)).To(BeTrue()) }).Should(Succeed()) @@ -214,8 +230,15 @@ var _ = Describe("Fabric Controller", func() { Eventually(func(g Gomega) { claim := &poolv1alpha1.Claim{} g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name + "-" + d.Name + "-lo0", Namespace: metav1.NamespaceDefault}, claim)).To(Succeed()) - g.Expect(claim.Spec.PoolRef.Name).To(Equal(pool.Name)) + g.Expect(claim.Spec.PoolRef.Name).To(Equal(loopbackPool.Name)) g.Expect(claim.Spec.PoolRef.Kind).To(Equal("IPAddressPool")) + g.Expect(claim.OwnerReferences).To(ContainElement( + SatisfyAll( + HaveField("Kind", "Fabric"), + HaveField("Name", fabric.Name), + HaveField("Controller", HaveValue(BeTrue())), + ), + )) }).Should(Succeed()) } @@ -282,31 +305,262 @@ var _ = Describe("Fabric Controller", func() { g.Expect(intf.Spec.IPv4.Addresses).To(HaveLen(1)) }).Should(Succeed()) } - }) - It("Should set the Fabric as controller owner of each Claim", func() { - By("Verifying the lo0 Claim for leaf-1 has the Fabric as controller owner") + By("Verifying underlay interfaces are patched with MTU 9216 and unnumbered IPv4 pointing to lo0") + for _, intf := range []*corev1alpha1.Interface{spineIntf, leafIntf} { + Eventually(func(g Gomega) { + i := &corev1alpha1.Interface{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(intf), i)).To(Succeed()) + g.Expect(i.Spec.MTU).To(Equal(int32(9216))) + g.Expect(i.Spec.IPv4).NotTo(BeNil()) + g.Expect(i.Spec.IPv4.Unnumbered).NotTo(BeNil()) + g.Expect(i.Spec.IPv4.Unnumbered.InterfaceRef.Name).To(Equal(fabric.Name + "-" + i.Spec.DeviceRef.Name + "-lo0")) + }).Should(Succeed()) + } + + By("Verifying the Fabric Ready condition is True once all phases are complete") Eventually(func(g Gomega) { - claim := &poolv1alpha1.Claim{} - g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name + "-" + leaf1.Name + "-lo0", Namespace: metav1.NamespaceDefault}, claim)).To(Succeed()) - g.Expect(claim.OwnerReferences).To(ContainElement( + f := &evpnv1alpha1.Fabric{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(fabric), f)).To(Succeed()) + g.Expect(f.Status.Conditions).To(ContainElement( SatisfyAll( - HaveField("Kind", "Fabric"), - HaveField("Name", fabric.Name), - HaveField("Controller", HaveValue(BeTrue())), + HaveField("Type", corev1alpha1.ReadyCondition), + HaveField("Status", metav1.ConditionTrue), ), )) }).Should(Succeed()) }) + }) - It("Should set the Fabric to Ready", func() { - By("Verifying the Fabric Ready condition is True once all phases are complete") + Context("When reconciling with numbered underlay addressing", func() { + var ( + loopbackPool *poolv1alpha1.IPAddressPool + prefixPool *poolv1alpha1.IPPrefixPool + spine1 *corev1alpha1.Device + leaf1 *corev1alpha1.Device + spineIntf *corev1alpha1.Interface + leafIntf *corev1alpha1.Interface + ) + + BeforeEach(func() { + By("Creating an IPAddressPool for loopback allocation") + loopbackPool = &poolv1alpha1.IPAddressPool{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "loopback-pool-", + Namespace: metav1.NamespaceDefault, + }, + Spec: poolv1alpha1.IPAddressPoolSpec{ + Prefixes: []corev1alpha1.IPPrefix{corev1alpha1.MustParsePrefix("10.0.0.0/24")}, + }, + } + Expect(k8sClient.Create(ctx, loopbackPool)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, loopbackPool))).To(Succeed()) + }) + + By("Creating an IPPrefixPool for P2P link allocation") + prefixPool = &poolv1alpha1.IPPrefixPool{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "p2p-pool-", + Namespace: metav1.NamespaceDefault, + }, + Spec: poolv1alpha1.IPPrefixPoolSpec{ + Prefixes: []poolv1alpha1.IPPrefixPoolPrefix{ + { + Prefix: corev1alpha1.MustParsePrefix("10.1.0.0/16"), + PrefixLength: 31, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, prefixPool)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, prefixPool))).To(Succeed()) + }) + + By("Creating spine-1") + spine1 = &corev1alpha1.Device{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "spine1-", + Namespace: metav1.NamespaceDefault, + Labels: map[string]string{"topology.kubernetes.io/zone": "test-zone", "role": "spine"}, + }, + Spec: corev1alpha1.DeviceSpec{Endpoint: corev1alpha1.Endpoint{Address: "192.168.0.1:9339"}}, + } + Expect(k8sClient.Create(ctx, spine1)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, spine1))).To(Succeed()) + }) + + By("Creating leaf-1") + leaf1 = &corev1alpha1.Device{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "leaf1-", + Namespace: metav1.NamespaceDefault, + Labels: map[string]string{"topology.kubernetes.io/zone": "test-zone", "role": "leaf"}, + }, + Spec: corev1alpha1.DeviceSpec{Endpoint: corev1alpha1.Endpoint{Address: "192.168.1.1:9339"}}, + } + Expect(k8sClient.Create(ctx, leaf1)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, leaf1))).To(Succeed()) + }) + + By("Creating a fabric-facing Interface on spine-1 with neighbor label pointing to leaf-1's interface") + // Create both interfaces first, then set the neighbor labels so each points to the other. + spineIntf = &corev1alpha1.Interface{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: spine1.Name + "-eth0-", + Namespace: metav1.NamespaceDefault, + Labels: map[string]string{"role": "fabric"}, + }, + Spec: corev1alpha1.InterfaceSpec{ + DeviceRef: corev1alpha1.LocalObjectReference{Name: spine1.Name}, + Name: "eth0", + Type: corev1alpha1.InterfaceTypePhysical, + AdminState: corev1alpha1.AdminStateUp, + }, + } + Expect(k8sClient.Create(ctx, spineIntf)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, spineIntf))).To(Succeed()) + }) + + leafIntf = &corev1alpha1.Interface{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: leaf1.Name + "-eth0-", + Namespace: metav1.NamespaceDefault, + Labels: map[string]string{"role": "fabric"}, + }, + Spec: corev1alpha1.InterfaceSpec{ + DeviceRef: corev1alpha1.LocalObjectReference{Name: leaf1.Name}, + Name: "eth0", + Type: corev1alpha1.InterfaceTypePhysical, + AdminState: corev1alpha1.AdminStateUp, + }, + } + Expect(k8sClient.Create(ctx, leafIntf)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, leafIntf))).To(Succeed()) + }) + + By("Setting neighbor labels to pair the interfaces as a link") + spineOrig := spineIntf.DeepCopy() + spineIntf.Labels[corev1alpha1.PhysicalInterfaceNeighborLabel] = leafIntf.Name + Expect(k8sClient.Patch(ctx, spineIntf, client.MergeFrom(spineOrig))).To(Succeed()) + + leafOrig := leafIntf.DeepCopy() + leafIntf.Labels[corev1alpha1.PhysicalInterfaceNeighborLabel] = spineIntf.Name + Expect(k8sClient.Patch(ctx, leafIntf, client.MergeFrom(leafOrig))).To(Succeed()) + }) + + It("Should allocate one /31 prefix Claim per link and assign host addresses to both ends", func() { + By("Creating the Fabric resource with numbered addressing") + fabric := &evpnv1alpha1.Fabric{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "fabric-", + Namespace: metav1.NamespaceDefault, + }, + Spec: evpnv1alpha1.FabricSpec{ + DeviceSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"topology.kubernetes.io/zone": "test-zone"}, + }, + Loopbacks: evpnv1alpha1.FabricLoopbacksSpec{ + IPAddressPoolRef: corev1alpha1.LocalObjectReference{Name: loopbackPool.Name}, + }, + Underlay: evpnv1alpha1.FabricUnderlaySpec{ + Protocol: evpnv1alpha1.UnderlayProtocolOSPF, + InterfaceSelector: metav1.LabelSelector{MatchLabels: map[string]string{"role": "fabric"}}, + Addressing: evpnv1alpha1.FabricUnderlayAddressingSpec{ + IPPrefixPoolRef: &corev1alpha1.LocalObjectReference{Name: prefixPool.Name}, + }, + }, + Overlay: evpnv1alpha1.FabricOverlaySpec{ + Protocol: evpnv1alpha1.OverlayProtocolIBGP, + IBGP: &evpnv1alpha1.FabricIBGPSpec{ + ASNumber: intstr.FromInt(65000), + RouteReflectors: []evpnv1alpha1.RouteReflectorGroup{ + { + Name: "spines", + DeviceSelector: metav1.LabelSelector{MatchLabels: map[string]string{"role": "spine"}}, + ClientDeviceSelector: metav1.LabelSelector{MatchLabels: map[string]string{"role": "leaf"}}, + }, + }, + }, + }, + BUM: evpnv1alpha1.FabricBUMSpec{ + Type: evpnv1alpha1.BUMTypeMulticast, + PIM: &evpnv1alpha1.FabricPIMSpec{ + AnycastRendezvousPoints: []evpnv1alpha1.AnycastRendezvousPoint{ + { + Name: "spine-rp", + MulticastGroups: []corev1alpha1.IPPrefix{corev1alpha1.MustParsePrefix("224.0.0.0/4")}, + DeviceSelector: metav1.LabelSelector{MatchLabels: map[string]string{"role": "spine"}}, + ClientDeviceSelector: metav1.LabelSelector{MatchLabels: map[string]string{"role": "leaf"}}, + }, + }, + }, + }, + VTEP: evpnv1alpha1.FabricVTEPSpec{ + DeviceSelector: metav1.LabelSelector{MatchLabels: map[string]string{"role": "leaf"}}, + }, + }, + } + Expect(k8sClient.Create(ctx, fabric)).To(Succeed()) + DeferCleanup(func() { + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, fabric))).To(Succeed()) + Expect(k8sClient.DeleteAllOf(ctx, &poolv1alpha1.Claim{}, client.InNamespace(metav1.NamespaceDefault))).To(Succeed()) + Eventually(func(g Gomega) { + list := &poolv1alpha1.ClaimList{} + g.Expect(k8sClient.List(ctx, list, client.InNamespace(metav1.NamespaceDefault))).To(Succeed()) + g.Expect(list.Items).To(BeEmpty()) + }).Should(Succeed()) + }) + + // Stable claim name: alphabetically sorted interface names. + claimName := fabric.Name + "-" + min(spineIntf.Name, leafIntf.Name) + "-" + max(spineIntf.Name, leafIntf.Name) + "-p2p" + + By("Verifying a single p2p Claim is created for the link") Eventually(func(g Gomega) { - f := &evpnv1alpha1.Fabric{} - g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: fabric.Name, Namespace: metav1.NamespaceDefault}, f)).To(Succeed()) - g.Expect(f.Status.Conditions).NotTo(BeEmpty()) - g.Expect(f.Status.Conditions[0].Type).To(Equal(corev1alpha1.ReadyCondition)) - g.Expect(f.Status.Conditions[0].Status).To(Equal(metav1.ConditionTrue)) + claim := &poolv1alpha1.Claim{} + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: claimName, Namespace: metav1.NamespaceDefault}, claim)).To(Succeed()) + g.Expect(claim.Spec.PoolRef.Kind).To(Equal("IPPrefixPool")) + g.Expect(claim.Spec.PoolRef.Name).To(Equal(prefixPool.Name)) + }).Should(Succeed()) + + By("Verifying the spine-side interface is patched with MTU 9216 and a /31 address") + Eventually(func(g Gomega) { + i := &corev1alpha1.Interface{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(spineIntf), i)).To(Succeed()) + g.Expect(i.Spec.MTU).To(Equal(int32(9216))) + g.Expect(i.Spec.IPv4).NotTo(BeNil()) + g.Expect(i.Spec.IPv4.Addresses).To(HaveLen(1)) + g.Expect(i.Spec.IPv4.Addresses[0].Bits()).To(Equal(31)) + }).Should(Succeed()) + + By("Verifying the leaf-side interface is patched with MTU 9216 and the peer /31 address") + Eventually(func(g Gomega) { + i := &corev1alpha1.Interface{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(leafIntf), i)).To(Succeed()) + g.Expect(i.Spec.MTU).To(Equal(int32(9216))) + g.Expect(i.Spec.IPv4).NotTo(BeNil()) + g.Expect(i.Spec.IPv4.Addresses).To(HaveLen(1)) + g.Expect(i.Spec.IPv4.Addresses[0].Bits()).To(Equal(31)) + }).Should(Succeed()) + + By("Verifying the two addresses are adjacent and from the same /31") + Eventually(func(g Gomega) { + si := &corev1alpha1.Interface{} + li := &corev1alpha1.Interface{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(spineIntf), si)).To(Succeed()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(leafIntf), li)).To(Succeed()) + g.Expect(si.Spec.IPv4.Addresses).To(HaveLen(1)) + g.Expect(li.Spec.IPv4.Addresses).To(HaveLen(1)) + spineAddr := si.Spec.IPv4.Addresses[0].Addr() + leafAddr := li.Spec.IPv4.Addresses[0].Addr() + // The two addresses must differ and share the same masked prefix. + g.Expect(spineAddr).NotTo(Equal(leafAddr)) + g.Expect(si.Spec.IPv4.Addresses[0].Masked()).To(Equal(li.Spec.IPv4.Addresses[0].Masked())) }).Should(Succeed()) }) }) diff --git a/internal/controller/evpn/suite_test.go b/internal/controller/evpn/suite_test.go index 4fdbfb5ff..c0b984e90 100644 --- a/internal/controller/evpn/suite_test.go +++ b/internal/controller/evpn/suite_test.go @@ -114,12 +114,24 @@ var _ = BeforeSuite(func() { }).SetupWithManager(k8sManager) Expect(err).NotTo(HaveOccurred()) + err = (&poolcontroller.IPPrefixPoolReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + }).SetupWithManager(k8sManager) + Expect(err).NotTo(HaveOccurred()) + err = (&poolcontroller.IPAddressReconciler{ Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(), }).SetupWithManager(ctx, k8sManager) Expect(err).NotTo(HaveOccurred()) + err = (&poolcontroller.IPPrefixReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + }).SetupWithManager(ctx, k8sManager) + Expect(err).NotTo(HaveOccurred()) + err = (&poolcontroller.ClaimReconciler{ Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(),