From e9daec51716663d1924adc3dc5192db3bdf9392c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20K=C3=A4stner?= Date: Thu, 18 Jun 2026 16:48:08 +0200 Subject: [PATCH 1/5] Enable verbose logging in development overlay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Felix Kästner --- config/develop/manager_patch.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/develop/manager_patch.yaml b/config/develop/manager_patch.yaml index e07e73a48..4fddb21a4 100644 --- a/config/develop/manager_patch.yaml +++ b/config/develop/manager_patch.yaml @@ -6,3 +6,4 @@ - --provider=openconfig - --requeue-interval=30s - --max-concurrent-reconciles=5 + - --zap-log-level=3 From 7175be5b1a29a2058458a5ce8ee520cea6153533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20K=C3=A4stner?= Date: Thu, 18 Jun 2026 16:48:34 +0200 Subject: [PATCH 2/5] Include data.msg in NX-API RPC error messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Felix Kästner --- internal/transport/nxapi/nxapi.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/transport/nxapi/nxapi.go b/internal/transport/nxapi/nxapi.go index 5881ecbfd..6606820b9 100644 --- a/internal/transport/nxapi/nxapi.go +++ b/internal/transport/nxapi/nxapi.go @@ -220,6 +220,12 @@ type RPCError struct { } func (e *RPCError) Error() string { + var detail struct { + Msg string `json:"msg"` + } + if json.Unmarshal(e.Data, &detail) == nil && detail.Msg != "" { + return fmt.Sprintf("nxapi: RPC error %d: %s: %s", e.Code, e.Message, detail.Msg) + } return fmt.Sprintf("nxapi: RPC error %d: %s", e.Code, e.Message) } From 466036416c60a0f48679488f485367855eeecfc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20K=C3=A4stner?= Date: Thu, 18 Jun 2026 16:49:00 +0200 Subject: [PATCH 3/5] Use ordered iota for NX-OS version comparison MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Version type was a plain string, which meant comparisons like VersionUnknown < VersionNX10_7_1 were lexicographic and incorrect ("Unknown" > "10.7(1)"). Switch to uint8 with iota so that relational operators reflect actual release ordering. Signed-off-by: Felix Kästner --- internal/provider/cisco/nxos/version.go | 61 ++++++++++++++++++++----- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/internal/provider/cisco/nxos/version.go b/internal/provider/cisco/nxos/version.go index bd21c1a1a..9ba556486 100644 --- a/internal/provider/cisco/nxos/version.go +++ b/internal/provider/cisco/nxos/version.go @@ -6,22 +6,55 @@ package nxos import "github.com/ironcore-dev/network-operator/internal/transport/gnmiext" // Version represents the operating system version of the target device. -type Version string +// Versions are ordered so that comparison operators (>, >=, <, <=) reflect +// the actual release ordering. +type Version uint8 const ( - VersionUnknown Version = "Unknown" - VersionNX10_4_3 Version = "10.4(3)" - VersionNX10_4_4 Version = "10.4(4)" - VersionNX10_4_5 Version = "10.4(5)" - VersionNX10_4_6 Version = "10.4(6)" - VersionNX10_5_1 Version = "10.5(1)" - VersionNX10_5_2 Version = "10.5(2)" - VersionNX10_5_3 Version = "10.5(3)" - VersionNX10_6_1 Version = "10.6(1)" - VersionNX10_6_2 Version = "10.6(2)" - VersionNX10_6_3 Version = "10.6(3)" + VersionUnknown Version = iota + VersionNX10_4_3 // 10.4(3) + VersionNX10_4_4 // 10.4(4) + VersionNX10_4_5 // 10.4(5) + VersionNX10_4_6 // 10.4(6) + VersionNX10_5_1 // 10.5(1) + VersionNX10_5_2 // 10.5(2) + VersionNX10_5_3 // 10.5(3) + VersionNX10_6_1 // 10.6(1) + VersionNX10_6_2 // 10.6(2) + VersionNX10_6_3 // 10.6(3) + + VersionNX10_7_1 // 10.7(1) ) +func (v Version) String() string { + switch v { + case VersionNX10_4_3: + return "10.4(3)" + case VersionNX10_4_4: + return "10.4(4)" + case VersionNX10_4_5: + return "10.4(5)" + case VersionNX10_4_6: + return "10.4(6)" + case VersionNX10_5_1: + return "10.5(1)" + case VersionNX10_5_2: + return "10.5(2)" + case VersionNX10_5_3: + return "10.5(3)" + case VersionNX10_6_1: + return "10.6(1)" + case VersionNX10_6_2: + return "10.6(2)" + case VersionNX10_6_3: + return "10.6(3)" + case VersionNX10_7_1: + return "10.7(1)" + default: + return "Unknown" + } +} + // nxosVersions maps the revision date of the Cisco-NX-OS-device yang model to the corresponding [Version]. // It is used to determine the version of the target device based on the capabilities returned by the device. var nxosVersions = map[string]Version{ @@ -35,6 +68,10 @@ var nxosVersions = map[string]Version{ "2025-08-12": VersionNX10_6_1, "2025-12-12": VersionNX10_6_2, "2026-04-24": VersionNX10_6_3, + + // VersionNX10_7_1 is the minimum version that supports CA chains via gNOI LoadCertificate. + // TODO: Update with the correct YANG model revision date once NX-OS 10.7(1) is released. + "2026-05-31": VersionNX10_7_1, } // NXVersion returns the NX-OS operating system version of the target device based on the supported models. From 59c5f39d86913b5f3bc4f2bbb35fd9943ad21c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20K=C3=A4stner?= Date: Thu, 18 Jun 2026 16:49:49 +0200 Subject: [PATCH 4/5] Include CA chain in tls.Certificate loaded from secrets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Append the ca.crt field (if present) to the PEM block passed to tls.X509KeyPair so the resulting tls.Certificate contains the full chain (leaf + intermediates + root), not just the leaf. Signed-off-by: Felix Kästner --- internal/clientutil/clientutil.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/clientutil/clientutil.go b/internal/clientutil/clientutil.go index 2093e3c0b..3feface7a 100644 --- a/internal/clientutil/clientutil.go +++ b/internal/clientutil/clientutil.go @@ -174,13 +174,20 @@ func (c *Client) TLSSecretPEM(ctx context.Context, ref *v1alpha1.SecretReference // Certificate loads a [tls.Certificate] from the referenced secret resource. // The secret must be of type 'kubernetes.io/tls' and contain the fields 'tls.crt' and 'tls.key'. +// If a 'ca.crt' field is present, it is appended to the certificate chain. func (c *Client) Certificate(ctx context.Context, ref *v1alpha1.SecretReference) (*tls.Certificate, error) { pem, err := c.TLSSecretPEM(ctx, ref) if err != nil { return nil, err } - certificate, err := tls.X509KeyPair(pem.Certificate, pem.PrivateKey) + certPEMBlock := pem.Certificate + if len(pem.CA) > 0 { + certPEMBlock = append(certPEMBlock, '\n') + certPEMBlock = append(certPEMBlock, pem.CA...) + } + + certificate, err := tls.X509KeyPair(certPEMBlock, pem.PrivateKey) if err != nil { return nil, fmt.Errorf("failed to load x509 key pair: %w", err) } From 7c354261c5fb7361eadde158e6c25e8d7f9efb33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20K=C3=A4stner?= Date: Thu, 18 Jun 2026 17:49:42 +0200 Subject: [PATCH 5/5] Add NX-API PKCS#12 workaround for certificate provisioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gNOI LoadCertificate on NX-OS < 10.7(1) does not support CA certificate chains and creates immutable KeyPair entries that block rotation. Work around both issues by importing a PKCS#12 bundle via NX-API CLI for versions below 10.7(1). The NX-API path encodes the certificate, chain, and private key into a PKCS#12 bundle (LegacyRC2 for NX-OS compatibility), writes it to bootflash in chunks via base64, and imports it with crypto ca import. Existing trustpoint and keypair are cleaned up via idempotent gNMI delete before each import. Also passes CA certificates to gNOI LoadCertificate for the >= 10.7(1) path so the full chain is available once that version ships with chain support. Signed-off-by: Felix Kästner --- go.mod | 1 + go.sum | 2 + internal/provider/cisco/nxos/cert.go | 67 +++++++++--- internal/provider/cisco/nxos/provider.go | 129 ++++++++++++++++++++--- 4 files changed, 168 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 31f543d34..c02853bb3 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( rsc.io/script v0.0.2 sigs.k8s.io/controller-runtime v0.24.1 sigs.k8s.io/yaml v1.6.0 + software.sslmate.com/src/go-pkcs12 v0.7.1 ) require ( diff --git a/go.sum b/go.sum index 4ae727d67..25f3357d6 100644 --- a/go.sum +++ b/go.sum @@ -311,3 +311,5 @@ sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80 sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= +software.sslmate.com/src/go-pkcs12 v0.7.1 h1:bxkUPRsvTPNRBZa4M/aSX4PyMOEbq3V8I6hbkG4F4Q8= +software.sslmate.com/src/go-pkcs12 v0.7.1/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= diff --git a/internal/provider/cisco/nxos/cert.go b/internal/provider/cisco/nxos/cert.go index e9cf67b2d..90f08cc44 100644 --- a/internal/provider/cisco/nxos/cert.go +++ b/internal/provider/cisco/nxos/cert.go @@ -6,14 +6,15 @@ package nxos import ( "bytes" "context" - "crypto/rand" "crypto/rsa" + "crypto/tls" "crypto/x509" "encoding/pem" "fmt" - "github.com/openconfig/gnoi/cert" + certpb "github.com/openconfig/gnoi/cert" "google.golang.org/grpc" + pkcs12 "software.sslmate.com/src/go-pkcs12" "github.com/ironcore-dev/network-operator/internal/transport/gnmiext" ) @@ -21,8 +22,9 @@ import ( // Certificate represents a X.509 certificate and its associated private key. // It can be used to load the certificate into a NX-OS device truspoint via gNOI. type Certificate struct { - Key *rsa.PrivateKey - Cert *x509.Certificate + Key *rsa.PrivateKey + Cert *x509.Certificate + CACerts []*x509.Certificate } // Load loads the certificate into the specified trustpoint via the gNOI [cert service]. @@ -39,26 +41,31 @@ func (c *Certificate) Load(ctx context.Context, conn *grpc.ClientConn, trustpoin return err } - // Only the `LoadCertificate` method is currently supported on the Nexus 9000 series, despite the fact that the gNOI certificate service is deprecated in favor of the gNSI certz service. - // See: https://www.cisco.com/c/en/us/td/docs/dcn/nx-os/nexus9000/104x/programmability/cisco-nexus-9000-series-nx-os-programmability-guide-104x/gnoi---operation-interface.html - _, err = cert.NewCertificateManagementClient(conn).LoadCertificate(ctx, &cert.LoadCertificateRequest{ //nolint:staticcheck - Certificate: &cert.Certificate{Type: cert.CertificateType_CT_X509, Certificate: b}, - KeyPair: &cert.KeyPair{PrivateKey: priv, PublicKey: pub}, - CertificateId: trustpoint, + var chain []*certpb.Certificate + for _, ca := range c.CACerts { + var buf bytes.Buffer + if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: ca.Raw}); err != nil { + return fmt.Errorf("failed to encode CA certificate: %w", err) + } + chain = append(chain, &certpb.Certificate{Type: certpb.CertificateType_CT_X509, Certificate: buf.Bytes()}) + } + + // Nexus 9000 series only supports the gNOI certificate management service, despite the fact it is deprecated in favor of the gNSI certz service. + // See: https://www.cisco.com/c/en/us/td/docs/dcn/nx-os/nexus9000/106x/programmability/cisco-nexus-9000-series-nx-os-programmability-guide-106x/gnoi---operation-interface.html + _, err = certpb.NewCertificateManagementClient(conn).LoadCertificate(ctx, &certpb.LoadCertificateRequest{ //nolint:staticcheck + Certificate: &certpb.Certificate{Type: certpb.CertificateType_CT_X509, Certificate: b}, + KeyPair: &certpb.KeyPair{PrivateKey: priv, PublicKey: pub}, + CertificateId: trustpoint, + CaCertificates: chain, }, grpc.WaitForReady(true)) return err } func (c *Certificate) Encode() ([]byte, error) { - // Self-sign the certificate as Cisco NX-OS does not support uploading a certificate chain via gNOI. - der, err := x509.CreateCertificate(rand.Reader, c.Cert, c.Cert, &c.Key.PublicKey, c.Key) - if err != nil { - return nil, fmt.Errorf("failed to create certificate: %w", err) - } var buf bytes.Buffer - err = pem.Encode(&buf, &pem.Block{ + err := pem.Encode(&buf, &pem.Block{ Type: "CERTIFICATE", - Bytes: der, + Bytes: c.Cert.Raw, }) if err != nil { return nil, fmt.Errorf("failed to encode certificate: %w", err) @@ -115,3 +122,29 @@ func (*KeyPair) IsListItem() {} func (k *KeyPair) XPath() string { return "System/userext-items/pkiext-items/keyring-items/KeyRing-list[name=" + k.Name + "]" } + +// EncodeCertificatePKCS12 encodes a tls.Certificate (leaf + chain + private key) into +// PKCS#12 format protected by the given passphrase. LegacyRC2 encoding is used +// for compatibility with NX-OS. +func EncodeCertificatePKCS12(cert *tls.Certificate, passphrase string) ([]byte, error) { + key, ok := cert.PrivateKey.(*rsa.PrivateKey) + if !ok { + return nil, fmt.Errorf("unsupported private key type: expected *rsa.PrivateKey, got %T", cert.PrivateKey) + } + + var chain []*x509.Certificate + for _, der := range cert.Certificate[1:] { + ca, err := x509.ParseCertificate(der) + if err != nil { + return nil, fmt.Errorf("failed to parse CA certificate: %w", err) + } + chain = append(chain, ca) + } + + pfx, err := pkcs12.LegacyRC2.Encode(key, cert.Leaf, chain, passphrase) + if err != nil { + return nil, fmt.Errorf("failed to encode PKCS#12: %w", err) + } + + return pfx, nil +} diff --git a/internal/provider/cisco/nxos/provider.go b/internal/provider/cisco/nxos/provider.go index 4a9c1df53..d9b10687d 100644 --- a/internal/provider/cisco/nxos/provider.go +++ b/internal/provider/cisco/nxos/provider.go @@ -9,6 +9,10 @@ import ( "context" "crypto/rand" "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/hex" + "encoding/json" "errors" "fmt" "maps" @@ -91,7 +95,7 @@ func (p *Provider) Connect(ctx context.Context, conn *deviceutil.Connection) (er } // NXAPI only uses the address for URI construction. c := *conn - c.Address = netip.MustParseAddrPort(conn.Address).String() + c.Address = netip.MustParseAddrPort(conn.Address).Addr().String() p.nxapi, err = nxapi.NewClient(&c, timeout) if err != nil { return fmt.Errorf("failed to create nxapi client: %w", err) @@ -641,27 +645,124 @@ func (p *Provider) GetPeerStatus(ctx context.Context, req *provider.BGPPeerStatu } func (p *Provider) EnsureCertificate(ctx context.Context, req *provider.EnsureCertificateRequest) error { - tp := new(Trustpoint) - tp.Name = req.ID + logger := logr.FromContextOrDiscard(ctx) - if err := p.Patch(ctx, tp); err != nil { - return err + if serial, err := p.installedCertSerial(ctx, req.ID); err == nil { + want := strings.ToUpper(req.Certificate.Leaf.SerialNumber.Text(16)) + if strings.TrimLeft(serial, "0") == strings.TrimLeft(want, "0") { + logger.V(1).Info("Certificate already installed with matching serial", "serial", serial) + return nil + } } - key, ok := req.Certificate.PrivateKey.(*rsa.PrivateKey) - if !ok { - return fmt.Errorf("unsupported private key type: expected *rsa.PrivateKey, got %T", req.Certificate.PrivateKey) + if NXVersion(p.client.Capabilities()) >= VersionNX10_7_1 { + tp := new(Trustpoint) + tp.Name = req.ID + + if err := p.Patch(ctx, tp); err != nil { + return err + } + + key, ok := req.Certificate.PrivateKey.(*rsa.PrivateKey) + if !ok { + return apistatus.NewInvalidArgumentError(apistatus.FieldViolation{ + Field: "spec.certificate.privateKey", + Description: fmt.Sprintf("unsupported private key type: expected *rsa.PrivateKey, got %T", req.Certificate.PrivateKey), + }) + } + + cert := &Certificate{Key: key, Cert: req.Certificate.Leaf} + for _, der := range req.Certificate.Certificate[1:] { + ca, err := x509.ParseCertificate(der) + if err != nil { + return fmt.Errorf("failed to parse CA certificate: %w", err) + } + cert.CACerts = append(cert.CACerts, ca) + } + + err := cert.Load(ctx, p.conn, req.ID) + if err != nil { + return fmt.Errorf("failed to upload certificate via gNOI: %w", err) + } + + logger.V(2).Info("Certificate upload completed") + return nil } - kp := new(KeyPair) - kp.Name = req.ID - if err := p.client.GetConfig(ctx, kp); !errors.Is(err, gnmiext.ErrNil) { - // If the key pair already exists, we cannot update it, so we skip the rest of the process. + var pass [16]byte + _, _ = rand.Read(pass[:]) + passphrase := hex.EncodeToString(pass[:]) + + pfx, err := EncodeCertificatePKCS12(req.Certificate, passphrase) + if err != nil { + return err + } + + // Delete any existing trustpoint and RSA key before importing. + // gNMI delete is idempotent, so this is safe even on a fresh device. + if err := p.DeleteCertificate(ctx, &provider.DeleteCertificateRequest{ID: req.ID}); err != nil { return err } - cert := &Certificate{Key: key, Cert: req.Certificate.Leaf} - return cert.Load(ctx, p.conn, req.ID) + b64 := base64.StdEncoding.EncodeToString(pfx) + file := "/bootflash/" + req.ID + ".pfx" + b64File := file + ".b64" + + cmds := []string{ + "feature bash-shell", + "crypto ca trustpoint " + req.ID, + } + const chunkSize = 512 + for i := 0; i < len(b64); i += chunkSize { + end := min(i+chunkSize, len(b64)) + op := " >> " + if i == 0 { + op = " > " + } + cmds = append(cmds, "run bash echo -n '"+b64[i:end]+"'"+op+b64File) + } + cmds = append( + cmds, + "run bash base64 -d "+b64File+" > "+file, + "run bash rm "+b64File, + "crypto ca import "+req.ID+" pkcs12 bootflash:///"+req.ID+".pfx "+passphrase, + "run bash rm "+file, + ) + + _, err = p.nxapi.Do(ctx, nxapi.NewRequest(cmds...).WithRollback(nxapi.Stop)) + if err != nil { + return fmt.Errorf("failed to upload certificate via NX-API: %w", err) + } + + logger.V(2).Info("Certificate upload completed") + return nil +} + +// installedCertSerial queries the device for the certificate installed under the +// given trustpoint and returns its serial number as an uppercase hex string. +// If the trustpoint does not exist or has no certificate, an error is returned. +func (p *Provider) installedCertSerial(ctx context.Context, trustpoint string) (string, error) { + res, err := p.nxapi.Do(ctx, nxapi.NewRequest("show crypto ca certificates "+trustpoint)) + if err != nil { + return "", err + } + if len(res) == 0 { + return "", errors.New("empty response") + } + var body struct { + Certificate struct { + Cert string `json:"certificate"` + } `json:"Certificate"` + } + if err := json.Unmarshal(res[0], &body); err != nil { + return "", err + } + for line := range strings.SplitSeq(body.Certificate.Cert, "\n") { + if after, ok := strings.CutPrefix(line, "serial="); ok { + return strings.ToUpper(strings.TrimSpace(after)), nil + } + } + return "", errors.New("serial not found in certificate output") } func (p *Provider) DeleteCertificate(ctx context.Context, req *provider.DeleteCertificateRequest) error {