From be0aa063bed65c5aa3c5c6b39387f60bb102786a Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Thu, 6 Aug 2026 12:52:42 -0400 Subject: [PATCH] feat(nvca): configure transport TLS mount path --- .../nvca-operator/nvca-operator/README.md | 1 + .../templates/operator-config-cm.yaml | 3 + .../nvca-operator/values.schema.json | 5 + .../nvca-operator/nvca-operator/values.yaml | 2 + .../nvca/deployments/nvca-operator/README.md | 1 + .../templates/operator-config-cm.yaml | 3 + .../nvca-operator/values.schema.json | 5 + .../deployments/nvca-operator/values.yaml | 2 + .../miniservice/transport_tls_test.go | 7 +- .../nvca/internal/transporttls/BUILD.bazel | 2 + .../internal/transporttls/transport_tls.go | 57 ++++- .../transporttls/transport_tls_test.go | 194 ++++++++++++++++++ .../nvca/pkg/nvca/transport_tls_test.go | 7 +- .../operator/reconcile/nvca_config_mapper.go | 18 +- .../reconcile/nvca_config_mapper_test.go | 48 +++++ .../reconcile/transport_tls_config_test.go | 75 +++++++ .../nvca/scripts/lint_helm.sh | 25 ++- .../go/lib/pkg/types/nvca/config/types.go | 1 + .../go/lib/pkg/types/nvca/config/types.go | 1 + 19 files changed, 439 insertions(+), 18 deletions(-) diff --git a/deploy/helm/nvca-operator/nvca-operator/README.md b/deploy/helm/nvca-operator/nvca-operator/README.md index 3df6bbfc4..5538be8b0 100644 --- a/deploy/helm/nvca-operator/nvca-operator/README.md +++ b/deploy/helm/nvca-operator/nvca-operator/README.md @@ -47,6 +47,7 @@ used in Kubernetes Clusters to run NVCF Workloads. | `operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.name` | Secret containing the workload transport trust bundle; empty disables the source. Example: `nvcf-trust`. | `""` | | `operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.key` | Secret data key containing certificate-only PEM. | `ca.crt` | | `operatorConfig.workload.transportTLS.fingerprint` | Optional SHA-256 pin; empty computes the Secret data fingerprint. | `""` | +| `operatorConfig.workload.transportTLS.installedBundleMountPath` | Optional `llm-worker` mount path for the installed transport trust bundle; empty uses `/etc/ssl/certs`. | `""` | ### resources Resource requests and limits for the nvca-operator container diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/operator-config-cm.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/operator-config-cm.yaml index 2eb92169a..3b5270fc0 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/operator-config-cm.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/operator-config-cm.yaml @@ -33,3 +33,6 @@ data: name: {{ $secretKeyRef.name | default "" | quote }} key: {{ $secretKeyRef.key | default "ca.crt" | quote }} fingerprint: {{ $transportTLS.fingerprint | default "" | quote }} + {{- if $transportTLS.installedBundleMountPath }} + installedBundleMountPath: {{ $transportTLS.installedBundleMountPath | quote }} + {{- end }} diff --git a/deploy/helm/nvca-operator/nvca-operator/values.schema.json b/deploy/helm/nvca-operator/nvca-operator/values.schema.json index 7df16658a..efe697435 100644 --- a/deploy/helm/nvca-operator/nvca-operator/values.schema.json +++ b/deploy/helm/nvca-operator/nvca-operator/values.schema.json @@ -239,6 +239,11 @@ "type": "string", "description": "Optional sha256 fingerprint pin for the trust bundle. Empty computes the fingerprint from the selected Secret data.", "default": "" + }, + "installedBundleMountPath": { + "type": "string", + "description": "Optional llm-worker mount path for the installed transport trust bundle. Empty uses /etc/ssl/certs.", + "default": "" } } } diff --git a/deploy/helm/nvca-operator/nvca-operator/values.yaml b/deploy/helm/nvca-operator/nvca-operator/values.yaml index 46ba9e90d..c4c732614 100644 --- a/deploy/helm/nvca-operator/nvca-operator/values.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/values.yaml @@ -130,6 +130,7 @@ agentConfig: ## @param operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.name Secret containing the workload transport trust bundle. Empty disables Secret-backed transport trust. Example: nvcf-trust. ## @param operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.key Secret data key containing certificate-only PEM. ## @param operatorConfig.workload.transportTLS.fingerprint Optional sha256 fingerprint pin for the trust bundle. Empty computes the fingerprint from the selected Secret data. +## @param operatorConfig.workload.transportTLS.installedBundleMountPath Optional llm-worker mount path for the installed transport trust bundle. Empty uses /etc/ssl/certs. operatorConfig: workload: transportTLS: @@ -138,6 +139,7 @@ operatorConfig: name: "" key: ca.crt fingerprint: "" + installedBundleMountPath: "" ## @section resources Resource requests and limits for the nvca-operator container ## @param resources.limits.cpu CPU limit for the nvca-operator container ## @param resources.limits.memory Memory limit for the nvca-operator container diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/README.md b/src/compute-plane-services/nvca/deployments/nvca-operator/README.md index e15fde4fe..2ac6b1dff 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/README.md +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/README.md @@ -47,6 +47,7 @@ used in Kubernetes Clusters to run NVCF Workloads. | `operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.name` | Secret containing the workload transport trust bundle; empty disables the source. Example: `nvcf-trust`. | `""` | | `operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.key` | Secret data key containing certificate-only PEM. | `ca.crt` | | `operatorConfig.workload.transportTLS.fingerprint` | Optional SHA-256 pin; empty computes the Secret data fingerprint. | `""` | +| `operatorConfig.workload.transportTLS.installedBundleMountPath` | Optional `llm-worker` mount path for the installed transport trust bundle; empty uses `/etc/ssl/certs`. | `""` | ### resources Resource requests and limits for the nvca-operator container diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/operator-config-cm.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/operator-config-cm.yaml index 2eb92169a..3b5270fc0 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/operator-config-cm.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/operator-config-cm.yaml @@ -33,3 +33,6 @@ data: name: {{ $secretKeyRef.name | default "" | quote }} key: {{ $secretKeyRef.key | default "ca.crt" | quote }} fingerprint: {{ $transportTLS.fingerprint | default "" | quote }} + {{- if $transportTLS.installedBundleMountPath }} + installedBundleMountPath: {{ $transportTLS.installedBundleMountPath | quote }} + {{- end }} diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/values.schema.json b/src/compute-plane-services/nvca/deployments/nvca-operator/values.schema.json index 8b7ed0081..853bab609 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/values.schema.json +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/values.schema.json @@ -239,6 +239,11 @@ "type": "string", "description": "Optional sha256 fingerprint pin for the trust bundle. Empty computes the fingerprint from the selected Secret data.", "default": "" + }, + "installedBundleMountPath": { + "type": "string", + "description": "Optional llm-worker mount path for the installed transport trust bundle. Empty uses /etc/ssl/certs.", + "default": "" } } } diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml index 30253ec52..e3e0e8844 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml @@ -146,6 +146,7 @@ agentConfig: ## @param operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.name Secret containing the workload transport trust bundle. Empty disables Secret-backed transport trust. Example: nvcf-trust. ## @param operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.key Secret data key containing certificate-only PEM. ## @param operatorConfig.workload.transportTLS.fingerprint Optional sha256 fingerprint pin for the trust bundle. Empty computes the fingerprint from the selected Secret data. +## @param operatorConfig.workload.transportTLS.installedBundleMountPath Optional llm-worker mount path for the installed transport trust bundle. Empty uses /etc/ssl/certs. operatorConfig: workload: transportTLS: @@ -154,6 +155,7 @@ operatorConfig: name: "" key: ca.crt fingerprint: "" + installedBundleMountPath: "" ## @section resources Resource requests and limits for the nvca-operator container ## @param resources.limits.cpu CPU limit for the nvca-operator container diff --git a/src/compute-plane-services/nvca/internal/miniservice/transport_tls_test.go b/src/compute-plane-services/nvca/internal/miniservice/transport_tls_test.go index cdea49748..681d73bea 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/transport_tls_test.go +++ b/src/compute-plane-services/nvca/internal/miniservice/transport_tls_test.go @@ -81,6 +81,7 @@ func TestPrepareTransportTLSForWorkloadsInjectsPodLLMWorker(t *testing.T) { TrustBundleFingerprint: testTransportTLSRootFingerprint, TrustBundlePEM: testTransportTLSRootCertPEM, InstallerImage: "nvcr.io/nvidia/nvcf-byoc/nvca:test", + InstalledBundleMountPath: "/nvcf/transport-tls", }, }, }, @@ -119,9 +120,11 @@ func TestPrepareTransportTLSForWorkloadsInjectsPodLLMWorker(t *testing.T) { assert.NotNil(t, findWorkloadInitContainer(podSpec, "nvcf-trust-bundle-install")) llmWorker := findWorkloadContainer(podSpec, function.LLMWorkerContainerName) require.NotNil(t, llmWorker) - assert.Equal(t, "/etc/ssl/certs/ca-certificates.crt", + assert.Equal(t, "/nvcf/transport-tls/ca-certificates.crt", findWorkloadEnvValue(llmWorker, "STARGATE_TLS_CERT_PATH")) - assert.NotNil(t, findWorkloadVolumeMount(llmWorker, "nvcf-trust-merged-certs")) + mount := findWorkloadVolumeMount(llmWorker, "nvcf-trust-merged-certs") + require.NotNil(t, mount) + assert.Equal(t, "/nvcf/transport-tls", mount.MountPath) for _, name := range []string{"inference", "smb-server"} { container := findWorkloadContainer(podSpec, name) diff --git a/src/compute-plane-services/nvca/internal/transporttls/BUILD.bazel b/src/compute-plane-services/nvca/internal/transporttls/BUILD.bazel index ca68c958d..2b2f7e79e 100644 --- a/src/compute-plane-services/nvca/internal/transporttls/BUILD.bazel +++ b/src/compute-plane-services/nvca/internal/transporttls/BUILD.bazel @@ -30,10 +30,12 @@ go_test( srcs = ["transport_tls_test.go"], embed = [":transporttls"], deps = [ + "//vendor/github.com/evanphx/json-patch/v5:json-patch", "//vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function", "//vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config", "//vendor/github.com/stretchr/testify/assert", "//vendor/github.com/stretchr/testify/require", "//vendor/k8s.io/api/core/v1:core", + "//vendor/k8s.io/apimachinery/pkg/util/strategicpatch", ], ) diff --git a/src/compute-plane-services/nvca/internal/transporttls/transport_tls.go b/src/compute-plane-services/nvca/internal/transporttls/transport_tls.go index e96761db0..68a19644b 100644 --- a/src/compute-plane-services/nvca/internal/transporttls/transport_tls.go +++ b/src/compute-plane-services/nvca/internal/transporttls/transport_tls.go @@ -19,6 +19,7 @@ package transporttls import ( "fmt" + "path" "regexp" "strings" @@ -48,8 +49,9 @@ const ( TrustBundleMountPath = "/nvcf/trust" MergedCertsMountPath = "/merged-certs" MergedCertsFile = "/merged-certs/ca-certificates.crt" + InstalledBundleFile = "ca-certificates.crt" SystemCertDir = "/etc/ssl/certs" - SystemCertFile = "/etc/ssl/certs/ca-certificates.crt" + SystemCertFile = "/etc/ssl/certs/" + InstalledBundleFile CertPathEnv = "STARGATE_TLS_CERT_PATH" ) @@ -65,6 +67,9 @@ func NormalizeConfig(cfg nvcaconfig.TransportTLSConfig) nvcaconfig.TransportTLSC if cfg.TrustBundleKey == "" { cfg.TrustBundleKey = DefaultTrustBundleKey } + if cfg.InstalledBundleMountPath == "" { + cfg.InstalledBundleMountPath = SystemCertDir + } cfg.TrustBundleFingerprint = strings.ToLower(strings.TrimSpace(cfg.TrustBundleFingerprint)) return cfg } @@ -74,6 +79,9 @@ func ValidateConfig(cfg nvcaconfig.TransportTLSConfig) error { case TrustModeSystem: return nil case TrustModeBundle: + if err := validateInstalledBundleMountPath(cfg.InstalledBundleMountPath); err != nil { + return err + } if errs := validation.IsDNS1123Subdomain(cfg.TrustBundleConfigMapName); len(errs) > 0 { return fmt.Errorf("transportTls.trustBundleConfigMapName is invalid: %s", strings.Join(errs, "; ")) } @@ -102,6 +110,23 @@ func ValidateConfig(cfg nvcaconfig.TransportTLSConfig) error { } } +func validateInstalledBundleMountPath(mountPath string) error { + if !path.IsAbs(mountPath) { + return fmt.Errorf("transportTls.installedBundleMountPath must be absolute") + } + if path.Clean(mountPath) != mountPath { + return fmt.Errorf("transportTls.installedBundleMountPath must be canonical") + } + if mountPath == "/" { + return fmt.Errorf("transportTls.installedBundleMountPath must not be root") + } + if mountPath == MergedCertsMountPath || strings.HasPrefix(mountPath, MergedCertsMountPath+"/") || + mountPath == TrustBundleMountPath || strings.HasPrefix(mountPath, TrustBundleMountPath+"/") { + return fmt.Errorf("transportTls.installedBundleMountPath uses reserved path %q", mountPath) + } + return nil +} + func FingerprintTrustBundle(trustBundlePEM string) (string, error) { return trustbundle.FingerprintPEM(trustBundlePEM) } @@ -127,22 +152,48 @@ func InjectIntoPodSpec(podSpec *corev1.PodSpec, cfg nvcaconfig.TransportTLSConfi if err != nil { return err } + if err := validateInstalledBundleMountConflict(&podSpec.Containers[llmWorkerIdx], cfg.InstalledBundleMountPath); err != nil { + return err + } upsertVolumes(podSpec, cfg) upsertInstallContainer(podSpec, installImage, installImagePullPolicy, cfg) llmWorker := &podSpec.Containers[llmWorkerIdx] upsertVolumeMount(&llmWorker.VolumeMounts, corev1.VolumeMount{ Name: MergedCertsVolumeName, - MountPath: SystemCertDir, + MountPath: cfg.InstalledBundleMountPath, ReadOnly: true, }) k8sutil.AddEnvsToContainer(llmWorker, corev1.EnvVar{ Name: CertPathEnv, - Value: SystemCertFile, + Value: cfg.InstalledBundleMountPath + "/" + InstalledBundleFile, }) return nil } +func validateInstalledBundleMountConflict(container *corev1.Container, mountPath string) error { + for _, mount := range container.VolumeMounts { + if mount.Name != MergedCertsVolumeName && mountPathsOverlap(mount.MountPath, mountPath) { + return fmt.Errorf("transportTls.installedBundleMountPath %q conflicts with volume mount %q on %q", + mountPath, mount.Name, container.Name) + } + } + return nil +} + +// mountPathsOverlap reports whether either mount hides the other. Kubernetes +// permits nested mounts, but using them for the NVCA bundle would make the +// installed certificate file ambiguous or inaccessible to the llm worker. +func mountPathsOverlap(first, second string) bool { + first = path.Clean(first) + second = path.Clean(second) + + if first == "/" || second == "/" { + return true + } + return first == second || strings.HasPrefix(first, second+"/") || strings.HasPrefix(second, first+"/") +} + func resolveInstallContainerImage( podSpec *corev1.PodSpec, cfg nvcaconfig.TransportTLSConfig, diff --git a/src/compute-plane-services/nvca/internal/transporttls/transport_tls_test.go b/src/compute-plane-services/nvca/internal/transporttls/transport_tls_test.go index 1de9b4363..1c77d7fee 100644 --- a/src/compute-plane-services/nvca/internal/transporttls/transport_tls_test.go +++ b/src/compute-plane-services/nvca/internal/transporttls/transport_tls_test.go @@ -18,13 +18,16 @@ limitations under the License. package transporttls import ( + "encoding/json" "testing" "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function" nvcaconfig "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config" + evanphxpatch "github.com/evanphx/json-patch/v5" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/strategicpatch" ) const testRootCertPEM = `-----BEGIN CERTIFICATE----- @@ -69,6 +72,12 @@ func TestValidateConfigRejectsMismatchedFingerprint(t *testing.T) { assert.Contains(t, err.Error(), "trustBundleFingerprint") } +func TestNormalizeConfigDefaultsInstalledBundleMountPath(t *testing.T) { + cfg := NormalizeConfig(nvcaconfig.TransportTLSConfig{}) + + assert.Equal(t, "/etc/ssl/certs", cfg.InstalledBundleMountPath) +} + func TestValidateConfigRejectsInvalidKubernetesNames(t *testing.T) { tests := []struct { name string @@ -115,6 +124,62 @@ func TestValidateConfigRejectsInvalidKubernetesNames(t *testing.T) { } } +func TestValidateConfigRejectsInvalidInstalledBundleMountPaths(t *testing.T) { + tests := []struct { + name string + mountPath string + wantErr string + }{ + { + name: "relative path", + mountPath: "nvcf/transport-tls", + wantErr: "must be absolute", + }, + { + name: "noncanonical path", + mountPath: "/nvcf/../transport-tls", + wantErr: "must be canonical", + }, + { + name: "root path", + mountPath: "/", + wantErr: "must not be root", + }, + { + name: "installer output path", + mountPath: MergedCertsMountPath, + wantErr: "reserved", + }, + { + name: "trust source path", + mountPath: TrustBundleMountPath, + wantErr: "reserved", + }, + { + name: "under installer output path", + mountPath: MergedCertsMountPath + "/nested", + wantErr: "reserved", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := NormalizeConfig(nvcaconfig.TransportTLSConfig{ + TrustMode: nvcaconfig.TrustModeBundle, + TrustBundleFingerprint: testRootFingerprint, + TrustBundlePEM: testRootCertPEM, + InstalledBundleMountPath: tt.mountPath, + }) + + err := ValidateConfig(cfg) + + require.Error(t, err) + assert.Contains(t, err.Error(), "installedBundleMountPath") + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + func TestInjectIntoPodSpecOnlyMutatesLLMWorker(t *testing.T) { podSpec := &corev1.PodSpec{ Containers: []corev1.Container{ @@ -187,6 +252,135 @@ func TestInjectIntoPodSpecUsesNVCAInstallCommand(t *testing.T) { assert.False(t, *installContainer.SecurityContext.RunAsNonRoot) } +func TestInjectIntoPodSpecUsesConfiguredInstalledBundleMountPath(t *testing.T) { + podSpec := &corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: function.LLMWorkerContainerName, + Image: "nvcr.io/nvcf/llm-worker:test", + }}, + } + + err := InjectIntoPodSpec(podSpec, NormalizeConfig(nvcaconfig.TransportTLSConfig{ + TrustMode: nvcaconfig.TrustModeBundle, + TrustBundleFingerprint: testRootFingerprint, + TrustBundlePEM: testRootCertPEM, + InstallerImage: "nvcr.io/nvidia/nvcf-byoc/nvca:test", + InstalledBundleMountPath: "/nvcf/transport-tls", + })) + require.NoError(t, err) + + llmWorker := findTestContainer(podSpec, function.LLMWorkerContainerName) + require.NotNil(t, llmWorker) + mount := findTestVolumeMount(llmWorker, MergedCertsVolumeName) + require.NotNil(t, mount) + assert.Equal(t, "/nvcf/transport-tls", mount.MountPath) + assert.Equal(t, "/nvcf/transport-tls/ca-certificates.crt", findTestEnvValue(llmWorker, CertPathEnv)) + + installContainer := findTestInitContainer(podSpec, InstallContainerName) + require.NotNil(t, installContainer) + assert.Equal(t, MergedCertsMountPath, findTestVolumeMount(installContainer, MergedCertsVolumeName).MountPath) + assert.Contains(t, installContainer.Args, MergedCertsFile) +} + +func TestInjectIntoPodSpecRejectsConflictingInstalledBundleMount(t *testing.T) { + podSpec := &corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: function.LLMWorkerContainerName, + Image: "nvcr.io/nvcf/llm-worker:test", + VolumeMounts: []corev1.VolumeMount{{ + Name: "kyverno-certs", + MountPath: "/nvcf/transport-tls", + }}, + }}, + } + + err := InjectIntoPodSpec(podSpec, NormalizeConfig(nvcaconfig.TransportTLSConfig{ + TrustMode: nvcaconfig.TrustModeBundle, + TrustBundleFingerprint: testRootFingerprint, + TrustBundlePEM: testRootCertPEM, + InstallerImage: "nvcr.io/nvidia/nvcf-byoc/nvca:test", + InstalledBundleMountPath: "/nvcf/transport-tls", + })) + + require.Error(t, err) + assert.Contains(t, err.Error(), "conflicts with volume mount") + assert.Nil(t, findTestVolume(podSpec, TrustBundleVolumeName)) + assert.Nil(t, findTestVolume(podSpec, MergedCertsVolumeName)) + assert.Nil(t, findTestInitContainer(podSpec, InstallContainerName)) +} + +func TestInjectIntoPodSpecRejectsOverlappingInstalledBundleMount(t *testing.T) { + for _, existingMountPath := range []string{"/", "/nvcf", "/nvcf/", "/nvcf/transport-tls/nested"} { + t.Run(existingMountPath, func(t *testing.T) { + podSpec := &corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: function.LLMWorkerContainerName, + Image: "nvcr.io/nvcf/llm-worker:test", + VolumeMounts: []corev1.VolumeMount{{ + Name: "existing-certs", + MountPath: existingMountPath, + }}, + }}, + } + + err := InjectIntoPodSpec(podSpec, NormalizeConfig(nvcaconfig.TransportTLSConfig{ + TrustMode: nvcaconfig.TrustModeBundle, + TrustBundleFingerprint: testRootFingerprint, + TrustBundlePEM: testRootCertPEM, + InstallerImage: "nvcr.io/nvidia/nvcf-byoc/nvca:test", + InstalledBundleMountPath: "/nvcf/transport-tls", + })) + + require.Error(t, err) + assert.Contains(t, err.Error(), "conflicts with volume mount") + assert.Nil(t, findTestVolume(podSpec, TrustBundleVolumeName)) + assert.Nil(t, findTestVolume(podSpec, MergedCertsVolumeName)) + assert.Nil(t, findTestInitContainer(podSpec, InstallContainerName)) + }) + } +} + +func TestInjectIntoPodSpecCoexistsWithAdmissionCertificateMounts(t *testing.T) { + pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: function.LLMWorkerContainerName, + Image: "nvcr.io/nvcf/llm-worker:test", + }}}} + base, err := json.Marshal(pod) + require.NoError(t, err) + + kyvernoStrategicMergePatch := []byte(`{"spec":{"volumes":[{"name":"kyverno-certs","emptyDir":{}}],"containers":[{"name":"llm-worker","volumeMounts":[{"name":"kyverno-certs","mountPath":"/etc/ssl/certs","readOnly":true}]}]}}`) + kyvernoMutated, err := strategicpatch.StrategicMergePatch(base, kyvernoStrategicMergePatch, corev1.Pod{}) + require.NoError(t, err) + + webhookJSONPatch, err := evanphxpatch.DecodePatch([]byte(`[ + {"op":"add","path":"/spec/volumes/-","value":{"name":"webhook-certs","emptyDir":{}}}, + {"op":"add","path":"/spec/containers/0/volumeMounts/-","value":{"name":"webhook-certs","mountPath":"/webhook/certs","readOnly":true}} +]`)) + require.NoError(t, err) + admissionMutated, err := webhookJSONPatch.Apply(kyvernoMutated) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(admissionMutated, pod)) + + err = InjectIntoPodSpec(&pod.Spec, NormalizeConfig(nvcaconfig.TransportTLSConfig{ + TrustMode: nvcaconfig.TrustModeBundle, + TrustBundleFingerprint: testRootFingerprint, + TrustBundlePEM: testRootCertPEM, + InstallerImage: "nvcr.io/nvidia/nvcf-byoc/nvca:test", + InstalledBundleMountPath: "/nvcf/transport-tls", + })) + require.NoError(t, err) + + llmWorker := findTestContainer(&pod.Spec, function.LLMWorkerContainerName) + require.NotNil(t, llmWorker) + assert.Equal(t, "/etc/ssl/certs", findTestVolumeMount(llmWorker, "kyverno-certs").MountPath) + assert.Equal(t, "/webhook/certs", findTestVolumeMount(llmWorker, "webhook-certs").MountPath) + assert.Equal(t, "/nvcf/transport-tls", findTestVolumeMount(llmWorker, MergedCertsVolumeName).MountPath) + assert.Equal(t, "/nvcf/transport-tls/ca-certificates.crt", findTestEnvValue(llmWorker, CertPathEnv)) + assert.NotNil(t, findTestVolume(&pod.Spec, TrustBundleVolumeName)) + assert.NotNil(t, findTestVolume(&pod.Spec, MergedCertsVolumeName)) + assert.NotNil(t, findTestInitContainer(&pod.Spec, InstallContainerName)) +} + func TestInjectIntoPodSpecUsesConfiguredInstallerImageWhenLegacyEnvIsSet(t *testing.T) { t.Setenv("NVCF_TRUST_BUNDLE_INSTALLER_IMAGE", "nvcr.io/private-mirror/nvca:test") podSpec := &corev1.PodSpec{ diff --git a/src/compute-plane-services/nvca/pkg/nvca/transport_tls_test.go b/src/compute-plane-services/nvca/pkg/nvca/transport_tls_test.go index d9a88e1ac..75561b325 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/transport_tls_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/transport_tls_test.go @@ -71,6 +71,7 @@ func TestCreatePodArtifactInstancesTransportTLSBundleInjectsOnlyLLMWorker(t *tes TrustBundleFingerprint: testTransportRootFingerprint, TrustBundlePEM: testTransportRootCertPEM, InstallerImage: "nvcr.io/nvidia/nvcf-byoc/nvca:test", + InstalledBundleMountPath: "/nvcf/transport-tls", }) req := newTransportTLSTestRequest() @@ -92,9 +93,11 @@ func TestCreatePodArtifactInstancesTransportTLSBundleInjectsOnlyLLMWorker(t *tes llmWorker := findTransportTLSContainer(createdPod, function.LLMWorkerContainerName) require.NotNil(t, llmWorker) - assert.Equal(t, "/etc/ssl/certs/ca-certificates.crt", + assert.Equal(t, "/nvcf/transport-tls/ca-certificates.crt", findTransportTLSEnvValue(llmWorker, "STARGATE_TLS_CERT_PATH")) - assert.NotNil(t, findTransportTLSVolumeMount(llmWorker, "nvcf-trust-merged-certs")) + mount := findTransportTLSVolumeMount(llmWorker, "nvcf-trust-merged-certs") + require.NotNil(t, mount) + assert.Equal(t, "/nvcf/transport-tls", mount.MountPath) for _, name := range []string{"inference", "smb-server"} { container := findTransportTLSContainer(createdPod, name) diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvca_config_mapper.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvca_config_mapper.go index 55d506714..a4cc4e0ef 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvca_config_mapper.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvca_config_mapper.go @@ -46,8 +46,9 @@ type nvcaOperatorWorkloadConfigDTO struct { } type nvcaOperatorTransportTLSConfigDTO struct { - TrustBundle *nvcaOperatorTrustBundleConfigDTO `yaml:"trustBundle"` - Fingerprint string `yaml:"fingerprint"` + TrustBundle *nvcaOperatorTrustBundleConfigDTO `yaml:"trustBundle"` + Fingerprint string `yaml:"fingerprint"` + InstalledBundleMountPath string `yaml:"installedBundleMountPath"` } type nvcaOperatorTrustBundleConfigDTO struct { @@ -164,11 +165,16 @@ func withSecretBackedTransportTLSMapper(namespace string, secrets corev1client.S return false, fmt.Errorf("transport TLS fingerprint %q does not match computed fingerprint %q", supplied, fingerprint) } - destination.Workload.TransportTLS = &nvcaconfig.TransportTLSConfig{ - TrustMode: nvcaconfig.TrustModeBundle, - TrustBundleFingerprint: fingerprint, - TrustBundlePEM: string(pemData), + transportTLSConfig := nvcaconfig.TransportTLSConfig{ + TrustMode: nvcaconfig.TrustModeBundle, + TrustBundleFingerprint: fingerprint, + TrustBundlePEM: string(pemData), + InstalledBundleMountPath: source.Workload.TransportTLS.InstalledBundleMountPath, } + if err := transporttls.ValidateConfig(transporttls.NormalizeConfig(transportTLSConfig)); err != nil { + return false, fmt.Errorf("validate transport TLS configuration: %w", err) + } + destination.Workload.TransportTLS = &transportTLSConfig return true, nil } } diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvca_config_mapper_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvca_config_mapper_test.go index 6e067ec8b..bd756fff8 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvca_config_mapper_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvca_config_mapper_test.go @@ -62,6 +62,39 @@ func TestNVCAConfigMapper_MapsSecretBackedTransportTLSToAgentConfig(t *testing.T config.Workload.TransportTLS.TrustBundleFingerprint) } +func TestNVCAConfigMapper_MapsInstalledBundleMountPathToAgentConfig(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClientsForIntegrationTests() + _, err := clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: nvcaOperatorConfigMapName}, + Data: map[string]string{agentConfigFile: `workload: + transportTLS: + trustBundle: + secretKeyRef: + name: nvcf-trust + key: ca.crt + installedBundleMountPath: /nvcf/transport-tls +`}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + _, err = clients.K8s.CoreV1().Secrets(NVCAOperatorNamespace).Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "nvcf-trust"}, + Data: map[string][]byte{"ca.crt": []byte(transportTrustTestPEM)}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + + config, found, err := newNVCAOperatorConfigMapMapper( + NVCAOperatorNamespace, + clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace), + clients.K8s.CoreV1().Secrets(NVCAOperatorNamespace), + ).getConfig(ctx) + + require.NoError(t, err) + require.True(t, found) + require.NotNil(t, config.Workload.TransportTLS) + assert.Equal(t, "/nvcf/transport-tls", config.Workload.TransportTLS.InstalledBundleMountPath) +} + func TestNVCAConfigMapper_MissingConfigIsNoop(t *testing.T) { ctx := newTestContext() clients := mockKubeClientsForIntegrationTests() @@ -233,3 +266,18 @@ func createTransportTrustSource(t *testing.T, ctx context.Context, clients *kube }, metav1.CreateOptions{}) require.NoError(t, err) } + +func setTransportTrustInstalledBundleMountPath( + t *testing.T, + ctx context.Context, + clients *kubeclients.KubeClients, + mountPath string, +) { + t.Helper() + configMap, err := clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Get(ctx, nvcaOperatorConfigMapName, + metav1.GetOptions{}) + require.NoError(t, err) + configMap.Data[agentConfigFile] += " installedBundleMountPath: " + mountPath + "\n" + _, err = clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Update(ctx, configMap, metav1.UpdateOptions{}) + require.NoError(t, err) +} diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/transport_tls_config_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/transport_tls_config_test.go index b30f439c8..63c0c833c 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/transport_tls_config_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/transport_tls_config_test.go @@ -84,6 +84,7 @@ func TestSecretBackedTransportTLS_UsesIdenticalEncodingAcrossClusterModes(t *tes operatorNamespace: NVCAOperatorNamespace, } createTransportTrustSource(t, ctx, clients) + setTransportTrustInstalledBundleMountPath(t, ctx, clients, "/nvcf/transport-tls") nb := ngcManagedBackendWithAgentConfig(nvidiaiov1.AgentConfig{}) nb.Spec.ClusterSource = clusterSource @@ -103,6 +104,7 @@ func TestSecretBackedTransportTLS_UsesIdenticalEncodingAcrossClusterModes(t *tes assert.Equal(t, nvcaconfig.TrustModeBundle, decodedConfig.Workload.TransportTLS.TrustMode) assert.Equal(t, transportTrustTestPEM, decodedConfig.Workload.TransportTLS.TrustBundlePEM) assert.Equal(t, "registry.example.test/nvca:2.52.0", decodedConfig.Workload.TransportTLS.InstallerImage) + assert.Equal(t, "/nvcf/transport-tls", decodedConfig.Workload.TransportTLS.InstalledBundleMountPath) checker, err := bc.newAgentConfigChangedCheck(ctx, nb) require.NoError(t, err) @@ -111,6 +113,36 @@ func TestSecretBackedTransportTLS_UsesIdenticalEncodingAcrossClusterModes(t *tes } } +func TestSecretBackedTransportTLS_InstalledBundleMountPathChangeTriggersAgentRollout(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClientsForIntegrationTests() + bc := &BackendK8sCache{ + clients: clients, + envType: nvidiaiov1.EnvTypeStage, + operatorNamespace: NVCAOperatorNamespace, + } + createTransportTrustSource(t, ctx, clients) + nb := ngcManagedBackendWithAgentConfig(nvidiaiov1.AgentConfig{}) + nb.Spec.NVCAImageConfig = nvidiaiov1.ImageConfig{Repository: "registry.example.test/nvca", Tag: "2.52.0"} + agentCfg, err := bc.newAgentConfig(ctx, nb) + require.NoError(t, err) + require.NoError(t, bc.setupAgentConfigConfigMap(ctx, nb, agentCfg)) + + setTransportTrustInstalledBundleMountPath(t, ctx, clients, "/nvcf/transport-tls") + checker, err := bc.newAgentConfigChangedCheck(ctx, nb) + require.NoError(t, err) + assert.True(t, checker()) + require.NoError(t, bc.setupAgentConfigConfigMap(ctx, nb, agentCfg)) + + storedConfig, err := clients.K8s.CoreV1().ConfigMaps(DefaultNVCASystemNamespace).Get(ctx, agentConfigConfigMapName, + metav1.GetOptions{}) + require.NoError(t, err) + decodedConfig, err := nvcaconfig.DecodeConfig([]byte(storedConfig.Data[agentConfigFile])) + require.NoError(t, err) + require.NotNil(t, decodedConfig.Workload.TransportTLS) + assert.Equal(t, "/nvcf/transport-tls", decodedConfig.Workload.TransportTLS.InstalledBundleMountPath) +} + func TestGetAgentConfigToMerge_RejectsTransportTLSSourceConflict(t *testing.T) { ctx := newTestContext() clients := mockKubeClientsForIntegrationTests() @@ -212,6 +244,49 @@ func TestSetupAgentConfigConfigMap_SecretTrustFailurePreservesLastGoodConfig(t * assert.Equal(t, lastGoodData, storedConfig.Data[agentConfigFile]) } +func TestSetupAgentConfigConfigMap_InvalidInstalledBundleMountPathPreservesLastGoodConfig(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClientsForIntegrationTests() + bc := &BackendK8sCache{ + clients: clients, + envType: nvidiaiov1.EnvTypeStage, + operatorNamespace: NVCAOperatorNamespace, + } + createTransportTrustSource(t, ctx, clients) + nb := ngcManagedBackendWithAgentConfig(nvidiaiov1.AgentConfig{}) + agentCfg, err := bc.newAgentConfig(ctx, nb) + require.NoError(t, err) + require.NoError(t, bc.setupAgentConfigConfigMap(ctx, nb, agentCfg)) + + lastGoodConfig, err := clients.K8s.CoreV1().ConfigMaps(DefaultNVCASystemNamespace).Get(ctx, agentConfigConfigMapName, + metav1.GetOptions{}) + require.NoError(t, err) + lastGoodData := lastGoodConfig.Data[agentConfigFile] + + operatorConfig, err := clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Get(ctx, nvcaOperatorConfigMapName, + metav1.GetOptions{}) + require.NoError(t, err) + operatorConfig.Data[agentConfigFile] = `workload: + transportTLS: + trustBundle: + secretKeyRef: + name: nvcf-trust + key: ca.crt + installedBundleMountPath: nvcf/transport-tls +` + _, err = clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Update(ctx, operatorConfig, metav1.UpdateOptions{}) + require.NoError(t, err) + + err = bc.setupAgentConfigConfigMap(ctx, nb, agentCfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "installedBundleMountPath must be absolute") + + storedConfig, err := clients.K8s.CoreV1().ConfigMaps(DefaultNVCASystemNamespace).Get(ctx, agentConfigConfigMapName, + metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, lastGoodData, storedConfig.Data[agentConfigFile]) +} + func TestConfigMapAddHandler_ReconcilesForOperatorConfig(t *testing.T) { ctx := newTestContext() bc, backend := newConfigMapEventTestCache(t, ctx) diff --git a/src/compute-plane-services/nvca/scripts/lint_helm.sh b/src/compute-plane-services/nvca/scripts/lint_helm.sh index f44d44819..aa4cfb26a 100755 --- a/src/compute-plane-services/nvca/scripts/lint_helm.sh +++ b/src/compute-plane-services/nvca/scripts/lint_helm.sh @@ -188,7 +188,8 @@ assert_transport_trust_config() { local label="${1}" local expected_name="${2}" local expected_fingerprint="${3}" - shift 3 + local expected_mount_path="${4}" + shift 4 local render_output render_output="$(mktemp)" @@ -206,16 +207,30 @@ assert_transport_trust_config() { rm -f "${render_output}" exit 1 fi + if [[ -z "${expected_mount_path}" ]]; then + if grep -Fq "installedBundleMountPath:" "${render_output}"; then + echo "Expected ${label} workload transport trust configuration to omit installedBundleMountPath" + cat "${render_output}" + rm -f "${render_output}" + exit 1 + fi + elif ! grep -Fq "installedBundleMountPath: \"${expected_mount_path}\"" "${render_output}"; then + echo "Expected ${label} workload transport trust configuration to set installedBundleMountPath" + cat "${render_output}" + rm -f "${render_output}" + exit 1 + fi rm -f "${render_output}" echo "✓ ${label} workload transport trust ConfigMap renders" } -assert_transport_trust_config "default" "" "" --set "ngcConfig.serviceKey=fakekey" -assert_transport_trust_config "configured" "nvcf-trust" "sha256:example" \ +assert_transport_trust_config "default" "" "" "" --set "ngcConfig.serviceKey=fakekey" +assert_transport_trust_config "configured" "nvcf-trust" "sha256:example" "/nvcf/transport-tls" \ --set "ngcConfig.serviceKey=fakekey" \ --set "operatorConfig.workload.transportTLS.trustBundle.secretKeyRef.name=nvcf-trust" \ - --set "operatorConfig.workload.transportTLS.fingerprint=sha256:example" -assert_transport_trust_config "reuse-values upgrade simulation" "" "" \ + --set "operatorConfig.workload.transportTLS.fingerprint=sha256:example" \ + --set "operatorConfig.workload.transportTLS.installedBundleMountPath=/nvcf/transport-tls" +assert_transport_trust_config "reuse-values upgrade simulation" "" "" "" \ --set "ngcConfig.serviceKey=fakekey" \ --values "${reuse_values_file}" \ --set "operatorConfig=null" diff --git a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config/types.go b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config/types.go index d3dd13085..38fb779b4 100644 --- a/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config/types.go +++ b/src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config/types.go @@ -846,6 +846,7 @@ type TransportTLSConfig struct { TrustBundleFingerprint string `yaml:"trustBundleFingerprint"` TrustBundlePEM string `yaml:"trustBundlePem"` InstallerImage string `yaml:"installerImage"` + InstalledBundleMountPath string `yaml:"installedBundleMountPath"` } func (t WorkloadConfig) Complete() WorkloadConfig { diff --git a/src/libraries/go/lib/pkg/types/nvca/config/types.go b/src/libraries/go/lib/pkg/types/nvca/config/types.go index d3dd13085..38fb779b4 100644 --- a/src/libraries/go/lib/pkg/types/nvca/config/types.go +++ b/src/libraries/go/lib/pkg/types/nvca/config/types.go @@ -846,6 +846,7 @@ type TransportTLSConfig struct { TrustBundleFingerprint string `yaml:"trustBundleFingerprint"` TrustBundlePEM string `yaml:"trustBundlePem"` InstallerImage string `yaml:"installerImage"` + InstalledBundleMountPath string `yaml:"installedBundleMountPath"` } func (t WorkloadConfig) Complete() WorkloadConfig {