From 8acd581960dec0aa1f13f3b46da47a0a1cf13bfa Mon Sep 17 00:00:00 2001 From: Tim Date: Wed, 8 Jul 2026 15:53:31 -0400 Subject: [PATCH] feat(traefik): make workload ingress highly available and Burstable Workload Traefik ran a single BestEffort replica, making it a SPOF and the first pod evicted under node pressure - a saturated node could take a workload's entire edge offline. Harden both the EKS (helm_aws) and AKS (clusters_azure) paths: - deployment.replicas defaults to 3, configurable per workload via traefik_deployment_replicas (mirrors the control-room knob) - resource requests/limits move Traefik into Burstable QoS - topologySpreadConstraints spread replicas across hosts (soft) - podDisruptionBudget keeps >=2 replicas up during drains - dedicated cluster-scoped traefik-critical PriorityClass Traefik is stateless here (certs come from cert-manager secrets, no local ACME), so multiple replicas need no coordination. --- lib/steps/clusters_azure.go | 50 +++++++++++++++- lib/steps/clusters_test.go | 86 +++++++++++++++++++++++++++ lib/steps/helm_aws.go | 51 ++++++++++++++-- lib/steps/helm_aws_test.go | 112 ++++++++++++++++++++++++++++++++++++ lib/types/workload.go | 8 +++ lib/types/workload_test.go | 22 +++++++ 6 files changed, 324 insertions(+), 5 deletions(-) create mode 100644 lib/steps/helm_aws_test.go diff --git a/lib/steps/clusters_azure.go b/lib/steps/clusters_azure.go index aacff8d..f1e92a8 100644 --- a/lib/steps/clusters_azure.go +++ b/lib/steps/clusters_azure.go @@ -22,6 +22,7 @@ import ( helmv3 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3" metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" rbacv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/rbac/v1" + schedulingv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/scheduling/v1" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -625,12 +626,40 @@ func azureClustersDeploy(ctx *pulumi.Context, _ types.Target, params azureCluste } // Traefik Helm release values — mirrors azure_traefik.py _define_helm_release + traefikReplicas := clusterCfg.Components.ResolveAzureComponents().TraefikDeploymentReplicas traefikValues := pulumi.Map{ "logs": pulumi.Map{ "general": pulumi.Map{ "level": pulumi.String("DEBUG"), }, }, + // HA hardening: run multiple replicas spread across nodes with resource + // requests (Burstable QoS) and a PDB so no single node failure/saturation + // takes down the workload edge. The resource requests mirror the + // control-room Traefik; the PDB, topology-spread, and priority class are + // new HA hardening beyond it. + "deployment": pulumi.Map{ + "replicas": pulumi.Int(traefikReplicas), + }, + "resources": pulumi.Map{ + "requests": pulumi.Map{"cpu": pulumi.String("200m"), "memory": pulumi.String("256Mi")}, + "limits": pulumi.Map{"cpu": pulumi.String("1000m"), "memory": pulumi.String("512Mi")}, + }, + "topologySpreadConstraints": pulumi.Array{ + pulumi.Map{ + "maxSkew": pulumi.Int(1), + "topologyKey": pulumi.String("kubernetes.io/hostname"), + "whenUnsatisfiable": pulumi.String("ScheduleAnyway"), + "labelSelector": pulumi.Map{ + "matchLabels": pulumi.Map{"app.kubernetes.io/name": pulumi.String("traefik")}, + }, + }, + }, + "podDisruptionBudget": pulumi.Map{ + "enabled": pulumi.Bool(true), + "maxUnavailable": pulumi.Int(1), + }, + "priorityClassName": pulumi.String("traefik-critical"), "ports": pulumi.Map{ "web": pulumi.Map{ "redirections": pulumi.Map{ @@ -752,6 +781,25 @@ func azureClustersDeploy(ctx *pulumi.Context, _ types.Target, params azureCluste } traefikValues["extraObjects"] = extraObjects + // Dedicated cluster-scoped PriorityClass so Traefik ingress pods resist + // eviction under node pressure. Value sits below the reserved system-* + // range (system-cluster-critical = 2000000000) but outranks ordinary + // workload pods. system-cluster-critical cannot be reused: admission + // restricts it to the kube-system namespace. + traefikPC, err := schedulingv1.NewPriorityClass(ctx, fmt.Sprintf("%s-%s-traefik-critical-priority", name, release), &schedulingv1.PriorityClassArgs{ + Metadata: metav1.ObjectMetaArgs{ + Name: pulumi.String("traefik-critical"), + Labels: pulumi.StringMap{"posit.team/managed-by": pulumi.String("ptd.pulumi_resources.azure_workload_clusters")}, + }, + Value: pulumi.Int(1000000000), + GlobalDefault: pulumi.Bool(false), + Description: pulumi.StringPtr("High priority for workload Traefik ingress pods so they resist eviction under node pressure"), + PreemptionPolicy: pulumi.StringPtr("PreemptLowerPriority"), + }, k8sProviderOpt) + if err != nil { + return fmt.Errorf("clusters: failed to create traefik priority class for %s: %w", release, err) + } + _, err = helmv3.NewRelease(ctx, fmt.Sprintf("%s-%s-traefik", name, release), &helmv3.ReleaseArgs{ Name: pulumi.String("traefik"), Chart: pulumi.String("traefik"), @@ -762,7 +810,7 @@ func azureClustersDeploy(ctx *pulumi.Context, _ types.Target, params azureCluste }, Atomic: pulumi.Bool(true), Values: traefikValues, - }, k8sProviderOpt, withTraefikAlias()) + }, k8sProviderOpt, withTraefikAlias(), pulumi.DependsOn([]pulumi.Resource{traefikPC})) if err != nil { return fmt.Errorf("clusters: failed to create traefik helm release for %s: %w", release, err) } diff --git a/lib/steps/clusters_test.go b/lib/steps/clusters_test.go index 798dc94..d3fb4b5 100644 --- a/lib/steps/clusters_test.go +++ b/lib/steps/clusters_test.go @@ -359,6 +359,92 @@ func TestAzureClustersDeployOneRelease(t *testing.T) { assert.Contains(t, names, "myworkload-20250101-helm-controller-namespace") } +// findTraefikRelease returns the traefik helm Release mock args for the given +// resource name, or nil if not found. +func findTraefikRelease(resources []pulumi.MockResourceArgs, name string) *pulumi.MockResourceArgs { + for i := range resources { + if resources[i].TypeToken == "kubernetes:helm.sh/v3:Release" && resources[i].Name == name { + return &resources[i] + } + } + return nil +} + +func TestAzureClustersDeployTraefikHA(t *testing.T) { + mocks := &clustersMocks{} + + err := pulumi.RunErr(func(ctx *pulumi.Context) error { + params := minimalAzureClustersParams("myworkload", []string{"20250101"}) + return azureClustersDeploy(ctx, nil, params) + }, pulumi.WithMocks("ptd-azure-workload-clusters", "myworkload", mocks)) + require.NoError(t, err) + + rel := findTraefikRelease(mocks.resources, "myworkload-20250101-traefik") + require.NotNil(t, rel, "traefik helm release not found") + + values := rel.Inputs["values"].ObjectValue() + + // deployment.replicas defaults to 3 (HA). + deployment := values["deployment"].ObjectValue() + assert.Equal(t, 3.0, deployment["replicas"].NumberValue()) + + // resources requests/limits present → Burstable QoS. + resources := values["resources"].ObjectValue() + requests := resources["requests"].ObjectValue() + assert.Equal(t, "200m", requests["cpu"].StringValue()) + assert.Equal(t, "256Mi", requests["memory"].StringValue()) + assert.Contains(t, resources, resource.PropertyKey("limits")) + + // topologySpreadConstraints spread across hosts. + tsc := values["topologySpreadConstraints"].ArrayValue() + require.Len(t, tsc, 1) + assert.Equal(t, "kubernetes.io/hostname", tsc[0].ObjectValue()["topologyKey"].StringValue()) + + // podDisruptionBudget enabled. + pdb := values["podDisruptionBudget"].ObjectValue() + assert.True(t, pdb["enabled"].BoolValue()) + assert.Equal(t, 1.0, pdb["maxUnavailable"].NumberValue()) + + // priorityClassName set. + assert.Equal(t, "traefik-critical", values["priorityClassName"].StringValue()) + + // Dedicated PriorityClass resource created (not an extraObject). + pc := findPriorityClass(mocks.resources) + require.NotNil(t, pc, "traefik-critical PriorityClass resource not found") + assert.Equal(t, "traefik-critical", pc.Inputs["metadata"].ObjectValue()["name"].StringValue()) + assert.Equal(t, 1000000000.0, pc.Inputs["value"].NumberValue()) +} + +// findPriorityClass returns the first PriorityClass mock args, or nil if none. +func findPriorityClass(resources []pulumi.MockResourceArgs) *pulumi.MockResourceArgs { + for i := range resources { + if resources[i].TypeToken == "kubernetes:scheduling.k8s.io/v1:PriorityClass" { + return &resources[i] + } + } + return nil +} + +func TestAzureClustersDeployTraefikReplicasOverride(t *testing.T) { + mocks := &clustersMocks{} + + err := pulumi.RunErr(func(ctx *pulumi.Context) error { + params := minimalAzureClustersParams("myworkload", []string{"20250101"}) + five := 5 + cfg := params.clusters["20250101"] + cfg.Components.TraefikDeploymentReplicas = &five + params.clusters["20250101"] = cfg + return azureClustersDeploy(ctx, nil, params) + }, pulumi.WithMocks("ptd-azure-workload-clusters", "myworkload", mocks)) + require.NoError(t, err) + + rel := findTraefikRelease(mocks.resources, "myworkload-20250101-traefik") + require.NotNil(t, rel, "traefik helm release not found") + + deployment := rel.Inputs["values"].ObjectValue()["deployment"].ObjectValue() + assert.Equal(t, 5.0, deployment["replicas"].NumberValue()) +} + func TestAzureClustersDeployTwoReleases(t *testing.T) { mocks := &clustersMocks{} diff --git a/lib/steps/helm_aws.go b/lib/steps/helm_aws.go index fbf8518..57573b4 100644 --- a/lib/steps/helm_aws.go +++ b/lib/steps/helm_aws.go @@ -279,7 +279,7 @@ func awsHelmDeploy(ctx *pulumi.Context, params awsHelmParams) error { } // 5. Traefik (namespace + helmchart + ingress) - if err := awsHelmTraefik(ctx, k8sOpt, name, release, params, routingWeight, resolved.TraefikVersion, withAlias); err != nil { + if err := awsHelmTraefik(ctx, k8sOpt, name, release, params, routingWeight, resolved.TraefikVersion, resolved.TraefikDeploymentReplicas, withAlias); err != nil { return err } @@ -532,7 +532,7 @@ func awsHelmSecretStoreCsiAws(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, } func awsHelmTraefik(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoundName, release string, - params awsHelmParams, weight, version string, + params awsHelmParams, weight, version string, replicas int, withAlias func(string, string) pulumi.ResourceOption) error { nsName := compoundName + "-" + release + "-traefik-ns" @@ -546,13 +546,56 @@ func awsHelmTraefik(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoundN return fmt.Errorf("helm: failed to create traefik namespace: %w", err) } + // Dedicated cluster-scoped PriorityClass so Traefik ingress pods are protected + // from eviction under node pressure. Value sits below the reserved system-* + // range (system-cluster-critical = 2000000000) but high enough to outrank + // ordinary workload pods. system-cluster-critical itself cannot be reused here: + // admission restricts it to the kube-system namespace. + pcResourceName := compoundName + "-" + release + "-traefik-critical-priority" + pc, err := schedulingv1.NewPriorityClass(ctx, pcResourceName, &schedulingv1.PriorityClassArgs{ + Metadata: metav1.ObjectMetaArgs{ + Name: pulumi.String("traefik-critical"), + Labels: pulumi.StringMap{"posit.team/managed-by": pulumi.String("ptd.pulumi_resources.aws_workload_helm")}, + }, + Value: pulumi.Int(1000000000), + GlobalDefault: pulumi.Bool(false), + Description: pulumi.StringPtr("High priority for workload Traefik ingress pods so they resist eviction under node pressure"), + PreemptionPolicy: pulumi.StringPtr("PreemptLowerPriority"), + }, k8sOpt, withAlias("kubernetes:scheduling.k8s.io/v1:PriorityClass", pcResourceName)) + if err != nil { + return fmt.Errorf("helm: failed to create traefik priority class: %w", err) + } + traefikValues := map[string]interface{}{ "image": map[string]interface{}{ "registry": "ghcr.io/traefik", }, "deployment": map[string]interface{}{ - "kind": "Deployment", + "kind": "Deployment", + "replicas": replicas, + }, + // Resource requests move Traefik out of BestEffort into Burstable QoS so it is + // not the first pod starved/evicted under node pressure. Mirrors control room. + "resources": map[string]interface{}{ + "requests": map[string]interface{}{"cpu": "200m", "memory": "256Mi"}, + "limits": map[string]interface{}{"cpu": "1000m", "memory": "512Mi"}, + }, + // Spread replicas across nodes so a single node failure cannot take out the edge. + "topologySpreadConstraints": []interface{}{ + map[string]interface{}{ + "maxSkew": 1, + "topologyKey": "kubernetes.io/hostname", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": map[string]interface{}{ + "matchLabels": map[string]interface{}{"app.kubernetes.io/name": "traefik"}, + }, + }, + }, + "podDisruptionBudget": map[string]interface{}{ + "enabled": true, + "maxUnavailable": 1, }, + "priorityClassName": "traefik-critical", "logs": map[string]interface{}{ "access": map[string]interface{}{"enabled": true}, "general": map[string]interface{}{"level": "DEBUG"}, @@ -613,7 +656,7 @@ func awsHelmTraefik(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoundN "spec": chartSpec, }, }, k8sOpt, withAlias("kubernetes:helm.cattle.io/v1:HelmChart", chartResourceName), - pulumi.DependsOn([]pulumi.Resource{ns})) + pulumi.DependsOn([]pulumi.Resource{ns, pc})) if err != nil { return fmt.Errorf("helm: failed to create traefik chart: %w", err) } diff --git a/lib/steps/helm_aws_test.go b/lib/steps/helm_aws_test.go new file mode 100644 index 0000000..7213882 --- /dev/null +++ b/lib/steps/helm_aws_test.go @@ -0,0 +1,112 @@ +package steps + +import ( + "sync" + "testing" + + "github.com/posit-dev/ptd/lib/types" + "github.com/pulumi/pulumi/sdk/v3/go/common/resource" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + yamlv2 "gopkg.in/yaml.v2" +) + +// helmAWSMocks records created resources for inspection. +type helmAWSMocks struct { + mu sync.Mutex + resources []pulumi.MockResourceArgs +} + +func (m *helmAWSMocks) NewResource(args pulumi.MockResourceArgs) (string, resource.PropertyMap, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.resources = append(m.resources, args) + return args.Name + "_id", args.Inputs, nil +} + +func (m *helmAWSMocks) Call(args pulumi.MockCallArgs) (resource.PropertyMap, error) { + return resource.PropertyMap{}, nil +} + +func (m *helmAWSMocks) find(name string) *pulumi.MockResourceArgs { + m.mu.Lock() + defer m.mu.Unlock() + for i := range m.resources { + if m.resources[i].Name == name { + return &m.resources[i] + } + } + return nil +} + +// runAwsHelmTraefik invokes awsHelmTraefik in isolation with no-op providers/aliases. +func runAwsHelmTraefik(t *testing.T, replicas int) *helmAWSMocks { + t.Helper() + mocks := &helmAWSMocks{} + noopOpt := pulumi.Aliases(nil) + withAlias := func(string, string) pulumi.ResourceOption { return pulumi.Aliases(nil) } + err := pulumi.RunErr(func(ctx *pulumi.Context) error { + params := awsHelmParams{ + compoundName: "wl01-staging", + trueName: "wl01", + environment: "staging", + cfg: types.AWSWorkloadConfig{}, + } + return awsHelmTraefik(ctx, noopOpt, "wl01-staging", "20250101", params, "100", "37.1.2", replicas, withAlias) + }, pulumi.WithMocks("ptd-aws-workload-helm", "wl01-staging", mocks)) + require.NoError(t, err) + return mocks +} + +func TestAwsHelmTraefikHA(t *testing.T) { + mocks := runAwsHelmTraefik(t, 3) + + // A dedicated traefik-critical PriorityClass is created. + pc := mocks.find("wl01-staging-20250101-traefik-critical-priority") + require.NotNil(t, pc, "traefik-critical PriorityClass not created") + assert.Equal(t, "traefik-critical", pc.Inputs["metadata"].ObjectValue()["name"].StringValue()) + assert.Equal(t, 1000000000.0, pc.Inputs["value"].NumberValue()) + + // The HelmChart CR carries the HA values in its valuesContent YAML. + chart := mocks.find("wl01-staging-20250101-traefik-helm-release") + require.NotNil(t, chart, "traefik HelmChart CR not created") + valuesContent := chart.Inputs["spec"].ObjectValue()["valuesContent"].StringValue() + + var values map[string]interface{} + require.NoError(t, yamlv2.Unmarshal([]byte(valuesContent), &values)) + + deployment := values["deployment"].(map[interface{}]interface{}) + assert.Equal(t, 3, deployment["replicas"]) + assert.Equal(t, "Deployment", deployment["kind"]) + + res := values["resources"].(map[interface{}]interface{}) + requests := res["requests"].(map[interface{}]interface{}) + assert.Equal(t, "200m", requests["cpu"]) + assert.Equal(t, "256Mi", requests["memory"]) + assert.Contains(t, res, "limits") + + tsc := values["topologySpreadConstraints"].([]interface{}) + require.Len(t, tsc, 1) + assert.Equal(t, "kubernetes.io/hostname", tsc[0].(map[interface{}]interface{})["topologyKey"]) + + pdb := values["podDisruptionBudget"].(map[interface{}]interface{}) + assert.Equal(t, true, pdb["enabled"]) + assert.Equal(t, 1, pdb["maxUnavailable"]) + + assert.Equal(t, "traefik-critical", values["priorityClassName"]) +} + +func TestAwsHelmTraefikReplicasOverride(t *testing.T) { + mocks := runAwsHelmTraefik(t, 5) + + chart := mocks.find("wl01-staging-20250101-traefik-helm-release") + require.NotNil(t, chart) + valuesContent := chart.Inputs["spec"].ObjectValue()["valuesContent"].StringValue() + + var values map[string]interface{} + require.NoError(t, yamlv2.Unmarshal([]byte(valuesContent), &values)) + + deployment := values["deployment"].(map[interface{}]interface{}) + assert.Equal(t, 5, deployment["replicas"]) +} diff --git a/lib/types/workload.go b/lib/types/workload.go index 2a29c10..d6074ac 100644 --- a/lib/types/workload.go +++ b/lib/types/workload.go @@ -195,6 +195,7 @@ type AWSWorkloadClusterComponents struct { // deployed in the clusters step (lib/steps/clusters_aws.go). TraefikForwardAuthVersion *string `json:"traefik_forward_auth_version" yaml:"traefik_forward_auth_version"` TraefikVersion *string `json:"traefik_version" yaml:"traefik_version"` + TraefikDeploymentReplicas *int `json:"traefik_deployment_replicas" yaml:"traefik_deployment_replicas"` // TigeraOperatorVersion pins the Calico/Tigera operator chart version. Consumed by // the eks step (Calico CNI). Mirrors Python WorkloadClusterComponentConfig.tigera_operator_version // (default "3.31.4"). Resolve via TigeraOperatorVersionOrDefault. @@ -227,6 +228,7 @@ type ResolvedAWSComponents struct { SecretStoreCsiDriverVersion string SecretStoreCsiDriverAwsProviderVersion string TraefikVersion string + TraefikDeploymentReplicas int } func resolveString(ptr *string, def string) string { @@ -262,6 +264,7 @@ func (c *AWSWorkloadClusterComponents) ResolveAWSComponents() ResolvedAWSCompone SecretStoreCsiDriverVersion: resolveString(c.SecretStoreCsiDriverVersion, "1.3.4"), SecretStoreCsiDriverAwsProviderVersion: resolveString(c.SecretStoreCsiDriverAwsProviderVersion, "0.3.5"), TraefikVersion: resolveString(c.TraefikVersion, "37.1.2"), + TraefikDeploymentReplicas: resolveInt(c.TraefikDeploymentReplicas, 3), } } @@ -586,6 +589,9 @@ type AzureWorkloadClusterComponentConfig struct { LokiVersion *string `yaml:"loki_version"` MimirVersion *string `yaml:"mimir_version"` NvidiaDevicePluginVersion *string `yaml:"nvidia_device_plugin_version"` + // TraefikDeploymentReplicas sets the number of workload Traefik ingress replicas. + // Defaults to 3 for high availability (resolve via ResolveAzureComponents). + TraefikDeploymentReplicas *int `yaml:"traefik_deployment_replicas"` } // ResolvedAzureComponents is the result of resolving AzureWorkloadClusterComponentConfig with defaults applied. @@ -597,6 +603,7 @@ type ResolvedAzureComponents struct { LokiVersion string MimirVersion string NvidiaDevicePluginVersion string + TraefikDeploymentReplicas int } // ResolveAzureComponents returns the component versions with defaults applied. @@ -609,6 +616,7 @@ func (c *AzureWorkloadClusterComponentConfig) ResolveAzureComponents() ResolvedA LokiVersion: resolveString(c.LokiVersion, "5.42.0"), MimirVersion: resolveString(c.MimirVersion, "5.2.1"), NvidiaDevicePluginVersion: resolveString(c.NvidiaDevicePluginVersion, "0.17.1"), + TraefikDeploymentReplicas: resolveInt(c.TraefikDeploymentReplicas, 3), } } diff --git a/lib/types/workload_test.go b/lib/types/workload_test.go index 97be5d0..6fbd231 100644 --- a/lib/types/workload_test.go +++ b/lib/types/workload_test.go @@ -376,6 +376,28 @@ func boolPtr(b bool) *bool { return &b } +func TestResolveAWSComponentsTraefikDeploymentReplicas(t *testing.T) { + // Default: nil components resolve to 3 replicas (HA). + var components AWSWorkloadClusterComponents + assert.Equal(t, 3, components.ResolveAWSComponents().TraefikDeploymentReplicas) + + // Override: explicit value is honored. + five := 5 + components.TraefikDeploymentReplicas = &five + assert.Equal(t, 5, components.ResolveAWSComponents().TraefikDeploymentReplicas) +} + +func TestResolveAzureComponentsTraefikDeploymentReplicas(t *testing.T) { + // Default: unset resolves to 3 replicas (HA). + var components AzureWorkloadClusterComponentConfig + assert.Equal(t, 3, components.ResolveAzureComponents().TraefikDeploymentReplicas) + + // Override: explicit value is honored. + two := 2 + components.TraefikDeploymentReplicas = &two + assert.Equal(t, 2, components.ResolveAzureComponents().TraefikDeploymentReplicas) +} + func TestUsesEksAccessEntries(t *testing.T) { cases := []struct { name string