feat(traefik): make workload ingress highly available and Burstable

lib/steps/clusters_azure.go

@@ -22,6 +22,7 @@ import (
 	helmv3 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3"
 	metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
 	rbacv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/rbac/v1"
+	schedulingv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/scheduling/v1"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 )
 
@@ -625,12 +626,40 @@ func azureClustersDeploy(ctx *pulumi.Context, _ types.Target, params azureCluste
 		}
 
 		// Traefik Helm release values — mirrors azure_traefik.py _define_helm_release
+		traefikReplicas := clusterCfg.Components.ResolveAzureComponents().TraefikDeploymentReplicas
 		traefikValues := pulumi.Map{
 			"logs": pulumi.Map{
 				"general": pulumi.Map{
 					"level": pulumi.String("DEBUG"),
 				},
 			},
+			// HA hardening: run multiple replicas spread across nodes with resource
+			// requests (Burstable QoS) and a PDB so no single node failure/saturation
+			// takes down the workload edge. The resource requests mirror the
+			// control-room Traefik; the PDB, topology-spread, and priority class are
+			// new HA hardening beyond it.
+			"deployment": pulumi.Map{
+				"replicas": pulumi.Int(traefikReplicas),
+			},
+			"resources": pulumi.Map{
+				"requests": pulumi.Map{"cpu": pulumi.String("200m"), "memory": pulumi.String("256Mi")},
+				"limits":   pulumi.Map{"cpu": pulumi.String("1000m"), "memory": pulumi.String("512Mi")},
+			},
+			"topologySpreadConstraints": pulumi.Array{
+				pulumi.Map{
+					"maxSkew":           pulumi.Int(1),
+					"topologyKey":       pulumi.String("kubernetes.io/hostname"),
+					"whenUnsatisfiable": pulumi.String("ScheduleAnyway"),
+					"labelSelector": pulumi.Map{
+						"matchLabels": pulumi.Map{"app.kubernetes.io/name": pulumi.String("traefik")},
+					},
+				},
+			},
+			"podDisruptionBudget": pulumi.Map{
+				"enabled":        pulumi.Bool(true),
+				"maxUnavailable": pulumi.Int(1),
+			},
+			"priorityClassName": pulumi.String("traefik-critical"),
 			"ports": pulumi.Map{
 				"web": pulumi.Map{
 					"redirections": pulumi.Map{
@@ -752,6 +781,25 @@ func azureClustersDeploy(ctx *pulumi.Context, _ types.Target, params azureCluste
 		}
 		traefikValues["extraObjects"] = extraObjects
 
+		// Dedicated cluster-scoped PriorityClass so Traefik ingress pods resist
+		// eviction under node pressure. Value sits below the reserved system-*
+		// range (system-cluster-critical = 2000000000) but outranks ordinary
+		// workload pods. system-cluster-critical cannot be reused: admission
+		// restricts it to the kube-system namespace.
+		traefikPC, err := schedulingv1.NewPriorityClass(ctx, fmt.Sprintf("%s-%s-traefik-critical-priority", name, release), &schedulingv1.PriorityClassArgs{
+			Metadata: metav1.ObjectMetaArgs{
+				Name:   pulumi.String("traefik-critical"),
+				Labels: pulumi.StringMap{"posit.team/managed-by": pulumi.String("ptd.pulumi_resources.azure_workload_clusters")},
+			},
+			Value:            pulumi.Int(1000000000),
+			GlobalDefault:    pulumi.Bool(false),
+			Description:      pulumi.StringPtr("High priority for workload Traefik ingress pods so they resist eviction under node pressure"),
+			PreemptionPolicy: pulumi.StringPtr("PreemptLowerPriority"),
+		}, k8sProviderOpt)
+		if err != nil {
+			return fmt.Errorf("clusters: failed to create traefik priority class for %s: %w", release, err)
+		}
+
 		_, err = helmv3.NewRelease(ctx, fmt.Sprintf("%s-%s-traefik", name, release), &helmv3.ReleaseArgs{
 			Name:      pulumi.String("traefik"),
 			Chart:     pulumi.String("traefik"),
@@ -762,7 +810,7 @@ func azureClustersDeploy(ctx *pulumi.Context, _ types.Target, params azureCluste
 			},
 			Atomic: pulumi.Bool(true),
 			Values: traefikValues,
-		}, k8sProviderOpt, withTraefikAlias())
+		}, k8sProviderOpt, withTraefikAlias(), pulumi.DependsOn([]pulumi.Resource{traefikPC}))
 		if err != nil {
 			return fmt.Errorf("clusters: failed to create traefik helm release for %s: %w", release, err)
 		}

lib/steps/clusters_test.go

@@ -359,6 +359,92 @@ func TestAzureClustersDeployOneRelease(t *testing.T) {
 	assert.Contains(t, names, "myworkload-20250101-helm-controller-namespace")
 }
 
+// findTraefikRelease returns the traefik helm Release mock args for the given
+// resource name, or nil if not found.
+func findTraefikRelease(resources []pulumi.MockResourceArgs, name string) *pulumi.MockResourceArgs {
+	for i := range resources {
+		if resources[i].TypeToken == "kubernetes:helm.sh/v3:Release" && resources[i].Name == name {
+			return &resources[i]
+		}
+	}
+	return nil
+}
+
+func TestAzureClustersDeployTraefikHA(t *testing.T) {
+	mocks := &clustersMocks{}
+
+	err := pulumi.RunErr(func(ctx *pulumi.Context) error {
+		params := minimalAzureClustersParams("myworkload", []string{"20250101"})
+		return azureClustersDeploy(ctx, nil, params)
+	}, pulumi.WithMocks("ptd-azure-workload-clusters", "myworkload", mocks))
+	require.NoError(t, err)
+
+	rel := findTraefikRelease(mocks.resources, "myworkload-20250101-traefik")
+	require.NotNil(t, rel, "traefik helm release not found")
+
+	values := rel.Inputs["values"].ObjectValue()
+
+	// deployment.replicas defaults to 3 (HA).
+	deployment := values["deployment"].ObjectValue()
+	assert.Equal(t, 3.0, deployment["replicas"].NumberValue())
+
+	// resources requests/limits present → Burstable QoS.
+	resources := values["resources"].ObjectValue()
+	requests := resources["requests"].ObjectValue()
+	assert.Equal(t, "200m", requests["cpu"].StringValue())
+	assert.Equal(t, "256Mi", requests["memory"].StringValue())
+	assert.Contains(t, resources, resource.PropertyKey("limits"))
+
+	// topologySpreadConstraints spread across hosts.
+	tsc := values["topologySpreadConstraints"].ArrayValue()
+	require.Len(t, tsc, 1)
+	assert.Equal(t, "kubernetes.io/hostname", tsc[0].ObjectValue()["topologyKey"].StringValue())
+
+	// podDisruptionBudget enabled.
+	pdb := values["podDisruptionBudget"].ObjectValue()
+	assert.True(t, pdb["enabled"].BoolValue())
+	assert.Equal(t, 1.0, pdb["maxUnavailable"].NumberValue())
+
+	// priorityClassName set.
+	assert.Equal(t, "traefik-critical", values["priorityClassName"].StringValue())
+
+	// Dedicated PriorityClass resource created (not an extraObject).
+	pc := findPriorityClass(mocks.resources)
+	require.NotNil(t, pc, "traefik-critical PriorityClass resource not found")
+	assert.Equal(t, "traefik-critical", pc.Inputs["metadata"].ObjectValue()["name"].StringValue())
+	assert.Equal(t, 1000000000.0, pc.Inputs["value"].NumberValue())
+}
+
+// findPriorityClass returns the first PriorityClass mock args, or nil if none.
+func findPriorityClass(resources []pulumi.MockResourceArgs) *pulumi.MockResourceArgs {
+	for i := range resources {
+		if resources[i].TypeToken == "kubernetes:scheduling.k8s.io/v1:PriorityClass" {
+			return &resources[i]
+		}
+	}
+	return nil
+}
+
+func TestAzureClustersDeployTraefikReplicasOverride(t *testing.T) {
+	mocks := &clustersMocks{}
+
+	err := pulumi.RunErr(func(ctx *pulumi.Context) error {
+		params := minimalAzureClustersParams("myworkload", []string{"20250101"})
+		five := 5
+		cfg := params.clusters["20250101"]
+		cfg.Components.TraefikDeploymentReplicas = &five
+		params.clusters["20250101"] = cfg
+		return azureClustersDeploy(ctx, nil, params)
+	}, pulumi.WithMocks("ptd-azure-workload-clusters", "myworkload", mocks))
+	require.NoError(t, err)
+
+	rel := findTraefikRelease(mocks.resources, "myworkload-20250101-traefik")
+	require.NotNil(t, rel, "traefik helm release not found")
+
+	deployment := rel.Inputs["values"].ObjectValue()["deployment"].ObjectValue()
+	assert.Equal(t, 5.0, deployment["replicas"].NumberValue())
+}
+
 func TestAzureClustersDeployTwoReleases(t *testing.T) {
 	mocks := &clustersMocks{}
 

lib/steps/helm_aws.go

@@ -279,7 +279,7 @@ func awsHelmDeploy(ctx *pulumi.Context, params awsHelmParams) error {
 		}
 
 		// 5. Traefik (namespace + helmchart + ingress)
-		if err := awsHelmTraefik(ctx, k8sOpt, name, release, params, routingWeight, resolved.TraefikVersion, withAlias); err != nil {
+		if err := awsHelmTraefik(ctx, k8sOpt, name, release, params, routingWeight, resolved.TraefikVersion, resolved.TraefikDeploymentReplicas, withAlias); err != nil {
 			return err
 		}
 
@@ -532,7 +532,7 @@ func awsHelmSecretStoreCsiAws(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption,
 }
 
 func awsHelmTraefik(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoundName, release string,
-	params awsHelmParams, weight, version string,
+	params awsHelmParams, weight, version string, replicas int,
 	withAlias func(string, string) pulumi.ResourceOption) error {
 
 	nsName := compoundName + "-" + release + "-traefik-ns"
@@ -546,13 +546,56 @@ func awsHelmTraefik(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoundN
 		return fmt.Errorf("helm: failed to create traefik namespace: %w", err)
 	}
 
+	// Dedicated cluster-scoped PriorityClass so Traefik ingress pods are protected
+	// from eviction under node pressure. Value sits below the reserved system-*
+	// range (system-cluster-critical = 2000000000) but high enough to outrank
+	// ordinary workload pods. system-cluster-critical itself cannot be reused here:
+	// admission restricts it to the kube-system namespace.
+	pcResourceName := compoundName + "-" + release + "-traefik-critical-priority"
+	pc, err := schedulingv1.NewPriorityClass(ctx, pcResourceName, &schedulingv1.PriorityClassArgs{
+		Metadata: metav1.ObjectMetaArgs{
+			Name:   pulumi.String("traefik-critical"),
+			Labels: pulumi.StringMap{"posit.team/managed-by": pulumi.String("ptd.pulumi_resources.aws_workload_helm")},
+		},
+		Value:            pulumi.Int(1000000000),
+		GlobalDefault:    pulumi.Bool(false),
+		Description:      pulumi.StringPtr("High priority for workload Traefik ingress pods so they resist eviction under node pressure"),
+		PreemptionPolicy: pulumi.StringPtr("PreemptLowerPriority"),
+	}, k8sOpt, withAlias("kubernetes:scheduling.k8s.io/v1:PriorityClass", pcResourceName))
+	if err != nil {
+		return fmt.Errorf("helm: failed to create traefik priority class: %w", err)
+	}
+
 	traefikValues := map[string]interface{}{
 		"image": map[string]interface{}{
 			"registry": "ghcr.io/traefik",
 		},
 		"deployment": map[string]interface{}{
-			"kind": "Deployment",
+			"kind":     "Deployment",
+			"replicas": replicas,
+		},
+		// Resource requests move Traefik out of BestEffort into Burstable QoS so it is
+		// not the first pod starved/evicted under node pressure. Mirrors control room.
+		"resources": map[string]interface{}{
+			"requests": map[string]interface{}{"cpu": "200m", "memory": "256Mi"},
+			"limits":   map[string]interface{}{"cpu": "1000m", "memory": "512Mi"},
+		},
+		// Spread replicas across nodes so a single node failure cannot take out the edge.
+		"topologySpreadConstraints": []interface{}{
+			map[string]interface{}{
+				"maxSkew":           1,
+				"topologyKey":       "kubernetes.io/hostname",
+				"whenUnsatisfiable": "ScheduleAnyway",
+				"labelSelector": map[string]interface{}{
+					"matchLabels": map[string]interface{}{"app.kubernetes.io/name": "traefik"},
+				},
+			},
+		},
+		"podDisruptionBudget": map[string]interface{}{
+			"enabled":        true,
+			"maxUnavailable": 1,
 		},
+		"priorityClassName": "traefik-critical",
 		"logs": map[string]interface{}{
 			"access":  map[string]interface{}{"enabled": true},
 			"general": map[string]interface{}{"level": "DEBUG"},
@@ -613,7 +656,7 @@ func awsHelmTraefik(ctx *pulumi.Context, k8sOpt pulumi.ResourceOption, compoundN
 			"spec": chartSpec,
 		},
 	}, k8sOpt, withAlias("kubernetes:helm.cattle.io/v1:HelmChart", chartResourceName),
-		pulumi.DependsOn([]pulumi.Resource{ns}))
+		pulumi.DependsOn([]pulumi.Resource{ns, pc}))
 	if err != nil {
 		return fmt.Errorf("helm: failed to create traefik chart: %w", err)
 	}

lib/steps/helm_aws_test.go

@@ -0,0 +1,112 @@
+package steps
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/posit-dev/ptd/lib/types"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	yamlv2 "gopkg.in/yaml.v2"
+)
+
+// helmAWSMocks records created resources for inspection.
+type helmAWSMocks struct {
+	mu        sync.Mutex
+	resources []pulumi.MockResourceArgs
+}
+
+func (m *helmAWSMocks) NewResource(args pulumi.MockResourceArgs) (string, resource.PropertyMap, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.resources = append(m.resources, args)
+	return args.Name + "_id", args.Inputs, nil
+}
+
+func (m *helmAWSMocks) Call(args pulumi.MockCallArgs) (resource.PropertyMap, error) {
+	return resource.PropertyMap{}, nil
+}
+
+func (m *helmAWSMocks) find(name string) *pulumi.MockResourceArgs {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	for i := range m.resources {
+		if m.resources[i].Name == name {
+			return &m.resources[i]
+		}
+	}
+	return nil
+}
+
+// runAwsHelmTraefik invokes awsHelmTraefik in isolation with no-op providers/aliases.
+func runAwsHelmTraefik(t *testing.T, replicas int) *helmAWSMocks {
+	t.Helper()
+	mocks := &helmAWSMocks{}
+	noopOpt := pulumi.Aliases(nil)
+	withAlias := func(string, string) pulumi.ResourceOption { return pulumi.Aliases(nil) }
+	err := pulumi.RunErr(func(ctx *pulumi.Context) error {
+		params := awsHelmParams{
+			compoundName: "wl01-staging",
+			trueName:     "wl01",
+			environment:  "staging",
+			cfg:          types.AWSWorkloadConfig{},
+		}
+		return awsHelmTraefik(ctx, noopOpt, "wl01-staging", "20250101", params, "100", "37.1.2", replicas, withAlias)
+	}, pulumi.WithMocks("ptd-aws-workload-helm", "wl01-staging", mocks))
+	require.NoError(t, err)
+	return mocks
+}
+
+func TestAwsHelmTraefikHA(t *testing.T) {
+	mocks := runAwsHelmTraefik(t, 3)
+
+	// A dedicated traefik-critical PriorityClass is created.
+	pc := mocks.find("wl01-staging-20250101-traefik-critical-priority")
+	require.NotNil(t, pc, "traefik-critical PriorityClass not created")
+	assert.Equal(t, "traefik-critical", pc.Inputs["metadata"].ObjectValue()["name"].StringValue())
+	assert.Equal(t, 1000000000.0, pc.Inputs["value"].NumberValue())
+
+	// The HelmChart CR carries the HA values in its valuesContent YAML.
+	chart := mocks.find("wl01-staging-20250101-traefik-helm-release")
+	require.NotNil(t, chart, "traefik HelmChart CR not created")
+	valuesContent := chart.Inputs["spec"].ObjectValue()["valuesContent"].StringValue()
+
+	var values map[string]interface{}
+	require.NoError(t, yamlv2.Unmarshal([]byte(valuesContent), &values))
+
+	deployment := values["deployment"].(map[interface{}]interface{})
+	assert.Equal(t, 3, deployment["replicas"])
+	assert.Equal(t, "Deployment", deployment["kind"])
+
+	res := values["resources"].(map[interface{}]interface{})
+	requests := res["requests"].(map[interface{}]interface{})
+	assert.Equal(t, "200m", requests["cpu"])
+	assert.Equal(t, "256Mi", requests["memory"])
+	assert.Contains(t, res, "limits")
+
+	tsc := values["topologySpreadConstraints"].([]interface{})
+	require.Len(t, tsc, 1)
+	assert.Equal(t, "kubernetes.io/hostname", tsc[0].(map[interface{}]interface{})["topologyKey"])
+
+	pdb := values["podDisruptionBudget"].(map[interface{}]interface{})
+	assert.Equal(t, true, pdb["enabled"])
+	assert.Equal(t, 1, pdb["maxUnavailable"])
+
+	assert.Equal(t, "traefik-critical", values["priorityClassName"])
+}
+
+func TestAwsHelmTraefikReplicasOverride(t *testing.T) {
+	mocks := runAwsHelmTraefik(t, 5)
+
+	chart := mocks.find("wl01-staging-20250101-traefik-helm-release")
+	require.NotNil(t, chart)
+	valuesContent := chart.Inputs["spec"].ObjectValue()["valuesContent"].StringValue()
+
+	var values map[string]interface{}
+	require.NoError(t, yamlv2.Unmarshal([]byte(valuesContent), &values))
+
+	deployment := values["deployment"].(map[interface{}]interface{})
+	assert.Equal(t, 5, deployment["replicas"])
+}