diff --git a/applications/samples/api/openapi.yaml b/applications/samples/api/openapi.yaml index ed4adc913..96da7ed44 100644 --- a/applications/samples/api/openapi.yaml +++ b/applications/samples/api/openapi.yaml @@ -43,6 +43,42 @@ paths: operationId: ping summary: test the application is up x-openapi-router-controller: samples.controllers.test_controller + + /write-file: + post: + tags: + - test + requestBody: + description: Optional content of the file to write. + required: false + content: + application/json: + schema: + type: object + properties: + content: + type: string + responses: + "200": + content: + application/json: + schema: + type: object + properties: + filename: + type: string + path: + type: string + hostname: + type: string + description: The file was written on the application volume + operationId: write_file + summary: writes a file on the application volume + description: >- + Writes a timestamped file on the application volume and returns the name of the pod + that handled the request. On a statefulset deployment, route this endpoint to the + leader service to have all writes land on pod 0. + x-openapi-router-controller: samples.controllers.test_controller /valid: get: tags: diff --git a/applications/samples/backend/samples/controllers/test_controller.py b/applications/samples/backend/samples/controllers/test_controller.py index 256ef08f9..fad979e26 100644 --- a/applications/samples/backend/samples/controllers/test_controller.py +++ b/applications/samples/backend/samples/controllers/test_controller.py @@ -29,5 +29,43 @@ def ping(): # noqa: E501 return time.time() +def write_file(body=None): # noqa: E501 + """writes a file on the application volume + + # noqa: E501 + + :param body: Optional content of the file to write. + :type body: dict | bytes + + :rtype: dict + """ + import os + import socket + import time + + from flask import has_request_context, request + + if body is None and has_request_context() and request.is_json: + body = request.get_json() + + try: + from cloudharness.applications import get_current_configuration + mountpath = get_current_configuration().harness.deployment.volume.mountpath + except Exception: + # not running inside a cloudharness deployment (e.g. unit tests) + mountpath = "/tmp/myvolume" + + hostname = socket.gethostname() + filename = f"{time.strftime('%Y%m%d-%H%M%S')}-{hostname}.txt" + path = os.path.join(mountpath, filename) + content = (body or {}).get("content") or f"written by {hostname}" + + os.makedirs(mountpath, exist_ok=True) + with open(path, "w") as f: + f.write(content) + + return {"filename": filename, "path": path, "hostname": hostname} + + def serialization(): return User(last_name="Last", first_name="First") diff --git a/applications/samples/backend/samples/test/test_sample.py b/applications/samples/backend/samples/test/test_sample.py index 62acfabab..ff00fe69b 100644 --- a/applications/samples/backend/samples/test/test_sample.py +++ b/applications/samples/backend/samples/test/test_sample.py @@ -1,2 +1,20 @@ def test_sample(): assert True + + +def test_write_file(): + import socket + + from samples.controllers import test_controller + + # outside a deployment the controller falls back to writing under /tmp/myvolume + result = test_controller.write_file({"content": "hello"}) + + assert result["hostname"] == socket.gethostname() + assert result["hostname"] in result["filename"] + with open(result["path"]) as f: + assert f.read() == "hello" + + result = test_controller.write_file() + with open(result["path"]) as f: + assert socket.gethostname() in f.read() diff --git a/applications/samples/deploy/values-statefulset.yaml b/applications/samples/deploy/values-statefulset.yaml new file mode 100644 index 000000000..64376bb67 --- /dev/null +++ b/applications/samples/deploy/values-statefulset.yaml @@ -0,0 +1,4 @@ +harness: + deployment: + statefulset: true + replicas: 2 diff --git a/applications/samples/deploy/values.yaml b/applications/samples/deploy/values.yaml index a8d3b6bd0..7e290ad2c 100644 --- a/applications/samples/deploy/values.yaml +++ b/applications/samples/deploy/values.yaml @@ -12,6 +12,8 @@ harness: use_services: - name: common deployment: + replicas: 2 + statefulset: true volume: name: my-shared-volume mountpath: /tmp/myvolume @@ -48,6 +50,9 @@ harness: white-listed: true - uri: /api/ping white-listed: true + - uri: /api/write-file + methods: [POST] + white-listed: true - uri: /api/openapi.json white-listed: true - uri: /icon.png diff --git a/deployment-configuration/helm/templates/auto-database.yaml b/deployment-configuration/helm/templates/auto-database.yaml index 295964df3..15fb3c4e9 100644 --- a/deployment-configuration/helm/templates/auto-database.yaml +++ b/deployment-configuration/helm/templates/auto-database.yaml @@ -17,32 +17,55 @@ spec: {{- if and (eq .app.harness.database.type "postgres") (dig "postgres" "operator" false .app.harness.database) }} {{- include "deploy_utils.database.postgres.operator" . }} {{- else }} +{{- $isStatefulSet := .app.harness.database.statefulset | default false }} +{{- /* A statefulset database owns its volume through volumeClaimTemplates. A PVC named after the + database is the leftover of a previous Deployment: it is kept (never deleted by helm) and + its data is copied into the statefulset volume by a migration job (see + auto-volume-migration.yaml). Delete the legacy PVC once migrated. */}} +{{- $legacyClaim := false }} +{{- if $isStatefulSet }} +{{- if lookup "v1" "PersistentVolumeClaim" .root.Values.namespace .app.harness.database.name }} +{{- $legacyClaim = true }} +{{- end }} +{{- end }} +{{- if or (not $isStatefulSet) $legacyClaim }} --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: {{ .app.harness.database.name | quote }} namespace: {{ .root.Values.namespace }} + {{- if $legacyClaim }} + annotations: + helm.sh/resource-policy: keep + {{- end }} spec: accessModes: - ReadWriteOnce resources: requests: storage: {{ .app.harness.database.size }} +{{- end }} --- apiVersion: apps/v1 -kind: Deployment +kind: {{ ternary "StatefulSet" "Deployment" $isStatefulSet }} metadata: name: {{ .app.harness.database.name | quote }} namespace: {{ .root.Values.namespace }} labels: usesvolume: {{ .app.harness.database.name }}-{{ .root.Values.namespace }} spec: + {{- if $isStatefulSet }} + # StatefulSet updates terminate the old pod before creating its replacement, so no + # Recreate strategy or node pinning is needed. + serviceName: {{ .app.harness.database.name | quote }} + {{- else }} # The database's ReadWriteOnce volume attaches to a single node and the pod is # pinned to it via podAffinity. Recreate terminates the old pod before starting # the new one, avoiding an unschedulable pod when the node can't fit both. strategy: type: Recreate + {{- end }} replicas: 1 selector: matchLabels: @@ -54,6 +77,7 @@ spec: service: db usesvolume: {{ .app.harness.database.name }}-{{ .root.Values.namespace }} spec: + {{- if not $isStatefulSet }} affinity: podAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -65,6 +89,18 @@ spec: - {{ .app.harness.database.name }}-{{ .root.Values.namespace }} - {{ .app.harness.database.name }} # Obsolete, kept for backwards compatability, to be removed at a later date topologyKey: "kubernetes.io/hostname" + {{- end }} + {{- if $legacyClaim }} + initContainers: + # Holds the pod until the volume-migration job has copied the legacy volume data into + # the statefulset volume. Delete the legacy PVC after verifying the migration. + - name: volume-migration + image: busybox:1.36 + command: ['sh', '-c', 'until [ -f /migration/target/.cloudharness-volume-migrated ]; do echo "waiting for the volume migration job"; sleep 5; done'] + volumeMounts: + - name: {{ .app.harness.database.name | quote }} + mountPath: /migration/target + {{- end }} containers: - name: {{ .app.harness.database.name | quote }} imagePullPolicy: IfNotPresent @@ -82,15 +118,33 @@ spec: - mountPath: /dev/shm name: dshm {{- end }} + {{- if or (not $isStatefulSet) (eq .app.harness.database.type "postgres") }} volumes: + {{- if not $isStatefulSet }} - name: {{ .app.harness.database.name | quote }} persistentVolumeClaim: claimName: {{ .app.harness.database.name | quote }} + {{- end }} {{- if eq .app.harness.database.type "postgres" }} - emptyDir: medium: Memory - name: dshm - {{- end }} + name: dshm + {{- end }} + {{- end }} + {{- if $isStatefulSet }} + volumeClaimTemplates: + - metadata: + name: {{ .app.harness.database.name | quote }} + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .app.harness.database.size }} + {{- end }} + {{- if $legacyClaim }} + {{- include "deploy_utils.volumeMigration" (dict "root" .root "name" .app.harness.database.name "pvc" .app.harness.database.name) }} + {{- end }} --- {{- if and .root.Values.backup.active (ne .app.harness.database.type "neo4j") }} {{- include (print "deploy_utils.database." .app.harness.database.type ".backup") . }} diff --git a/deployment-configuration/helm/templates/auto-deployments.yaml b/deployment-configuration/helm/templates/auto-deployments.yaml index 77b97d3a4..a17ef3710 100644 --- a/deployment-configuration/helm/templates/auto-deployments.yaml +++ b/deployment-configuration/helm/templates/auto-deployments.yaml @@ -1,6 +1,34 @@ {{- define "deploy_utils.deployment" }} +{{- $isStatefulSet := .app.harness.deployment.statefulset | default false }} +{{- $volume := .app.harness.deployment.volume }} +{{- /* $rwoVolume: a ReadWriteOnce (non-nfs) volume pins the pod to a single node — it drives both + the Recreate strategy and the podAffinity. $ownVolume additionally requires the PVC to be + managed here (auto), which a statefulset turns into a volumeClaimTemplate. The checks are + nested so hasKey/index never runs on a nil volume: helm < 3.10 (go < 1.18) does not + short-circuit 'and'/'or', so a flat expression would fail for volume-less apps. */}} +{{- $rwoVolume := false }} +{{- $ownVolume := false }} +{{- if $volume }} +{{- if or (not (hasKey $volume "usenfs")) (not $volume.usenfs) }} +{{- $rwoVolume = true }} +{{- if or (not (hasKey $volume "auto")) $volume.auto }} +{{- $ownVolume = true }} +{{- end }} +{{- end }} +{{- end }} +{{- $useClaimTemplate := and $isStatefulSet $ownVolume }} +{{- /* A PVC named after the volume is the leftover of a previous Deployment: a migration job + (see auto-volume-migration.yaml) copies its data into each statefulset volume, and an init + container holds each pod until its volume has been migrated. Delete the legacy PVC once + migrated to end the migration. */}} +{{- $legacyClaim := false }} +{{- if $useClaimTemplate }} +{{- if lookup "v1" "PersistentVolumeClaim" .root.Values.namespace $volume.name }} +{{- $legacyClaim = true }} +{{- end }} +{{- end }} apiVersion: apps/v1 -kind: Deployment +kind: {{ ternary "StatefulSet" "Deployment" $isStatefulSet }} metadata: name: {{ .app.harness.deployment.name| quote }} namespace: {{ .root.Values.namespace }} @@ -11,18 +39,18 @@ metadata: {{- end }} {{- include "deploy_utils.labels" .root | indent 4 }} spec: - {{- if .app.harness.deployment.volume }} - {{- if or (not (hasKey .app.harness.deployment.volume "usenfs")) (not .app.harness.deployment.volume.usenfs) }} + {{- if $isStatefulSet }} + # StatefulSet updates terminate the old pod before creating its replacement, so no + # Recreate strategy or node pinning is needed even for ReadWriteOnce volumes. + serviceName: {{ .app.harness.service.name | quote }} + {{- else if $rwoVolume }} # A ReadWriteOnce volume attaches to a single node and the pod is pinned to the # volume's node via podAffinity (see below). Recreate terminates the old pod # before starting the new one, avoiding an unschedulable pod when the node can't # fit both. NFS (ReadWriteMany) volumes have no pinning and can roll normally. - # Nested ifs (not a flat 'and'): helm < 3.10 (go < 1.18) does not short-circuit, - # so hasKey must not run on a nil volume. strategy: type: Recreate {{- end }} - {{- end }} replicas: {{ .app.harness.deployment.replicas | default 1 }} selector: matchLabels: @@ -47,8 +75,7 @@ spec: - name: {{ .root.Values.registry.secret.name }} {{- end }} {{- end }} - {{- if and .app.harness.deployment.volume }} - {{- if or (not (hasKey .app.harness.deployment.volume "usenfs")) (not .app.harness.deployment.volume.usenfs) }} + {{- if and $rwoVolume (not $isStatefulSet) }} affinity: podAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -61,9 +88,7 @@ spec: - {{ .app.harness.deployment.volume.name }} # Obsolete, kept for backwards compatability, to be removed at a later date topologyKey: "kubernetes.io/hostname" {{- end }} - {{- end }} - {{- if .app.harness.deployment.extraContainers }} - {{- $hasInit := false }} + {{- $hasInit := $legacyClaim }} {{- range $name, $container := .app.harness.deployment.extraContainers }} {{- if and $container.auto $container.initContainer }} {{- $hasInit = true }} @@ -71,13 +96,22 @@ spec: {{- end }} {{- if $hasInit }} initContainers: + {{- if $legacyClaim }} + # Holds the pod until the volume-migration job has copied the legacy volume data into + # this pod's volume. Delete the legacy PVC after verifying the migration to drop this gate. + - name: volume-migration + image: busybox:1.36 + command: ['sh', '-c', 'until [ -f /migration/target/.cloudharness-volume-migrated ]; do echo "waiting for the volume migration job"; sleep 5; done'] + volumeMounts: + - name: {{ $volume.name }} + mountPath: /migration/target + {{- end }} {{- range $name, $container := .app.harness.deployment.extraContainers }} {{- if and $container.auto $container.initContainer }} {{- include "deploy_utils.extraContainerSpec" (dict "name" $name "container" $container "app" $.app "root" $.root) | nindent 6 }} {{- end }} {{- end }} {{- end }} - {{- end }} containers: - name: {{ .app.harness.deployment.name| default "cloudharness-docs" | quote }} image: {{ .app.harness.deployment.image }} @@ -152,7 +186,7 @@ spec: secretName: {{ $dep }} {{- end }} {{- end }} - {{- if .app.harness.deployment.volume }} + {{- if and .app.harness.deployment.volume (not $useClaimTemplate) }} - name: {{ .app.harness.deployment.volume.name }} persistentVolumeClaim: claimName: {{ .app.harness.deployment.volume.name }} @@ -175,6 +209,21 @@ spec: secretName: {{ printf "%s-db" .app.harness.deployment.name }} {{- end }} {{- end }} + {{- if $useClaimTemplate }} + volumeClaimTemplates: + - metadata: + name: {{ $volume.name }} + spec: + accessModes: + - ReadWriteOnce + storageClassName: standard + resources: + requests: + storage: {{ $volume.size }} + {{- end }} + {{- if $legacyClaim }} + {{- include "deploy_utils.volumeMigration" (dict "root" .root "name" .app.harness.deployment.name "pvc" $volume.name) }} + {{- end }} --- {{- end }} diff --git a/deployment-configuration/helm/templates/auto-services.yaml b/deployment-configuration/helm/templates/auto-services.yaml index 4c58a815b..bb276a9de 100644 --- a/deployment-configuration/helm/templates/auto-services.yaml +++ b/deployment-configuration/helm/templates/auto-services.yaml @@ -16,6 +16,25 @@ spec: targetPort: {{ .app.harness.deployment.port }} name: http --- +{{- if .app.harness.deployment.statefulset }} +{{/* Leader service: always resolves to pod 0 of the statefulset, to be used for write requests */}} +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-rw" .app.harness.service.name | quote }} + labels: + app: {{ .app.harness.deployment.name | quote }} +{{ include "deploy_utils.labels" .root | indent 4 }} +spec: + selector: + app: {{ .app.harness.deployment.name| quote }} + statefulset.kubernetes.io/pod-name: {{ printf "%s-0" .app.harness.deployment.name | quote }} + ports: + - port: {{ .app.harness.service.port }} + targetPort: {{ .app.harness.deployment.port }} + name: http +--- +{{- end }} {{- end }} {{- range $app := .Values.apps }} {{- if $app.harness.service.auto }} diff --git a/deployment-configuration/helm/templates/auto-volume-migration.yaml b/deployment-configuration/helm/templates/auto-volume-migration.yaml new file mode 100644 index 000000000..2b9d8b93c --- /dev/null +++ b/deployment-configuration/helm/templates/auto-volume-migration.yaml @@ -0,0 +1,112 @@ +{{/* Volume migration: copies the data of a legacy (pre-statefulset) PVC into the statefulset + volumes. The job mounts only the legacy volume and streams the data through the Kubernetes + API (tar over kubectl exec, as kubectl cp does) into the volume-migration init container of + each statefulset pod, which mounts only its own volume: the two volumes are never mounted by + the same pod, so the migration also works when they end up in different zones (multi-zone + and multi-region clusters). Rendered only while the legacy PVC exists: delete it to end the + migration. */}} +{{- define "deploy_utils.volumeMigration" }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ printf "%s-volume-migration" .name | quote }} + namespace: {{ .root.Values.namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ printf "%s-volume-migration" .name | quote }} + namespace: {{ .root.Values.namespace }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + - apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ printf "%s-volume-migration" .name | quote }} + namespace: {{ .root.Values.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ printf "%s-volume-migration" .name | quote }} +subjects: + - kind: ServiceAccount + name: {{ printf "%s-volume-migration" .name | quote }} + namespace: {{ .root.Values.namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + # The release revision suffix makes each upgrade create a fresh job (job specs are immutable); + # completed jobs clean themselves up via ttlSecondsAfterFinished. + name: {{ printf "%s-volume-migration-r%v" .name .root.Release.Revision | quote }} + namespace: {{ .root.Values.namespace }} + labels: + app: {{ printf "%s-volume-migration" .name | quote }} +spec: + ttlSecondsAfterFinished: 600 + backoffLimit: 20 + template: + metadata: + labels: + app: {{ printf "%s-volume-migration" .name | quote }} + spec: + serviceAccountName: {{ printf "%s-volume-migration" .name | quote }} + restartPolicy: OnFailure + containers: + - name: volume-migration + # Fully open source kubectl image (alpine-based: provides sh and tar), see + # https://github.com/alpine-docker/k8s + image: alpine/k8s:1.34.9 + command: + - sh + - -c + - | + set -e + name={{ .name | quote }} + ns={{ .root.Values.namespace | quote }} + marker=.cloudharness-volume-migrated + replicas=$(kubectl -n $ns get statefulset $name -o jsonpath='{.spec.replicas}') + for i in $(seq 0 $((replicas - 1))); do + pod=$name-$i + echo "waiting for pod $pod" + until kubectl -n $ns get pod $pod > /dev/null 2>&1; do sleep 5; done + while true; do + phase=$(kubectl -n $ns get pod $pod -o jsonpath='{.status.phase}') + running=$(kubectl -n $ns get pod $pod -o jsonpath='{.status.initContainerStatuses[?(@.name=="volume-migration")].state.running}') + if [ -n "$running" ] || [ "$phase" = "Running" ]; then break; fi + sleep 5 + done + if [ "$phase" = "Running" ]; then + echo "$pod is already running: nothing to do" + continue + fi + if kubectl -n $ns exec $pod -c volume-migration -- test -f /migration/target/$marker; then + echo "$pod is already migrated" + continue + fi + echo "copying the legacy volume data to $pod" + tar cf - -C /migration/legacy . | kubectl -n $ns exec -i $pod -c volume-migration -- tar xf - -C /migration/target + kubectl -n $ns exec $pod -c volume-migration -- touch /migration/target/$marker + echo "$pod migrated" + done + echo "volume migration of $name completed" + volumeMounts: + - name: legacy + mountPath: /migration/legacy + readOnly: true + volumes: + - name: legacy + persistentVolumeClaim: + claimName: {{ .pvc | quote }} + readOnly: true +{{- end }} diff --git a/deployment-configuration/helm/templates/auto-volumes.yaml b/deployment-configuration/helm/templates/auto-volumes.yaml index 3656630d4..15edb91c7 100644 --- a/deployment-configuration/helm/templates/auto-volumes.yaml +++ b/deployment-configuration/helm/templates/auto-volumes.yaml @@ -1,9 +1,29 @@ {{- define "deploy_utils.pvolume" }} -{{- if or (not (hasKey .app.harness.deployment.volume "auto")) .app.harness.deployment.volume.auto }} +{{- $volume := .app.harness.deployment.volume }} +{{- $isStatefulSet := .app.harness.deployment.statefulset | default false }} +{{- $ownVolume := false }} +{{- if and (or (not (hasKey $volume "usenfs")) (not $volume.usenfs)) (or (not (hasKey $volume "auto")) $volume.auto) }} +{{- $ownVolume = true }} +{{- end }} +{{- /* Statefulsets own their (non-nfs) volume through volumeClaimTemplates: the standalone PVC + is only kept while a legacy one exists, so that its data can be migrated. Delete the legacy + PVC once migrated. */}} +{{- $legacyClaim := false }} +{{- if and $isStatefulSet $ownVolume }} +{{- if lookup "v1" "PersistentVolumeClaim" .root.Values.namespace $volume.name }} +{{- $legacyClaim = true }} +{{- end }} +{{- end }} +{{- if and (or (not (hasKey $volume "auto")) $volume.auto) (or (not (and $isStatefulSet $ownVolume)) $legacyClaim) }} apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: {{ .app.harness.deployment.volume.name }} + name: {{ $volume.name }} + {{- if $legacyClaim }} + annotations: + # Legacy volume pending migration to the statefulset volumes: never deleted by helm. + helm.sh/resource-policy: keep + {{- end }} labels: app: {{ .app.harness.deployment.name| quote }} spec: diff --git a/deployment-configuration/helm/templates/ingress.yaml b/deployment-configuration/helm/templates/ingress.yaml index b0c28ebcd..0f0c647ea 100644 --- a/deployment-configuration/helm/templates/ingress.yaml +++ b/deployment-configuration/helm/templates/ingress.yaml @@ -9,6 +9,10 @@ {{- if and $app.harness.secured $secured_gatekeepers $app.harness.uri_role_mapping (eq $app.harness.gateway.pathType "Prefix") }} {{- range $mapping := $app.harness.uri_role_mapping }} {{- if and (hasKey $mapping "white-listed") (index $mapping "white-listed") }} + {{- /* White-listed uris bypass the gatekeeper; the ones declaring a write method are + routed to the leader (pod 0) service of a statefulset deployment. */}} + {{- $isWrite := and $app.harness.deployment.statefulset $mapping.methods (or (has "POST" $mapping.methods) (has "PUT" $mapping.methods) (has "PATCH" $mapping.methods) (has "DELETE" $mapping.methods)) }} + {{- $backendService := ternary (printf "%s-rw" $app.harness.service.name) $app.harness.service.name (not (not $isWrite)) }} {{- $uri := $mapping.uri }} {{- if and (ne $uri "/") (ne $uri "/*") }} {{- if hasSuffix "/*" $uri }} @@ -18,7 +22,7 @@ pathType: Prefix backend: service: - name: {{ $app.harness.service.name | quote }} + name: {{ $backendService | quote }} port: number: {{ $app.harness.service.port | default 80 }} {{- else if (not (contains "*" $uri)) }} @@ -27,7 +31,39 @@ pathType: ImplementationSpecific backend: service: - name: {{ $app.harness.service.name | quote }} + name: {{ $backendService | quote }} + port: + number: {{ $app.harness.service.port | default 80 }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- /* uri_role_mapping entries declaring a write method are routed to the leader (pod 0) + service. Skipped for gatekeeper-secured apps, as routing straight to the service would + bypass authentication: those must forward writes at the application level. */}} + {{- if and $app.harness.deployment.statefulset $app.harness.uri_role_mapping (not (and $app.harness.secured $secured_gatekeepers)) (eq $app.harness.gateway.pathType "Prefix") }} + {{- range $mapping := $app.harness.uri_role_mapping }} + {{- if and $mapping.methods (or (has "POST" $mapping.methods) (has "PUT" $mapping.methods) (has "PATCH" $mapping.methods) (has "DELETE" $mapping.methods)) }} + {{- $uri := $mapping.uri }} + {{- if and (ne $uri "/") (ne $uri "/*") }} + {{- if hasSuffix "/*" $uri }} + {{- $cleanPath := trimSuffix "/*" $uri }} + {{- $pathWithoutSlash := trimPrefix "/" $cleanPath }} + - path: {{ printf "%s%s" $app.harness.gateway.path $pathWithoutSlash }} + pathType: Prefix + backend: + service: + name: {{ printf "%s-rw" $app.harness.service.name | quote }} + port: + number: {{ $app.harness.service.port | default 80 }} + {{- else if (not (contains "*" $uri)) }} + {{- $pathWithoutSlash := trimPrefix "/" $uri }} + - path: {{ printf "%s%s" $app.harness.gateway.path $pathWithoutSlash }} + pathType: ImplementationSpecific + backend: + service: + name: {{ printf "%s-rw" $app.harness.service.name | quote }} port: number: {{ $app.harness.service.port | default 80 }} {{- end }} diff --git a/deployment-configuration/value-template.yaml b/deployment-configuration/value-template.yaml index e5e8aa136..da5a3920d 100644 --- a/deployment-configuration/value-template.yaml +++ b/deployment-configuration/value-template.yaml @@ -15,7 +15,7 @@ harness: build: [] # -- When true, the application is shielded with a getekeeper secured: false - # -- Uri/Role mapping for the gatekeeper + # -- Uri/Role mapping for the gatekeeper. Entries can also specify `methods`: on secured applications they restrict the gatekeeper resource to those methods; on statefulset deployments, uris declaring a write method (POST/PUT/PATCH/DELETE) are routed through the ingress to the leader service (pod 0). uri_role_mapping: - uri: /* roles: @@ -38,6 +38,8 @@ harness: port: 8080 # -- volume specification volume: + # -- When true, the deployment is rendered as a StatefulSet instead of a Deployment. Recommended for deployments with a (non-nfs) volume: updates terminate the old pod before creating the new one. The volume is provisioned per replica through volumeClaimTemplates; data of a pre-existing PVC named after the volume is copied into each statefulset volume by a migration job (delete the legacy PVC once migrated). + statefulset: false # -- Deployment resources. resources: requests: @@ -96,6 +98,8 @@ harness: image_ref: # -- expose database to the public with ingress expose: false + # -- When true, the database is rendered as a StatefulSet instead of a Deployment, provisioning its volume through volumeClaimTemplates. Data of a pre-existing database PVC is copied into the statefulset volume by a migration job (delete the legacy PVC once migrated). Ignored when postgres.operator is true. + statefulset: false # -- Set to "" to set the set the string in the CI/CD as a secret. Only set the full value for dev/testing connect_string: # -- settings for mongo database (for type==mongo) diff --git a/docs/applications/databases.md b/docs/applications/databases.md index a52755978..3e1563aff 100644 --- a/docs/applications/databases.md +++ b/docs/applications/databases.md @@ -38,6 +38,15 @@ harness: `expose`: This option allows you to expose the database port through a load balancer. Do not use on production! +`statefulset`: When true, the database is rendered as a Kubernetes `StatefulSet` instead of a +`Deployment`, removing the need for the `Recreate` strategy and podAffinity pinning normally +used to work around the ReadWriteOnce volume. The volume is provisioned through +`volumeClaimTemplates`; if a database PVC from a previous Deployment exists in the cluster, it +is kept (never deleted by Helm) and its data is copied into the statefulset volume by a +migration job — delete the legacy PVC once migrated (see +[Volumes](./volumes.md#deploying-as-a-statefulset)). Ignored when `postgres.operator: true`, +since the CNPG operator manages its own workloads. + ### Specific database settings diff --git a/docs/applications/volumes.md b/docs/applications/volumes.md index 0764fc6a5..8d17e06b0 100644 --- a/docs/applications/volumes.md +++ b/docs/applications/volumes.md @@ -60,4 +60,96 @@ harness: auto: true size: 5Gi usenfs: true -``` \ No newline at end of file +``` + +## Deploying as a StatefulSet + +By default, a deployment with a (non-nfs) volume is rendered as a Kubernetes `Deployment` with a +`Recreate` update strategy and podAffinity pinning it to the node holding the volume, since a +ReadWriteOnce volume can only attach to one node at a time. + +Setting `harness.deployment.statefulset: true` renders it as a `StatefulSet` instead. StatefulSet +updates terminate the old pod before creating its replacement, so neither the `Recreate` strategy +nor node pinning is needed. + +```yaml +harness: + ... + deployment: + ... + statefulset: true + volume: + name: my-volume + mountpath: /usr/src/app/myvolume + auto: true + size: 5Gi +``` + +The volume is provisioned per replica through `volumeClaimTemplates` (PVCs named +`--`). Exceptions: nfs volumes (`usenfs: true`) and externally managed +volumes (`auto: false`) keep mounting their common PVC by name — per-replica claims would +un-share them. + +**Migrating from an existing Deployment**: if a PVC named after the volume exists in the cluster +at deploy time (left over from the pre-statefulset Deployment), it is treated as a legacy volume: + +- it is kept in the release with `helm.sh/resource-policy: keep`, so Helm never deletes it; +- a `-volume-migration` Job is deployed that mounts the legacy volume (read-only) and + streams its data through the Kubernetes API (tar over `kubectl exec`, as `kubectl cp` does) + into each statefulset pod. The legacy and statefulset volumes are never mounted by the same + pod, so the migration also works on multi-zone/multi-region clusters where the two volumes + may not be attachable to the same node; +- each statefulset pod runs a `volume-migration` init container that mounts only the pod's own + volume and holds the pod until the job has copied the data into it (guarded by a + `.cloudharness-volume-migrated` marker, so each volume is migrated exactly once). + +Once the migration is verified, delete the legacy PVC (`kubectl delete pvc `): the +next deployment drops the migration job, its RBAC resources and the init container gate. Note +that the data flows through the Kubernetes API server: for very large volumes consider a manual +migration instead. + +Note that `volumeClaimTemplates` are immutable once created: changing `volume.size` afterwards +won't resize the PVCs and requires a manual resize. Flipping the flag on a live release causes +Helm to delete the Deployment and create the StatefulSet, with a brief window of downtime while +the volume detaches and reattaches. + +### Routing writes to a single pod (leader service) + +For a StatefulSet, an additional service named `-rw` is automatically created that +always resolves to pod 0. This supports a single-writer pattern when running multiple replicas +over a shared volume: any pod can serve reads, but write requests are handled only by pod 0. + +Writes can be routed to the leader in two ways: + +- **From the application**: forward mutating requests to `http://-rw` when the pod + is not the leader (inside a pod, `HOSTNAME` equals the pod name, so the leader check is + `HOSTNAME == "-0"`). This is the most reliable option, as only the + application knows which requests actually write. +- **From the ingress**: declare the write endpoints in `harness.uri_role_mapping` with their + `methods`; uris declaring a write method (POST/PUT/PATCH/DELETE) are routed to the leader + service instead of the load-balanced one: + + ```yaml + harness: + ... + deployment: + statefulset: true + replicas: 3 + uri_role_mapping: + - uri: /api/edit/* + methods: [POST, PUT, PATCH, DELETE] + - uri: /api/upload + methods: [POST] + ``` + + Routing granularity is the uri, not the method (plain Kubernetes Ingress cannot match methods), + so all requests to those uris — including GETs — go to pod 0. On secured applications the same + `methods` field restricts the gatekeeper resource to those methods, and leader routing applies + only to `white-listed` uris (which bypass the gatekeeper by design): routing a secured uri + around the auth proxy would bypass authentication, so non-white-listed write endpoints must + forward writes at the application level. See the samples application `/api/write-file` + endpoint for a working example. + +Note that pod 0 restarts last during rolling updates, but while it restarts write requests fail: +clients should retry. Also mind that after a write, replicas may briefly serve stale reads (see +the NFS caching notes above). \ No newline at end of file diff --git a/docs/model/DatabaseDeploymentConfig.md b/docs/model/DatabaseDeploymentConfig.md index 143a76cb9..b7aea5546 100644 --- a/docs/model/DatabaseDeploymentConfig.md +++ b/docs/model/DatabaseDeploymentConfig.md @@ -13,6 +13,7 @@ Name | Type | Description | Notes **user** | **str** | database username | [optional] **var_pass** | **str** | Database password | [optional] **image_ref** | **str** | Used for referencing images from the build | [optional] +**statefulset** | **bool** | When true, the auto-generated database is rendered as a StatefulSet instead of a Deployment, provisioning its volume through volumeClaimTemplates. The data of a pre-existing database PVC is copied into the statefulset volume by a migration job (delete the legacy PVC once migrated). Ignored when postgres.operator is true. | [optional] **mongo** | **Dict[str, object]** | | [optional] **postgres** | **Dict[str, object]** | | [optional] **neo4j** | **object** | Neo4j database specific configuration | [optional] diff --git a/docs/model/DeploymentAutoArtifactConfig.md b/docs/model/DeploymentAutoArtifactConfig.md index 07e067a99..1b5de8692 100644 --- a/docs/model/DeploymentAutoArtifactConfig.md +++ b/docs/model/DeploymentAutoArtifactConfig.md @@ -13,6 +13,7 @@ Name | Type | Description | Notes **image** | **str** | Image name to use in the deployment. Leave it blank to set from the application's Docker file | [optional] **resources** | [**DeploymentResourcesConf**](DeploymentResourcesConf.md) | | [optional] **volume** | [**DeploymentVolumeSpec**](DeploymentVolumeSpec.md) | | [optional] +**statefulset** | **bool** | When true, the workload is rendered as a Kubernetes StatefulSet instead of a Deployment. Recommended for deployments with a ReadWriteOnce volume: updates terminate the old pod before creating the new one, so no Recreate strategy or node pinning is needed. The volume, unless nfs-shared or externally managed (auto false), is provisioned per replica through volumeClaimTemplates. A pre-existing PVC named after the volume (left over from a previous Deployment) is migrated automatically: a migration job streams its data into each statefulset volume through the Kubernetes API, so the volumes are never mounted by the same pod (works on multi-zone clusters); delete the legacy PVC once migrated. | [optional] **network** | [**NetworkConfig**](NetworkConfig.md) | | [optional] **extra_containers** | [**Dict[str, ExtraContainerConfig]**](ExtraContainerConfig.md) | Extra containers (init containers and sidecars) for the deployment. Each key is a container name mapping to an ExtraContainerConfig. | [optional] diff --git a/docs/model/UriRoleMappingConfig.md b/docs/model/UriRoleMappingConfig.md index 4438687a4..f716cc5c2 100644 --- a/docs/model/UriRoleMappingConfig.md +++ b/docs/model/UriRoleMappingConfig.md @@ -8,6 +8,7 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **uri** | **str** | | **roles** | **List[str]** | Roles allowed to access the present uri | [optional] +**methods** | **List[str]** | HTTP methods (uppercase) the mapping applies to. Empty means all methods. Passed through to the gatekeeper resource when `secured` is true. When `deployment.statefulset` is true, uris declaring a write method (POST/PUT/PATCH/DELETE) are also routed through the ingress to the leader service (pod 0). | [optional] **white_listed** | **bool** | | [optional] ## Example diff --git a/libraries/models/api/openapi.yaml b/libraries/models/api/openapi.yaml index 3c9abf03a..079b98c36 100644 --- a/libraries/models/api/openapi.yaml +++ b/libraries/models/api/openapi.yaml @@ -643,6 +643,14 @@ components: description: Used for referencing images from the build type: string example: 'image_ref: myownpgimage' + statefulset: + description: >- + When true, the auto-generated database is rendered as a StatefulSet instead + of a Deployment, provisioning its volume through volumeClaimTemplates. The + data of a pre-existing database PVC is copied into the statefulset volume by + a migration job (delete the legacy PVC once migrated). Ignored when + postgres.operator is true. + type: boolean mongo: $ref: '#/components/schemas/FreeObject' description: Mongo db specific configuration @@ -681,6 +689,16 @@ components: type: array items: type: string + methods: + description: >- + HTTP methods (uppercase) the mapping applies to. Empty means all methods. + Passed through to the gatekeeper resource when `secured` is true. When + `deployment.statefulset` is true, uris declaring a write method + (POST/PUT/PATCH/DELETE) are also routed through the ingress to the leader + service (pod 0). + type: array + items: + type: string white-listed: description: '' type: boolean @@ -1232,6 +1250,19 @@ components: volume: $ref: '#/components/schemas/DeploymentVolumeSpec' description: Volume specification + statefulset: + description: >- + When true, the workload is rendered as a Kubernetes StatefulSet instead of a + Deployment. Recommended for deployments with a ReadWriteOnce volume: updates + terminate the old pod before creating the new one, so no Recreate strategy or + node pinning is needed. The volume, unless nfs-shared or externally managed + (auto false), is provisioned per replica through volumeClaimTemplates. A + pre-existing PVC named after the volume (left over from a previous Deployment) + is migrated automatically: a migration job streams its data into each + statefulset volume through the Kubernetes API, so the volumes are never + mounted by the same pod (works on multi-zone clusters); delete the legacy PVC + once migrated. + type: boolean network: $ref: '#/components/schemas/NetworkConfig' description: Define the network policy for the deployment diff --git a/libraries/models/cloudharness_model/models/database_deployment_config.py b/libraries/models/cloudharness_model/models/database_deployment_config.py index 649a3f123..1f8ada043 100644 --- a/libraries/models/cloudharness_model/models/database_deployment_config.py +++ b/libraries/models/cloudharness_model/models/database_deployment_config.py @@ -38,13 +38,14 @@ class DatabaseDeploymentConfig(CloudHarnessBaseModel): user: Optional[StrictStr] = Field(default=None, description="database username") var_pass: Optional[StrictStr] = Field(default=None, description="Database password", alias="pass") image_ref: Optional[StrictStr] = Field(default=None, description="Used for referencing images from the build") + statefulset: Optional[StrictBool] = Field(default=None, description="When true, the auto-generated database is rendered as a StatefulSet instead of a Deployment, provisioning its volume through volumeClaimTemplates. The data of a pre-existing database PVC is copied into the statefulset volume by a migration job (delete the legacy PVC once migrated). Ignored when postgres.operator is true.") mongo: Optional[Dict[str, Any]] = None postgres: Optional[Dict[str, Any]] = None neo4j: Optional[Any] = Field(default=None, description="Neo4j database specific configuration") resources: Optional[DeploymentResourcesConf] = None connect_string: Optional[StrictStr] = Field(default=None, description="Specify if the database is external. If not null, auto deployment if set will not be used. Leave it as an empty string and the connect string will be provided as a secret to be provided at CI/CD (recommended)") additional_properties: Dict[str, Any] = {} - __properties: ClassVar[List[str]] = ["auto", "name", "type", "size", "user", "pass", "image_ref", "mongo", "postgres", "neo4j", "resources", "connect_string"] + __properties: ClassVar[List[str]] = ["auto", "name", "type", "size", "user", "pass", "image_ref", "statefulset", "mongo", "postgres", "neo4j", "resources", "connect_string"] @field_validator('type') def type_validate_regular_expression(cls, value): @@ -108,6 +109,7 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "user": obj.get("user"), "pass": obj.get("pass"), "image_ref": obj.get("image_ref"), + "statefulset": obj.get("statefulset"), "mongo": obj.get("mongo"), "postgres": obj.get("postgres"), "neo4j": obj.get("neo4j"), diff --git a/libraries/models/cloudharness_model/models/deployment_auto_artifact_config.py b/libraries/models/cloudharness_model/models/deployment_auto_artifact_config.py index 62029d912..ed08a588a 100644 --- a/libraries/models/cloudharness_model/models/deployment_auto_artifact_config.py +++ b/libraries/models/cloudharness_model/models/deployment_auto_artifact_config.py @@ -41,10 +41,11 @@ class DeploymentAutoArtifactConfig(CloudHarnessBaseModel): image: Optional[Annotated[str, Field(strict=True)]] = Field(default=None, description="Image name to use in the deployment. Leave it blank to set from the application's Docker file") resources: Optional[DeploymentResourcesConf] = None volume: Optional[DeploymentVolumeSpec] = None + statefulset: Optional[StrictBool] = Field(default=None, description="When true, the workload is rendered as a Kubernetes StatefulSet instead of a Deployment. Recommended for deployments with a ReadWriteOnce volume: updates terminate the old pod before creating the new one, so no Recreate strategy or node pinning is needed. The volume, unless nfs-shared or externally managed (auto false), is provisioned per replica through volumeClaimTemplates. A pre-existing PVC named after the volume (left over from a previous Deployment) is migrated automatically: a migration job streams its data into each statefulset volume through the Kubernetes API, so the volumes are never mounted by the same pod (works on multi-zone clusters); delete the legacy PVC once migrated.") network: Optional[NetworkConfig] = None extra_containers: Optional[Dict[str, ExtraContainerConfig]] = Field(default=None, description="Extra containers (init containers and sidecars) for the deployment. Each key is a container name mapping to an ExtraContainerConfig.", alias="extraContainers") additional_properties: Dict[str, Any] = {} - __properties: ClassVar[List[str]] = ["auto", "name", "port", "replicas", "image", "resources", "volume", "network", "extraContainers"] + __properties: ClassVar[List[str]] = ["auto", "name", "port", "replicas", "image", "resources", "volume", "statefulset", "network", "extraContainers"] @field_validator('image') def image_validate_regular_expression(cls, value): @@ -121,6 +122,7 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "image": obj.get("image"), "resources": DeploymentResourcesConf.from_dict(obj["resources"]) if obj.get("resources") is not None else None, "volume": DeploymentVolumeSpec.from_dict(obj["volume"]) if obj.get("volume") is not None else None, + "statefulset": obj.get("statefulset"), "network": NetworkConfig.from_dict(obj["network"]) if obj.get("network") is not None else None, "extraContainers": dict( (_k, ExtraContainerConfig.from_dict(_v)) diff --git a/libraries/models/cloudharness_model/models/uri_role_mapping_config.py b/libraries/models/cloudharness_model/models/uri_role_mapping_config.py index 7ef35c249..7082b02c5 100644 --- a/libraries/models/cloudharness_model/models/uri_role_mapping_config.py +++ b/libraries/models/cloudharness_model/models/uri_role_mapping_config.py @@ -32,9 +32,10 @@ class UriRoleMappingConfig(CloudHarnessBaseModel): """ # noqa: E501 uri: Annotated[str, Field(strict=True)] roles: Optional[List[StrictStr]] = Field(default=None, description="Roles allowed to access the present uri") + methods: Optional[List[StrictStr]] = Field(default=None, description="HTTP methods (uppercase) the mapping applies to. Empty means all methods. Passed through to the gatekeeper resource when `secured` is true. When `deployment.statefulset` is true, uris declaring a write method (POST/PUT/PATCH/DELETE) are also routed through the ingress to the leader service (pod 0).") white_listed: Optional[StrictBool] = Field(default=None, alias="white-listed") additional_properties: Dict[str, Any] = {} - __properties: ClassVar[List[str]] = ["uri", "roles", "white-listed"] + __properties: ClassVar[List[str]] = ["uri", "roles", "methods", "white-listed"] @field_validator('uri') def uri_validate_regular_expression(cls, value): @@ -82,6 +83,7 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: _obj = cls.model_validate({ "uri": obj.get("uri"), "roles": obj.get("roles"), + "methods": obj.get("methods"), "white-listed": obj.get("white-listed") }) # store additional fields in additional_properties diff --git a/libraries/models/docs/DatabaseDeploymentConfig.md b/libraries/models/docs/DatabaseDeploymentConfig.md index 143a76cb9..b7aea5546 100644 --- a/libraries/models/docs/DatabaseDeploymentConfig.md +++ b/libraries/models/docs/DatabaseDeploymentConfig.md @@ -13,6 +13,7 @@ Name | Type | Description | Notes **user** | **str** | database username | [optional] **var_pass** | **str** | Database password | [optional] **image_ref** | **str** | Used for referencing images from the build | [optional] +**statefulset** | **bool** | When true, the auto-generated database is rendered as a StatefulSet instead of a Deployment, provisioning its volume through volumeClaimTemplates. The data of a pre-existing database PVC is copied into the statefulset volume by a migration job (delete the legacy PVC once migrated). Ignored when postgres.operator is true. | [optional] **mongo** | **Dict[str, object]** | | [optional] **postgres** | **Dict[str, object]** | | [optional] **neo4j** | **object** | Neo4j database specific configuration | [optional] diff --git a/libraries/models/docs/DeploymentAutoArtifactConfig.md b/libraries/models/docs/DeploymentAutoArtifactConfig.md index 07e067a99..1b5de8692 100644 --- a/libraries/models/docs/DeploymentAutoArtifactConfig.md +++ b/libraries/models/docs/DeploymentAutoArtifactConfig.md @@ -13,6 +13,7 @@ Name | Type | Description | Notes **image** | **str** | Image name to use in the deployment. Leave it blank to set from the application's Docker file | [optional] **resources** | [**DeploymentResourcesConf**](DeploymentResourcesConf.md) | | [optional] **volume** | [**DeploymentVolumeSpec**](DeploymentVolumeSpec.md) | | [optional] +**statefulset** | **bool** | When true, the workload is rendered as a Kubernetes StatefulSet instead of a Deployment. Recommended for deployments with a ReadWriteOnce volume: updates terminate the old pod before creating the new one, so no Recreate strategy or node pinning is needed. The volume, unless nfs-shared or externally managed (auto false), is provisioned per replica through volumeClaimTemplates. A pre-existing PVC named after the volume (left over from a previous Deployment) is migrated automatically: a migration job streams its data into each statefulset volume through the Kubernetes API, so the volumes are never mounted by the same pod (works on multi-zone clusters); delete the legacy PVC once migrated. | [optional] **network** | [**NetworkConfig**](NetworkConfig.md) | | [optional] **extra_containers** | [**Dict[str, ExtraContainerConfig]**](ExtraContainerConfig.md) | Extra containers (init containers and sidecars) for the deployment. Each key is a container name mapping to an ExtraContainerConfig. | [optional] diff --git a/libraries/models/docs/UriRoleMappingConfig.md b/libraries/models/docs/UriRoleMappingConfig.md index 4438687a4..f716cc5c2 100644 --- a/libraries/models/docs/UriRoleMappingConfig.md +++ b/libraries/models/docs/UriRoleMappingConfig.md @@ -8,6 +8,7 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **uri** | **str** | | **roles** | **List[str]** | Roles allowed to access the present uri | [optional] +**methods** | **List[str]** | HTTP methods (uppercase) the mapping applies to. Empty means all methods. Passed through to the gatekeeper resource when `secured` is true. When `deployment.statefulset` is true, uris declaring a write method (POST/PUT/PATCH/DELETE) are also routed through the ingress to the leader service (pod 0). | [optional] **white_listed** | **bool** | | [optional] ## Example diff --git a/tools/deployment-cli-tools/tests/test_helm.py b/tools/deployment-cli-tools/tests/test_helm.py index 57126bc24..346bd465d 100644 --- a/tools/deployment-cli-tools/tests/test_helm.py +++ b/tools/deployment-cli-tools/tests/test_helm.py @@ -365,6 +365,166 @@ def test_cnpg_postgres_parameters_render_only_when_set(tmp_path): assert 'postgresql' not in cluster['spec'] +def test_statefulset_option(tmp_path): + out_folder = tmp_path / 'test_statefulset_option' + # nfsserver is included to provide the storage class values needed by the usenfs case + create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_folder, domain="my.local", + env='withpostgres', local=False, include=["myapp", "nfsserver"], exclude=["legacy"]) + + helm_path = out_folder / HELM_CHART_PATH + shutil.rmtree(helm_path / 'charts') + values_path = helm_path / 'values.yaml' + with open(values_path, 'r') as values_file: + values = yaml.safe_load(values_file) + + myapp = values['apps']['myapp'] + harness = myapp['harness'] + dep_name = harness['deployment']['name'] + db_name = harness['database']['name'] + service_name = harness['service']['name'] + + harness['deployment']['auto'] = True + harness['deployment']['volume'] = { + 'name': 'myapp-data', 'mountpath': '/data', 'size': '1Gi', 'auto': True, + } + with open(values_path, 'w') as values_file: + yaml.safe_dump(values, values_file) + + # Default: Deployments with the Recreate/affinity workaround and a standalone PVC + manifests = render_helm_chart(helm_path) + dep = find_manifest(manifests, 'Deployment', dep_name) + assert dep['spec']['strategy']['type'] == 'Recreate' + assert 'affinity' in dep['spec']['template']['spec'] + find_manifest(manifests, 'PersistentVolumeClaim', 'myapp-data') + db_dep = find_manifest(manifests, 'Deployment', db_name) + assert db_dep['spec']['strategy']['type'] == 'Recreate' + assert 'affinity' in db_dep['spec']['template']['spec'] + find_manifest(manifests, 'PersistentVolumeClaim', db_name) + + # Opt in to StatefulSets: volumes are provisioned via volumeClaimTemplates. The legacy + # volume migration (job copying a pre-existing PVC found by `lookup` into the statefulset + # volumes) cannot be exercised here: `helm template` runs without a cluster, so `lookup` + # finds nothing. + harness['deployment']['statefulset'] = True + harness['database']['statefulset'] = True + with open(values_path, 'w') as values_file: + yaml.safe_dump(values, values_file) + + manifests = render_helm_chart(helm_path) + sts = find_manifest(manifests, 'StatefulSet', dep_name) + assert sts['spec']['serviceName'] == service_name + assert 'strategy' not in sts['spec'] + assert 'affinity' not in sts['spec']['template']['spec'] + assert 'initContainers' not in sts['spec']['template']['spec'] + claims = [v['persistentVolumeClaim']['claimName'] + for v in sts['spec']['template']['spec']['volumes'] if 'persistentVolumeClaim' in v] + assert 'myapp-data' not in claims + assert sts['spec']['volumeClaimTemplates'][0]['metadata']['name'] == 'myapp-data' + assert not any(m for m in manifests + if m.get('kind') == 'PersistentVolumeClaim' and m.get('metadata', {}).get('name') == 'myapp-data') + + db_sts = find_manifest(manifests, 'StatefulSet', db_name) + assert db_sts['spec']['serviceName'] == db_name + assert 'strategy' not in db_sts['spec'] + assert 'affinity' not in db_sts['spec']['template']['spec'] + assert 'initContainers' not in db_sts['spec']['template']['spec'] + assert db_sts['spec']['volumeClaimTemplates'][0]['metadata']['name'] == db_name + assert not any(m for m in manifests + if m.get('kind') == 'PersistentVolumeClaim' and m.get('metadata', {}).get('name') == db_name) + find_manifest(manifests, 'Service', db_name) + # without a legacy PVC no migration resources are rendered + assert not any(m for m in manifests if 'volume-migration' in m.get('metadata', {}).get('name', '')) + + # nfs (shared) volumes are never per-replica: the statefulset keeps mounting the common + # PVC by claimName and no volumeClaimTemplates are created. + harness['deployment']['volume']['usenfs'] = True + with open(values_path, 'w') as values_file: + yaml.safe_dump(values, values_file) + + manifests = render_helm_chart(helm_path) + sts = find_manifest(manifests, 'StatefulSet', dep_name) + assert 'volumeClaimTemplates' not in sts['spec'] + claims = [v['persistentVolumeClaim']['claimName'] + for v in sts['spec']['template']['spec']['volumes'] if 'persistentVolumeClaim' in v] + assert 'myapp-data' in claims + shared_pvc = find_manifest(manifests, 'PersistentVolumeClaim', 'myapp-data') + assert shared_pvc['spec']['accessModes'] == ['ReadWriteMany'] + + # volume.auto: false means the PVC is managed externally: always reference it by + # claimName, never via volumeClaimTemplates. + harness['deployment']['volume']['usenfs'] = False + harness['deployment']['volume']['auto'] = False + with open(values_path, 'w') as values_file: + yaml.safe_dump(values, values_file) + + manifests = render_helm_chart(helm_path) + sts = find_manifest(manifests, 'StatefulSet', dep_name) + assert 'volumeClaimTemplates' not in sts['spec'] + claims = [v['persistentVolumeClaim']['claimName'] + for v in sts['spec']['template']['spec']['volumes'] if 'persistentVolumeClaim' in v] + assert 'myapp-data' in claims + + +def test_statefulset_leader_service(tmp_path): + out_folder = tmp_path / 'test_statefulset_leader_service' + create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_folder, domain="my.local", + env='withpostgres', local=False, include=["myapp"], exclude=["legacy"]) + + helm_path = out_folder / HELM_CHART_PATH + shutil.rmtree(helm_path / 'charts') + values_path = helm_path / 'values.yaml' + with open(values_path, 'r') as values_file: + values = yaml.safe_load(values_file) + + harness = values['apps']['myapp']['harness'] + dep_name = harness['deployment']['name'] + service_name = harness['service']['name'] + rw_name = f"{service_name}-rw" + + def ingress_paths(manifests): + ingress = find_manifest(manifests, 'Ingress', 'myapp') + return [path for rule in ingress['spec']['rules'] for path in rule['http']['paths']] + + # write methods in uri_role_mapping without statefulset: no leader service, no leader routing + harness['uri_role_mapping'] = harness.get('uri_role_mapping', []) + [ + {'uri': '/api/edit/*', 'methods': ['POST', 'PUT', 'PATCH']}, + {'uri': '/upload', 'methods': ['POST']}, + {'uri': '/api/remove', 'methods': ['DELETE']}, + {'uri': '/readonly', 'methods': ['GET']}, + ] + with open(values_path, 'w') as values_file: + yaml.safe_dump(values, values_file) + + manifests = render_helm_chart(helm_path) + assert not any(m for m in manifests + if m.get('kind') == 'Service' and m.get('metadata', {}).get('name') == rw_name) + assert not any(p for p in ingress_paths(manifests) + if p['backend']['service']['name'] == rw_name) + + harness['deployment']['statefulset'] = True + with open(values_path, 'w') as values_file: + yaml.safe_dump(values, values_file) + + manifests = render_helm_chart(helm_path) + rw_service = find_manifest(manifests, 'Service', rw_name) + assert rw_service['spec']['selector']['app'] == dep_name + assert rw_service['spec']['selector']['statefulset.kubernetes.io/pod-name'] == f"{dep_name}-0" + main_service = find_manifest(manifests, 'Service', service_name) + assert rw_service['spec']['ports'] == main_service['spec']['ports'] + + paths = ingress_paths(manifests) + rw_paths = {p['path']: p for p in paths if p['backend']['service']['name'] == rw_name} + # wildcard uris map to Prefix rules, plain uris to ImplementationSpecific; any write method + # (POST/PUT/PATCH/DELETE) triggers leader routing, while entries without one (the default + # catch-all, /readonly) are not routed to the leader + assert set(rw_paths) == {'/api/edit', '/upload', '/api/remove'} + assert rw_paths['/api/edit']['pathType'] == 'Prefix' + assert rw_paths['/upload']['pathType'] == 'ImplementationSpecific' + # the catch-all still routes to the normal service + assert any(p for p in paths + if p['path'] == '/' and p['backend']['service']['name'] == service_name) + + def test_clear_all_dbconfig_if_nodb(tmp_path): out_folder = tmp_path / 'test_clear_all_dbconfig_if_nodb'