openstack-k8s-operators · pinikomarov · Jul 8, 2026
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -45,6 +45,17 @@ updates:
       - "docker"
     open-pull-requests-limit: 5
 
+  # Docker images in approve-installplan-generic utility (ESO/VSO InstallPlan approver)
+  - package-ecosystem: "docker"
+    directory: "/components/utilities/approve-installplan-generic"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+    labels:
+      - "dependencies"
+      - "docker"
+    open-pull-requests-limit: 5
+
   # Docker images in controlplane post-delete hook
   - package-ecosystem: "docker"
     directory: "/components/argocd/hooks/postDelete/controlplane"

diff --git a/components/secrets/external-secrets-operator/community/subscription.yaml b/components/secrets/external-secrets-operator/community/subscription.yaml
@@ -8,7 +8,7 @@ metadata:
   namespace: openshift-operators
 spec:
   channel: stable
-  installPlanApproval: Automatic
+  installPlanApproval: Manual
   name: external-secrets-operator
   source: community-operators
   sourceNamespace: openshift-marketplace
diff --git a/components/secrets/external-secrets-operator/redhat/kustomization.yaml b/components/secrets/external-secrets-operator/redhat/kustomization.yaml
@@ -6,12 +6,38 @@ resources:
   - operatorgroup.yaml
   - external-secrets-namespace.yaml
   - vault-egress-netpol.yaml
-  - externalsecretsconfig.yaml
+  # externalsecretsconfig.yaml is intentionally excluded from this component.
+  # The CRD (externalsecretsconfigs.operator.openshift.io) is created by the operator
+  # CSV during the operator-dependencies sync. ArgoCD's sync-time REST mapper does not
+  # know the type exists, so applying it here would fail with "no matches for kind".
+  # Instead it is managed by a dedicated eso-config ArgoCD Application that starts a
+  # fresh sync (fresh REST mapper) only AFTER operator-dependencies is Synced+Healthy.
+# approve-installplan must run after the Subscription exists (sync-wave ordering).
 components:
   - ../community
+  - ../../../utilities/approve-installplan-generic
 patches:
+  - target:
+      kind: Subscription
+      name: external-secrets-operator
+    patch: |-
+      - op: add
+        path: /metadata/annotations/argocd.argoproj.io~1sync-wave
+        value: "0"
   - path: patch-subscription-redhat.json
     target:
       kind: Subscription
       name: external-secrets-operator
       namespace: openshift-operators
+  # The redhat patch moves the ESO subscription to external-secrets-operator namespace,
+  # so the approve Job must poll that namespace instead of openshift-operators.
+  # WEBHOOK_NAMESPACE is intentionally left empty (default): the ESO operand Deployments
+  # only appear after ExternalSecretsConfig is applied by the separate eso-config app
+  # which runs AFTER operator-dependencies. Waiting here would cause a deadlock.
+  - target:
+      kind: Job
+      name: approve-installplan-openshift-operators
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/containers/0/env/0/value
+        value: "external-secrets-operator"
diff --git a/components/secrets/external-secrets-operator/redhat/patch-subscription-redhat.json b/components/secrets/external-secrets-operator/redhat/patch-subscription-redhat.json
@@ -4,5 +4,6 @@
   {"op": "remove", "path": "/metadata/labels"},
   {"op": "replace", "path": "/spec/channel", "value": "stable-v1"},
   {"op": "replace", "path": "/spec/name", "value": "openshift-external-secrets-operator"},
-  {"op": "replace", "path": "/spec/source", "value": "redhat-operators"}
+  {"op": "replace", "path": "/spec/source", "value": "redhat-operators"},
+  {"op": "add", "path": "/spec/startingCSV", "value": "openshift-external-secrets-operator.v1.1.0"}
 ]
diff --git a/components/secrets/vault-secrets-operator/kustomization.yaml b/components/secrets/vault-secrets-operator/kustomization.yaml
@@ -3,3 +3,14 @@ apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
 resources:
   - subscription.yaml
+# approve-installplan must run after the Subscription exists (sync-wave ordering).
+components:
+  - ../../utilities/approve-installplan-generic
+patches:
+  - target:
+      kind: Subscription
+      name: vault-secrets-operator
+    patch: |-
+      - op: add
+        path: /metadata/annotations/argocd.argoproj.io~1sync-wave
+        value: "0"
diff --git a/components/secrets/vault-secrets-operator/subscription.yaml b/components/secrets/vault-secrets-operator/subscription.yaml
@@ -8,7 +8,8 @@ metadata:
   namespace: openshift-operators
 spec:
   channel: stable
-  installPlanApproval: Automatic
+  installPlanApproval: Manual
+  startingCSV: vault-secrets-operator.v0.10.0
   name: vault-secrets-operator
   source: certified-operators
   sourceNamespace: openshift-marketplace
diff --git a/components/utilities/approve-installplan-generic/job.yaml b/components/utilities/approve-installplan-generic/job.yaml
@@ -0,0 +1,132 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: approve-installplan-openshift-operators
+  namespace: openshift-gitops
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  backoffLimit: 1
+  template:
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: installplan-approver
+          image: registry.redhat.io/openshift4/ose-tools-rhel9@sha256:17d4647d21d7571c11d6f983400da6544376fe8bde8372776e4779ebd8c6fd53  # :latest as of 2026-07-06
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 200m
+              memory: 256Mi
+            requests:
+              cpu: 50m
+              memory: 64Mi
+          env:
+            - name: NAMESPACE
+              value: "openshift-operators"
+            - name: RETRIES
+              value: "60"
+            - name: DELAY
+              value: "10"
+            - name: DEBUG
+              value: "true"
+            - name: WEBHOOK_NAMESPACE
+              value: ""
+          command:
+            - /bin/bash
+            - -c
+            - |
+              set -eo pipefail
+              if [ "${DEBUG}" = "true" ]; then
+                set -x
+              fi
+              NAMESPACE=${NAMESPACE:-"openshift-operators"}
+              RETRIES=${RETRIES:-60}
+              DELAY=${DELAY:-10}
+
+              log() {
+                echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $@" >&2
+              }
+
+              approve_pending_installplans() {
+                log "Looking for unapproved Manual InstallPlans in namespace ${NAMESPACE}..."
+                for i in $(seq 1 $RETRIES); do
+                  pending=$(oc get installplan -n $NAMESPACE -o json \
+                    | jq -r '.items[] | select(.spec.approval=="Manual" and .spec.approved==false) | .metadata.name')
+                  if [ -n "$pending" ]; then
+                    for plan in $pending; do
+                      log "Approving InstallPlan: $plan"
+                      oc patch installplan "$plan" -n $NAMESPACE --type merge \
+                        -p '{"spec":{"approved":true}}' >&2
+                    done
+                    return 0
+                  fi
+                  # All Manual InstallPlans already approved — nothing left to approve.
+                  already=$(oc get installplan -n $NAMESPACE -o json \
+                    | jq -r '.items[] | select(.spec.approval=="Manual" and .spec.approved==true) | .metadata.name')
+                  if [ -n "$already" ]; then
+                    log "All Manual InstallPlans already approved: $already — skipping."
+                    return 0
+                  fi
+                  log "Attempt $i/$RETRIES: No pending InstallPlans found yet, retrying..."
+                  sleep $DELAY
+                done
+                log "Error: No Manual InstallPlans found to approve within the time limit."
+                exit 1
+              }
+
+              wait_for_installplans_complete() {
+                log "Waiting for all Manual InstallPlans in ${NAMESPACE} to complete..."
+                for i in $(seq 1 $RETRIES); do
+                  incomplete=$(oc get installplan -n $NAMESPACE -o json \
+                    | jq -r '.items[] | select(.spec.approval=="Manual" and .status.phase!="Complete") | .metadata.name')
+                  if [ -z "$incomplete" ]; then
+                    log "All Manual InstallPlans completed successfully."
+                    return 0
+                  fi
+                  log "Attempt $i/$RETRIES: Waiting for InstallPlans: $incomplete"
+                  sleep $DELAY
+                done
+                log "Error: InstallPlans did not complete within the time limit."
+                exit 1
+              }
+
+              wait_for_deployments_available() {
+                if [ -z "${WEBHOOK_NAMESPACE:-}" ]; then
+                  return 0
+                fi
+                log "Waiting for Deployments to appear in ${WEBHOOK_NAMESPACE}..."
+                for i in $(seq 1 $RETRIES); do
+                  count=$(oc get deployment -n $WEBHOOK_NAMESPACE --no-headers 2>/dev/null | wc -l)
+                  if [ "$count" -gt 0 ]; then
+                    log "Found $count Deployment(s) in ${WEBHOOK_NAMESPACE}, waiting for Available..."
+                    oc wait deployment --all \
+                      -n $WEBHOOK_NAMESPACE \
+                      --for=condition=Available \
+                      --timeout=300s >&2
+                    log "All Deployments Available in ${WEBHOOK_NAMESPACE}."
+                    return 0
+                  fi
+                  log "Attempt $i/$RETRIES: No Deployments in ${WEBHOOK_NAMESPACE} yet, retrying..."
+                  sleep $DELAY
+                done
+                log "Error: No Deployments appeared in ${WEBHOOK_NAMESPACE} within the time limit."
+                exit 1
+              }
+
+              log "Starting InstallPlan approval for namespace ${NAMESPACE}"
+              approve_pending_installplans
+              wait_for_installplans_complete
+              wait_for_deployments_available
+              log "Job completed successfully."
+      restartPolicy: Never
+      serviceAccountName: openshift-gitops-argocd-application-controller
diff --git a/components/utilities/approve-installplan-generic/kustomization.yaml b/components/utilities/approve-installplan-generic/kustomization.yaml
@@ -0,0 +1,10 @@
+---
+# Generic InstallPlan approver for operators in openshift-operators namespace.
+# Include this component in any Kustomization that deploys a Subscription with
+# installPlanApproval: Manual into openshift-operators (e.g. ESO, VSO).
+# The Subscription should be in sync-wave "0" so the Job (wave "1") runs after it.
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+resources:
+  - job.yaml
+  - rbac.yaml
diff --git a/components/utilities/approve-installplan-generic/rbac.yaml b/components/utilities/approve-installplan-generic/rbac.yaml
@@ -0,0 +1,37 @@
+---
+# ClusterRole and binding so the OpenShift GitOps Argo CD application controller
+# can approve OLM InstallPlans in openshift-operators for ESO and VSO.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: installplan-approver-openshift-operators-role
+rules:
+  - apiGroups:
+      - operators.coreos.com
+    resources:
+      - installplans
+    verbs:
+      - get
+      - list
+      - patch
+  - apiGroups:
+      - apps
+    resources:
+      - deployments
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: installplan-approver-openshift-operators-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: installplan-approver-openshift-operators-role
+subjects:
+  - kind: ServiceAccount
+    name: openshift-gitops-argocd-application-controller
+    namespace: openshift-gitops
diff --git a/openshift-gitops.deploy/subscribe/subscription.yaml b/openshift-gitops.deploy/subscribe/subscription.yaml
@@ -8,6 +8,10 @@ metadata:
   namespace: openshift-gitops-operator
 spec:
   channel: latest
+  # installPlanApproval is intentionally Automatic: this subscription is applied
+  # by the Ansible bootstrap before ArgoCD exists, so no GitOps Job can approve
+  # a Manual InstallPlan at that stage. Upgrade gating is handled via the
+  # openshift_gitops_deploy_git_ref pin in the automation repo (FIND-010).
   installPlanApproval: Automatic
   name: openshift-gitops-operator
   source: redhat-operators

diff --git a/resources/external-secrets-config/externalsecretsconfig.yaml b/resources/external-secrets-config/externalsecretsconfig.yaml
@@ -0,0 +1,11 @@
+---
+# ExternalSecretsConfig is required by the Red Hat External Secrets Operator (RHESO)
+# to deploy the actual ESO controller pods (external-secrets, webhook, cert-controller).
+# Sync wave and SkipDryRunOnMissingResource are managed via the argocd/annotations
+# component patch, which must be included by the consumer in the ArgoCD Application
+# spec (spec.source.kustomize.components) — not in the base kustomization.
+apiVersion: operator.openshift.io/v1alpha1
+kind: ExternalSecretsConfig
+metadata:
+  name: cluster
+spec: {}
diff --git a/resources/external-secrets-config/kustomization.yaml b/resources/external-secrets-config/kustomization.yaml
@@ -0,0 +1,20 @@
+---
+# Standalone Kustomization for the ExternalSecretsConfig CR.
+# This path is synced by a dedicated 'eso-config' ArgoCD Application that runs
+# AFTER the operator-dependencies app is Synced+Healthy.
+#
+# Why a separate app?
+# The externalsecretsconfigs.operator.openshift.io CRD is registered by the ESO
+# operator CSV during the operator-dependencies sync. ArgoCD's REST mapper is
+# built once at sync start, so it cannot see a CRD that appears mid-sync.
+# A separate sync starts with a fresh REST mapper that knows the CRD exists.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - externalsecretsconfig.yaml
+  # wave 1: ExternalSecretsConfig triggers ESO operator to create webhook Deployments
+  # wave 2: wait-for-webhook Job polls until ESO Deployments are Available
+  # ArgoCD waits for wave 2 before marking eso-config Synced+Healthy, ensuring
+  # openstack-secrets finds ESO webhook endpoints when creating SecretStore resources.
+  - wait-for-webhook.yaml
diff --git a/resources/external-secrets-config/wait-for-webhook.yaml b/resources/external-secrets-config/wait-for-webhook.yaml
@@ -0,0 +1,62 @@
+---
+# Job that waits for ESO webhook Deployments to be Available after
+# ExternalSecretsConfig is applied. ArgoCD waits for this Job to complete
+# (wave 2) before marking eso-config as Synced+Healthy, ensuring the ESO
+# webhook is ready when openstack-secrets tries to create SecretStore resources.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: wait-for-eso-webhook
+  namespace: openshift-gitops
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  backoffLimit: 1
+  template:
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: wait-for-webhook
+          image: registry.redhat.io/openshift4/ose-tools-rhel9@sha256:17d4647d21d7571c11d6f983400da6544376fe8bde8372776e4779ebd8c6fd53  # :latest as of 2026-07-06
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 100m
+              memory: 128Mi
+            requests:
+              cpu: 50m
+              memory: 64Mi
+          command:
+            - /bin/bash
+            - -c
+            - |
+              set -eo pipefail
+              NAMESPACE="external-secrets"
+              RETRIES=60
+              DELAY=5
+              echo "Waiting for Deployments to appear in ${NAMESPACE}..."
+              for i in $(seq 1 $RETRIES); do
+                count=$(oc get deployment -n $NAMESPACE --no-headers 2>/dev/null | wc -l)
+                if [ "$count" -gt 0 ]; then
+                  echo "Found $count Deployment(s), waiting for Available..."
+                  oc wait deployment --all -n $NAMESPACE \
+                    --for=condition=Available --timeout=300s
+                  echo "All ESO webhook Deployments Available."
+                  exit 0
+                fi
+                echo "Attempt $i/$RETRIES: No Deployments yet, retrying in ${DELAY}s..."
+                sleep $DELAY
+              done
+              echo "Error: ESO Deployments did not appear in time."
+              exit 1
+      restartPolicy: Never
+      serviceAccountName: openshift-gitops-argocd-application-controller