From e046542c0426e9254e9bb3992135dad593b3d4da Mon Sep 17 00:00:00 2001 From: pkomarov Date: Wed, 8 Jul 2026 13:24:44 +0300 Subject: [PATCH] security(olm): Manual InstallPlan approval for ESO and VSO (OSPRH-32422 / FIND-010) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch ESO and VSO Subscriptions from Automatic to Manual install-plan approval so OLM no longer silently installs new operator versions without Git-reviewed consent. These operators hold secret-access credentials, making unreviewed upgrades a K07 (Vulnerable/Unverified Components) risk. - components/secrets/vault-secrets-operator/subscription.yaml - components/secrets/external-secrets-operator/community/subscription.yaml The openshift-gitops-operator subscription is intentionally left as Automatic: it is applied by the Ansible bootstrap before ArgoCD exists, so no GitOps Job can approve a Manual InstallPlan at that stage. Upgrade gating for that operator is handled via the openshift_gitops_deploy_git_ref pin in the automation repo. Add components/utilities/approve-installplan-generic — a lightweight Kustomize Component (Job + RBAC, sync-wave 1) that approves all pending Manual InstallPlans in openshift-operators and waits for completion. Unlike the openstack-specific approve-installplan component, this one carries no openstack CRD dependency checks. Wire it into: - components/secrets/vault-secrets-operator (subscription → wave 0) - components/secrets/external-secrets-operator/redhat (subscription → wave 0) Add Dependabot docker entry for approve-installplan-generic to track the ose-tools-rhel9 image digest weekly, matching the existing pattern. startingCSV pinning is managed manually via PR when operators are intentionally upgraded — no Renovate/Dependabot datasource exists for OLM OperatorHub CSVs. The PR review is the human gate (K07 compliant). Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .github/dependabot.yml | 11 ++ .../community/subscription.yaml | 2 +- .../redhat/kustomization.yaml | 28 +++- .../redhat/patch-subscription-redhat.json | 3 +- .../vault-secrets-operator/kustomization.yaml | 11 ++ .../vault-secrets-operator/subscription.yaml | 3 +- .../approve-installplan-generic/job.yaml | 132 ++++++++++++++++++ .../kustomization.yaml | 10 ++ .../approve-installplan-generic/rbac.yaml | 37 +++++ .../subscribe/subscription.yaml | 4 + .../externalsecretsconfig.yaml | 11 ++ .../kustomization.yaml | 20 +++ .../wait-for-webhook.yaml | 62 ++++++++ 13 files changed, 330 insertions(+), 4 deletions(-) create mode 100644 components/utilities/approve-installplan-generic/job.yaml create mode 100644 components/utilities/approve-installplan-generic/kustomization.yaml create mode 100644 components/utilities/approve-installplan-generic/rbac.yaml create mode 100644 resources/external-secrets-config/externalsecretsconfig.yaml create mode 100644 resources/external-secrets-config/kustomization.yaml create mode 100644 resources/external-secrets-config/wait-for-webhook.yaml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 82b544c..05498f0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -45,6 +45,17 @@ updates: - "docker" open-pull-requests-limit: 5 + # Docker images in approve-installplan-generic utility (ESO/VSO InstallPlan approver) + - package-ecosystem: "docker" + directory: "/components/utilities/approve-installplan-generic" + schedule: + interval: "weekly" + day: "monday" + labels: + - "dependencies" + - "docker" + open-pull-requests-limit: 5 + # Docker images in controlplane post-delete hook - package-ecosystem: "docker" directory: "/components/argocd/hooks/postDelete/controlplane" diff --git a/components/secrets/external-secrets-operator/community/subscription.yaml b/components/secrets/external-secrets-operator/community/subscription.yaml index 12261f9..1f272d4 100644 --- a/components/secrets/external-secrets-operator/community/subscription.yaml +++ b/components/secrets/external-secrets-operator/community/subscription.yaml @@ -8,7 +8,7 @@ metadata: namespace: openshift-operators spec: channel: stable - installPlanApproval: Automatic + installPlanApproval: Manual name: external-secrets-operator source: community-operators sourceNamespace: openshift-marketplace diff --git a/components/secrets/external-secrets-operator/redhat/kustomization.yaml b/components/secrets/external-secrets-operator/redhat/kustomization.yaml index 58eed40..c3de610 100644 --- a/components/secrets/external-secrets-operator/redhat/kustomization.yaml +++ b/components/secrets/external-secrets-operator/redhat/kustomization.yaml @@ -6,12 +6,38 @@ resources: - operatorgroup.yaml - external-secrets-namespace.yaml - vault-egress-netpol.yaml - - externalsecretsconfig.yaml + # externalsecretsconfig.yaml is intentionally excluded from this component. + # The CRD (externalsecretsconfigs.operator.openshift.io) is created by the operator + # CSV during the operator-dependencies sync. ArgoCD's sync-time REST mapper does not + # know the type exists, so applying it here would fail with "no matches for kind". + # Instead it is managed by a dedicated eso-config ArgoCD Application that starts a + # fresh sync (fresh REST mapper) only AFTER operator-dependencies is Synced+Healthy. +# approve-installplan must run after the Subscription exists (sync-wave ordering). components: - ../community + - ../../../utilities/approve-installplan-generic patches: + - target: + kind: Subscription + name: external-secrets-operator + patch: |- + - op: add + path: /metadata/annotations/argocd.argoproj.io~1sync-wave + value: "0" - path: patch-subscription-redhat.json target: kind: Subscription name: external-secrets-operator namespace: openshift-operators + # The redhat patch moves the ESO subscription to external-secrets-operator namespace, + # so the approve Job must poll that namespace instead of openshift-operators. + # WEBHOOK_NAMESPACE is intentionally left empty (default): the ESO operand Deployments + # only appear after ExternalSecretsConfig is applied by the separate eso-config app + # which runs AFTER operator-dependencies. Waiting here would cause a deadlock. + - target: + kind: Job + name: approve-installplan-openshift-operators + patch: |- + - op: replace + path: /spec/template/spec/containers/0/env/0/value + value: "external-secrets-operator" diff --git a/components/secrets/external-secrets-operator/redhat/patch-subscription-redhat.json b/components/secrets/external-secrets-operator/redhat/patch-subscription-redhat.json index cae9587..0f1ecc9 100644 --- a/components/secrets/external-secrets-operator/redhat/patch-subscription-redhat.json +++ b/components/secrets/external-secrets-operator/redhat/patch-subscription-redhat.json @@ -4,5 +4,6 @@ {"op": "remove", "path": "/metadata/labels"}, {"op": "replace", "path": "/spec/channel", "value": "stable-v1"}, {"op": "replace", "path": "/spec/name", "value": "openshift-external-secrets-operator"}, - {"op": "replace", "path": "/spec/source", "value": "redhat-operators"} + {"op": "replace", "path": "/spec/source", "value": "redhat-operators"}, + {"op": "add", "path": "/spec/startingCSV", "value": "openshift-external-secrets-operator.v1.1.0"} ] diff --git a/components/secrets/vault-secrets-operator/kustomization.yaml b/components/secrets/vault-secrets-operator/kustomization.yaml index f9f33bb..f1638d7 100644 --- a/components/secrets/vault-secrets-operator/kustomization.yaml +++ b/components/secrets/vault-secrets-operator/kustomization.yaml @@ -3,3 +3,14 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component resources: - subscription.yaml +# approve-installplan must run after the Subscription exists (sync-wave ordering). +components: + - ../../utilities/approve-installplan-generic +patches: + - target: + kind: Subscription + name: vault-secrets-operator + patch: |- + - op: add + path: /metadata/annotations/argocd.argoproj.io~1sync-wave + value: "0" diff --git a/components/secrets/vault-secrets-operator/subscription.yaml b/components/secrets/vault-secrets-operator/subscription.yaml index 3364647..e77d463 100644 --- a/components/secrets/vault-secrets-operator/subscription.yaml +++ b/components/secrets/vault-secrets-operator/subscription.yaml @@ -8,7 +8,8 @@ metadata: namespace: openshift-operators spec: channel: stable - installPlanApproval: Automatic + installPlanApproval: Manual + startingCSV: vault-secrets-operator.v0.10.0 name: vault-secrets-operator source: certified-operators sourceNamespace: openshift-marketplace diff --git a/components/utilities/approve-installplan-generic/job.yaml b/components/utilities/approve-installplan-generic/job.yaml new file mode 100644 index 0000000..62168df --- /dev/null +++ b/components/utilities/approve-installplan-generic/job.yaml @@ -0,0 +1,132 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: approve-installplan-openshift-operators + namespace: openshift-gitops + annotations: + argocd.argoproj.io/sync-wave: "1" +spec: + backoffLimit: 1 + template: + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: installplan-approver + image: registry.redhat.io/openshift4/ose-tools-rhel9@sha256:17d4647d21d7571c11d6f983400da6544376fe8bde8372776e4779ebd8c6fd53 # :latest as of 2026-07-06 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 50m + memory: 64Mi + env: + - name: NAMESPACE + value: "openshift-operators" + - name: RETRIES + value: "60" + - name: DELAY + value: "10" + - name: DEBUG + value: "true" + - name: WEBHOOK_NAMESPACE + value: "" + command: + - /bin/bash + - -c + - | + set -eo pipefail + if [ "${DEBUG}" = "true" ]; then + set -x + fi + NAMESPACE=${NAMESPACE:-"openshift-operators"} + RETRIES=${RETRIES:-60} + DELAY=${DELAY:-10} + + log() { + echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $@" >&2 + } + + approve_pending_installplans() { + log "Looking for unapproved Manual InstallPlans in namespace ${NAMESPACE}..." + for i in $(seq 1 $RETRIES); do + pending=$(oc get installplan -n $NAMESPACE -o json \ + | jq -r '.items[] | select(.spec.approval=="Manual" and .spec.approved==false) | .metadata.name') + if [ -n "$pending" ]; then + for plan in $pending; do + log "Approving InstallPlan: $plan" + oc patch installplan "$plan" -n $NAMESPACE --type merge \ + -p '{"spec":{"approved":true}}' >&2 + done + return 0 + fi + # All Manual InstallPlans already approved — nothing left to approve. + already=$(oc get installplan -n $NAMESPACE -o json \ + | jq -r '.items[] | select(.spec.approval=="Manual" and .spec.approved==true) | .metadata.name') + if [ -n "$already" ]; then + log "All Manual InstallPlans already approved: $already — skipping." + return 0 + fi + log "Attempt $i/$RETRIES: No pending InstallPlans found yet, retrying..." + sleep $DELAY + done + log "Error: No Manual InstallPlans found to approve within the time limit." + exit 1 + } + + wait_for_installplans_complete() { + log "Waiting for all Manual InstallPlans in ${NAMESPACE} to complete..." + for i in $(seq 1 $RETRIES); do + incomplete=$(oc get installplan -n $NAMESPACE -o json \ + | jq -r '.items[] | select(.spec.approval=="Manual" and .status.phase!="Complete") | .metadata.name') + if [ -z "$incomplete" ]; then + log "All Manual InstallPlans completed successfully." + return 0 + fi + log "Attempt $i/$RETRIES: Waiting for InstallPlans: $incomplete" + sleep $DELAY + done + log "Error: InstallPlans did not complete within the time limit." + exit 1 + } + + wait_for_deployments_available() { + if [ -z "${WEBHOOK_NAMESPACE:-}" ]; then + return 0 + fi + log "Waiting for Deployments to appear in ${WEBHOOK_NAMESPACE}..." + for i in $(seq 1 $RETRIES); do + count=$(oc get deployment -n $WEBHOOK_NAMESPACE --no-headers 2>/dev/null | wc -l) + if [ "$count" -gt 0 ]; then + log "Found $count Deployment(s) in ${WEBHOOK_NAMESPACE}, waiting for Available..." + oc wait deployment --all \ + -n $WEBHOOK_NAMESPACE \ + --for=condition=Available \ + --timeout=300s >&2 + log "All Deployments Available in ${WEBHOOK_NAMESPACE}." + return 0 + fi + log "Attempt $i/$RETRIES: No Deployments in ${WEBHOOK_NAMESPACE} yet, retrying..." + sleep $DELAY + done + log "Error: No Deployments appeared in ${WEBHOOK_NAMESPACE} within the time limit." + exit 1 + } + + log "Starting InstallPlan approval for namespace ${NAMESPACE}" + approve_pending_installplans + wait_for_installplans_complete + wait_for_deployments_available + log "Job completed successfully." + restartPolicy: Never + serviceAccountName: openshift-gitops-argocd-application-controller diff --git a/components/utilities/approve-installplan-generic/kustomization.yaml b/components/utilities/approve-installplan-generic/kustomization.yaml new file mode 100644 index 0000000..7c38a0c --- /dev/null +++ b/components/utilities/approve-installplan-generic/kustomization.yaml @@ -0,0 +1,10 @@ +--- +# Generic InstallPlan approver for operators in openshift-operators namespace. +# Include this component in any Kustomization that deploys a Subscription with +# installPlanApproval: Manual into openshift-operators (e.g. ESO, VSO). +# The Subscription should be in sync-wave "0" so the Job (wave "1") runs after it. +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - job.yaml + - rbac.yaml diff --git a/components/utilities/approve-installplan-generic/rbac.yaml b/components/utilities/approve-installplan-generic/rbac.yaml new file mode 100644 index 0000000..2fef040 --- /dev/null +++ b/components/utilities/approve-installplan-generic/rbac.yaml @@ -0,0 +1,37 @@ +--- +# ClusterRole and binding so the OpenShift GitOps Argo CD application controller +# can approve OLM InstallPlans in openshift-operators for ESO and VSO. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: installplan-approver-openshift-operators-role +rules: + - apiGroups: + - operators.coreos.com + resources: + - installplans + verbs: + - get + - list + - patch + - apiGroups: + - apps + resources: + - deployments + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: installplan-approver-openshift-operators-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: installplan-approver-openshift-operators-role +subjects: + - kind: ServiceAccount + name: openshift-gitops-argocd-application-controller + namespace: openshift-gitops diff --git a/openshift-gitops.deploy/subscribe/subscription.yaml b/openshift-gitops.deploy/subscribe/subscription.yaml index 3f1bc4d..dcb4abf 100644 --- a/openshift-gitops.deploy/subscribe/subscription.yaml +++ b/openshift-gitops.deploy/subscribe/subscription.yaml @@ -8,6 +8,10 @@ metadata: namespace: openshift-gitops-operator spec: channel: latest + # installPlanApproval is intentionally Automatic: this subscription is applied + # by the Ansible bootstrap before ArgoCD exists, so no GitOps Job can approve + # a Manual InstallPlan at that stage. Upgrade gating is handled via the + # openshift_gitops_deploy_git_ref pin in the automation repo (FIND-010). installPlanApproval: Automatic name: openshift-gitops-operator source: redhat-operators diff --git a/resources/external-secrets-config/externalsecretsconfig.yaml b/resources/external-secrets-config/externalsecretsconfig.yaml new file mode 100644 index 0000000..257fdc4 --- /dev/null +++ b/resources/external-secrets-config/externalsecretsconfig.yaml @@ -0,0 +1,11 @@ +--- +# ExternalSecretsConfig is required by the Red Hat External Secrets Operator (RHESO) +# to deploy the actual ESO controller pods (external-secrets, webhook, cert-controller). +# Sync wave and SkipDryRunOnMissingResource are managed via the argocd/annotations +# component patch, which must be included by the consumer in the ArgoCD Application +# spec (spec.source.kustomize.components) — not in the base kustomization. +apiVersion: operator.openshift.io/v1alpha1 +kind: ExternalSecretsConfig +metadata: + name: cluster +spec: {} diff --git a/resources/external-secrets-config/kustomization.yaml b/resources/external-secrets-config/kustomization.yaml new file mode 100644 index 0000000..1c66542 --- /dev/null +++ b/resources/external-secrets-config/kustomization.yaml @@ -0,0 +1,20 @@ +--- +# Standalone Kustomization for the ExternalSecretsConfig CR. +# This path is synced by a dedicated 'eso-config' ArgoCD Application that runs +# AFTER the operator-dependencies app is Synced+Healthy. +# +# Why a separate app? +# The externalsecretsconfigs.operator.openshift.io CRD is registered by the ESO +# operator CSV during the operator-dependencies sync. ArgoCD's REST mapper is +# built once at sync start, so it cannot see a CRD that appears mid-sync. +# A separate sync starts with a fresh REST mapper that knows the CRD exists. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - externalsecretsconfig.yaml + # wave 1: ExternalSecretsConfig triggers ESO operator to create webhook Deployments + # wave 2: wait-for-webhook Job polls until ESO Deployments are Available + # ArgoCD waits for wave 2 before marking eso-config Synced+Healthy, ensuring + # openstack-secrets finds ESO webhook endpoints when creating SecretStore resources. + - wait-for-webhook.yaml diff --git a/resources/external-secrets-config/wait-for-webhook.yaml b/resources/external-secrets-config/wait-for-webhook.yaml new file mode 100644 index 0000000..aa3f4e0 --- /dev/null +++ b/resources/external-secrets-config/wait-for-webhook.yaml @@ -0,0 +1,62 @@ +--- +# Job that waits for ESO webhook Deployments to be Available after +# ExternalSecretsConfig is applied. ArgoCD waits for this Job to complete +# (wave 2) before marking eso-config as Synced+Healthy, ensuring the ESO +# webhook is ready when openstack-secrets tries to create SecretStore resources. +apiVersion: batch/v1 +kind: Job +metadata: + name: wait-for-eso-webhook + namespace: openshift-gitops + annotations: + argocd.argoproj.io/sync-wave: "2" + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + backoffLimit: 1 + template: + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: wait-for-webhook + image: registry.redhat.io/openshift4/ose-tools-rhel9@sha256:17d4647d21d7571c11d6f983400da6544376fe8bde8372776e4779ebd8c6fd53 # :latest as of 2026-07-06 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 50m + memory: 64Mi + command: + - /bin/bash + - -c + - | + set -eo pipefail + NAMESPACE="external-secrets" + RETRIES=60 + DELAY=5 + echo "Waiting for Deployments to appear in ${NAMESPACE}..." + for i in $(seq 1 $RETRIES); do + count=$(oc get deployment -n $NAMESPACE --no-headers 2>/dev/null | wc -l) + if [ "$count" -gt 0 ]; then + echo "Found $count Deployment(s), waiting for Available..." + oc wait deployment --all -n $NAMESPACE \ + --for=condition=Available --timeout=300s + echo "All ESO webhook Deployments Available." + exit 0 + fi + echo "Attempt $i/$RETRIES: No Deployments yet, retrying in ${DELAY}s..." + sleep $DELAY + done + echo "Error: ESO Deployments did not appear in time." + exit 1 + restartPolicy: Never + serviceAccountName: openshift-gitops-argocd-application-controller