diff --git a/.github/workflows/xks-push-ci-automation.yaml b/.github/workflows/xks-push-ci-automation.yaml new file mode 100644 index 00000000000..24e11338b33 --- /dev/null +++ b/.github/workflows/xks-push-ci-automation.yaml @@ -0,0 +1,215 @@ +name: Build, Deploy and Test on EKS + +on: + push: + branches: + - "master" + - "v*" + +env: + IMG: gitops-operator:test + CLUSTER_NAME: gitops-ci-${{ github.run_id }} + AWS_REGION: us-east-1 + +jobs: + ci-build: + name: Build image, deploy to EKS cluster + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: 'go.mod' + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Install eksctl + run: | + ARCH=amd64 + PLATFORM=$(uname -s)_$ARCH + curl -sLO "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_$PLATFORM.tar.gz" + tar -xzf eksctl_$PLATFORM.tar.gz -C /tmp + sudo mv /tmp/eksctl /usr/local/bin + + - name: Create EKS cluster + run: | + hack/scripts/eks-cluster.sh create "${{ env.CLUSTER_NAME }}" "${{ env.AWS_REGION }}" + + - name: Login to Amazon ECR + id: ecr-login + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2 + + - name: Create ECR repository + run: | + aws ecr create-repository \ + --repository-name gitops-operator \ + --region ${{ env.AWS_REGION }} 2>/dev/null || true + + - name: Build and push image to ECR + env: + ECR_REGISTRY: ${{ steps.ecr-login.outputs.registry }} + run: | + ECR_IMG="${ECR_REGISTRY}/gitops-operator:${{ github.sha }}" + make docker-build IMG="${ECR_IMG}" + docker push "${ECR_IMG}" + echo "ECR_IMG=${ECR_IMG}" >> "$GITHUB_ENV" + + # TODO: check if porting is required for non-OCP clusters + - name: Disable webhook and conversion for non-OCP cluster + run: | + sed -i 's|^- ../prometheus|#- ../prometheus|' config/default/kustomization.yaml + sed -i 's|^- ../webhook|#- ../webhook|' config/default/kustomization.yaml + sed -i 's|^- patches/webhook_in_argocds.yaml|#- patches/webhook_in_argocds.yaml|' config/crd/kustomization.yaml + sed -i 's|^- patches/cainjection_in_argocds.yaml|#- patches/cainjection_in_argocds.yaml|' config/crd/kustomization.yaml + echo "=== Verify sed applied ===" + grep -n 'webhook\|prometheus' config/default/kustomization.yaml + grep -n 'patches/' config/crd/kustomization.yaml + + - name: Install CRDs + run: | + make install + + - name: Deploy operator + run: | + make deploy IMG="${ECR_IMG}" + + - name: Verify Controller Manager deployment is available + run: | + kubectl get deployment -n openshift-gitops-operator + kubectl describe deployment -n openshift-gitops-operator + kubectl wait --for=condition=available --timeout=300s \ + deployment/openshift-gitops-operator-controller-manager \ + -n openshift-gitops-operator + + - name: Create ArgoCD instance + run: | + kubectl create ns test-argocd + kubectl apply -f - <<'EOF' + apiVersion: argoproj.io/v1beta1 + kind: ArgoCD + metadata: + name: argocd + namespace: test-argocd + EOF + + - name: Wait for ArgoCD component pods to exist + run: | + EXPECTED_LABELS=("argocd-application-controller" "argocd-redis" "argocd-repo-server" "argocd-server") + TIMEOUT=300 + INTERVAL=10 + ELAPSED=0 + + echo "Waiting for ArgoCD component pods to exist in test-argocd..." + while true; do + ALL_EXIST=true + for label in "${EXPECTED_LABELS[@]}"; do + if ! kubectl get pod -n test-argocd -l "app.kubernetes.io/name=${label}" --no-headers 2>/dev/null | grep -q .; then + ALL_EXIST=false + break + fi + done + + if $ALL_EXIST; then + echo "All ArgoCD component pods exist after ${ELAPSED}s." + break + fi + + if [ $ELAPSED -ge $TIMEOUT ]; then + echo "Timed out after ${TIMEOUT}s waiting for ArgoCD pods." + kubectl get pods -n test-argocd + kubectl get argocd -n test-argocd -o yaml + exit 1 + fi + + sleep $INTERVAL + ELAPSED=$((ELAPSED + INTERVAL)) + done + + - name: Verify ArgoCD components are ready + run: | + kubectl get pods -n test-argocd + kubectl wait --for=condition=Ready -n test-argocd pod --timeout=300s \ + -l 'app.kubernetes.io/name in (argocd-application-controller,argocd-redis,argocd-repo-server,argocd-server)' + echo "All ArgoCD components are ready." + kubectl get pods -n test-argocd + + - name: Run e2e tests + run: | + make e2e-tests-parallel-ginkgo + + - name: Run non-default e2e tests + run: | + make e2e-tests-sequential-ginkgo + + - name: Collect e2e debug info on failure + if: failure() + run: | + echo "=== All pods across namespaces ===" + kubectl get pods -A -o wide || true + echo "" + echo "=== ArgoCD instances ===" + kubectl get argocds -A -o yaml 2>/dev/null || true + echo "" + echo "=== GitopsService ===" + kubectl get gitopsservices -A -o yaml 2>/dev/null || true + + - name: Collect operator debug info on failure + if: failure() + run: | + echo "=== Deployment status ===" + kubectl get deployment -n openshift-gitops-operator -o wide || true + echo "" + echo "=== Pod status ===" + kubectl get pods -n openshift-gitops-operator -o wide || true + echo "" + echo "=== Pod descriptions ===" + kubectl describe pods -n openshift-gitops-operator || true + echo "" + echo "=== Controller manager logs ===" + kubectl logs deployment/openshift-gitops-operator-controller-manager \ + -n openshift-gitops-operator --all-containers=true --tail=200 || true + echo "" + echo "=== Events in operator namespace ===" + kubectl get events -n openshift-gitops-operator --sort-by='.lastTimestamp' || true + echo "" + echo "=== CRD conversion config ===" + kubectl get crd argocds.argoproj.io -o jsonpath='{.spec.conversion}' || true + echo "" + + - name: Collect ArgoCD debug info on failure + if: failure() + run: | + echo "=== ArgoCD resources ===" + kubectl get argocds -n test-argocd -o yaml 2>/dev/null || true + echo "" + echo "=== Pods in test-argocd ===" + kubectl get pods -n test-argocd -o wide 2>/dev/null || true + echo "" + echo "=== Pod descriptions in test-argocd ===" + kubectl describe pods -n test-argocd 2>/dev/null || true + echo "" + echo "=== Events in test-argocd ===" + kubectl get events -n test-argocd --sort-by='.lastTimestamp' 2>/dev/null || true + + - name: Delete EKS cluster + if: always() + run: | + FORCE=true hack/scripts/eks-cluster.sh delete "${{ env.CLUSTER_NAME }}" "${{ env.AWS_REGION }}" + + - name: Delete ECR repository + if: always() + run: | + aws ecr delete-repository \ + --repository-name gitops-operator \ + --region ${{ env.AWS_REGION }} \ + --force 2>/dev/null || true diff --git a/hack/scripts/eks-cluster.sh b/hack/scripts/eks-cluster.sh new file mode 100755 index 00000000000..2a02e2701b1 --- /dev/null +++ b/hack/scripts/eks-cluster.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# +# Manage an AWS EKS cluster (3 nodes) suitable for Argo CD HA. +# +# Usage: +# ./eks-cluster.sh create [CLUSTER_NAME] [REGION] +# ./eks-cluster.sh delete [CLUSTER_NAME] [REGION] +# ./eks-cluster.sh status [CLUSTER_NAME] [REGION] +# +# Defaults: +# CLUSTER_NAME = argocd-ha +# REGION = us-east-1 + +set -euo pipefail + +usage() { + echo "Usage: $0 {create|delete|status} [CLUSTER_NAME] [REGION]" + exit 1 +} + +[[ $# -ge 1 ]] || usage + +ACTION="$1"; shift +CLUSTER_NAME="${1:-argocd-ha}" +REGION="${2:-us-east-1}" + +K8S_VERSION="1.30" +NODE_TYPE="m5.xlarge" +NODE_COUNT=3 +NODE_VOLUME_SIZE=100 + +required_cmds=(aws eksctl) +[[ "${ACTION}" == "create" || "${ACTION}" == "status" ]] && required_cmds+=(kubectl) +for cmd in "${required_cmds[@]}"; do + if ! command -v "${cmd}" &>/dev/null; then + echo "ERROR: ${cmd} not found in PATH" >&2 + exit 1 + fi +done + +cmd_create() { + echo "==> Creating EKS cluster: ${CLUSTER_NAME} in ${REGION}" + echo " Kubernetes ${K8S_VERSION}, ${NODE_COUNT}x ${NODE_TYPE}, ${NODE_VOLUME_SIZE}GiB volumes" + + eksctl create cluster \ + --name "${CLUSTER_NAME}" \ + --region "${REGION}" \ + --version "${K8S_VERSION}" \ + --nodegroup-name "argocd-workers" \ + --node-type "${NODE_TYPE}" \ + --nodes "${NODE_COUNT}" \ + --nodes-min "${NODE_COUNT}" \ + --nodes-max "${NODE_COUNT}" \ + --node-volume-size "${NODE_VOLUME_SIZE}" \ + --managed \ + --with-oidc \ + --ssh-access=false \ + --asg-access + + echo "==> Cluster ready" + kubectl get nodes -o wide +} + +cmd_delete() { + echo "==> This will DELETE cluster '${CLUSTER_NAME}' in ${REGION} and all associated resources" + if [[ "${FORCE:-false}" != "true" ]]; then + read -rp " Continue? [y/N] " confirm + [[ "${confirm}" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; } + fi + + echo "==> Deleting EKS cluster: ${CLUSTER_NAME}" + eksctl delete cluster \ + --name "${CLUSTER_NAME}" \ + --region "${REGION}" \ + --wait + + echo "==> Cleanup complete" +} + +cmd_status() { + echo "==> Cluster: ${CLUSTER_NAME} (${REGION})" + if eksctl get cluster --name "${CLUSTER_NAME}" --region "${REGION}" 2>/dev/null; then + echo "" + echo "==> Nodes:" + kubectl get nodes -o wide 2>/dev/null || echo " (kubectl not configured for this cluster)" + else + echo " Cluster not found" + fi +} + +case "${ACTION}" in + create) cmd_create ;; + delete) cmd_delete ;; + status) cmd_status ;; + *) usage ;; +esac