From e6227bf8bd3441e7c4170a482ce03a2528759f75 Mon Sep 17 00:00:00 2001 From: Joseph Antony Vaikath Date: Thu, 31 Jul 2025 13:21:32 -0400 Subject: [PATCH 1/6] Add e2e backup and restore test suite for oadp-cli (#1835) * start cli e2e work * First pass done * Install oadp-cli in CI image * Fix linting * Refactor code * Add kubectl * Add TEST_CLI option to Makefile and update testing documentation - Introduced TEST_CLI variable to control CLI-based backup/restore testing in the Makefile. - Updated TESTING.md to include information about the new TEST_CLI environment variable. - Added 'cli' label to the CLI test suite for better categorization. * Manual oadp-cli build and mv * Fail cli e2e if oadp not found * Add logging before cli commands execute * fix * Tabs not spaces --- Makefile | 7 + build/ci-Dockerfile | 22 +- docs/developer/testing/TESTING.md | 3 + tests/e2e/backup_restore_cli_suite_test.go | 377 +++++++++++++++++++++ tests/e2e/lib/backup_cli.go | 261 ++++++++++++++ tests/e2e/lib/cli_common.go | 58 ++++ tests/e2e/lib/restore_cli.go | 263 ++++++++++++++ 7 files changed, 987 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/backup_restore_cli_suite_test.go create mode 100644 tests/e2e/lib/backup_cli.go create mode 100644 tests/e2e/lib/cli_common.go create mode 100644 tests/e2e/lib/restore_cli.go diff --git a/Makefile b/Makefile index 5f2eefff984..8d3dabb8d18 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,8 @@ OC_CLI = $(shell which oc) TEST_VIRT ?= false TEST_HCP ?= false TEST_UPGRADE ?= false +TEST_CLI ?= false +SKIP_MUST_GATHER ?= false # TOOL VERSIONS # All version-related variables are defined here for easy maintenance @@ -615,6 +617,11 @@ ifeq ($(TEST_HCP),true) else TEST_FILTER += && (! hcp) endif +ifeq ($(TEST_CLI),true) + TEST_FILTER += && (cli) +else + TEST_FILTER += && (! cli) +endif SETTINGS_TMP=/tmp/test-settings .PHONY: test-e2e-setup diff --git a/build/ci-Dockerfile b/build/ci-Dockerfile index 8208510d941..9339479d7e6 100644 --- a/build/ci-Dockerfile +++ b/build/ci-Dockerfile @@ -6,7 +6,21 @@ WORKDIR /go/src/github.com/openshift/oadp-operator COPY ./ . RUN go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo -RUN go mod download -RUN chmod -R 777 ./ -RUN chmod -R 777 $(go env GOPATH) -RUN mkdir -p $(go env GOCACHE) && chmod -R 777 $(go env GOCACHE) + +# Clone and install oadp-cli pinned to oadp-1.4 +RUN git clone --depth=1 --branch oadp-1.4 https://github.com/migtools/oadp-cli.git /tmp/oadp-cli && \ + cd /tmp/oadp-cli && \ + make build && \ + cp kubectl-oadp /usr/local/bin/ && \ + chmod +x /usr/local/bin/kubectl-oadp && \ + rm -rf /tmp/oadp-cli + +# Install kubectl (multi-arch) +RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') && \ + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl" && \ + chmod +x kubectl && \ + mv kubectl /usr/local/bin/ + +RUN go mod download && \ + mkdir -p $(go env GOCACHE) && \ + chmod -R 777 ./ $(go env GOCACHE) $(go env GOPATH) diff --git a/docs/developer/testing/TESTING.md b/docs/developer/testing/TESTING.md index 649f62751b9..fbf529ac9b0 100644 --- a/docs/developer/testing/TESTING.md +++ b/docs/developer/testing/TESTING.md @@ -25,6 +25,9 @@ To get started, you need to provide the following **required** environment varia | `MUST_GATHER_IMAGE` | Container image to use for must-gather collection via `oc adm must-gather` | `quay.io/konveyor/oadp-must-gather:oadp-1.4` | false | | `MUST_GATHER_REPO` | GitHub repo (e.g., `openshift/oadp-must-gather`) to build must-gather from source. Sets `MUST_GATHER_IMAGE` to a ttl.sh image automatically | - | false | | `MUST_GATHER_BRANCH` | Branch to use when building from `MUST_GATHER_REPO` | `oadp-1.4` | false | +| `TEST_UPGRADE` | Exclusively run upgrade tests. Need to first run `make catalog-test-upgrade`, if testing non production operator | `false` | false | +| `TEST_CLI` | Exclusively run CLI-based backup/restore testing | `false` | false | +| `SKIP_MUST_GATHER` | must-gather is compiled locally in the Makefile, may cause issue if local and cluster arch do not match| `false` | false | The expected format for `OADP_CRED_FILE` and `CI_CRED_FILE` files is: ``` diff --git a/tests/e2e/backup_restore_cli_suite_test.go b/tests/e2e/backup_restore_cli_suite_test.go new file mode 100644 index 00000000000..b3bfb01f642 --- /dev/null +++ b/tests/e2e/backup_restore_cli_suite_test.go @@ -0,0 +1,377 @@ +package e2e_test + +import ( + "fmt" + "log" + "os/exec" + "strings" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + "github.com/openshift/oadp-operator/tests/e2e/lib" +) + +// CLI-specific backup execution +func runBackupViaCLI(brCase BackupRestoreCase, backupName string) bool { + nsRequiresResticDCWorkaround, err := lib.NamespaceRequiresResticDCWorkaround(dpaCR.Client, brCase.Namespace) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + if strings.Contains(brCase.Name, "twovol") { + volumeSyncDelay := 30 * time.Second + log.Printf("Sleeping for %v to allow volume to be in sync with /tmp/log/ for case %s", volumeSyncDelay, brCase.Name) + time.Sleep(volumeSyncDelay) + } + + // Create backup via CLI + log.Printf("Creating backup %s for case %s via CLI", backupName, brCase.Name) + err = lib.CreateBackupForNamespacesViaCLI(backupName, []string{brCase.Namespace}, brCase.BackupRestoreType == lib.RESTIC || brCase.BackupRestoreType == lib.KOPIA, brCase.BackupRestoreType == lib.CSIDataMover) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Wait for backup via CLI + gomega.Eventually(lib.IsBackupDoneViaCLI(backupName), brCase.BackupTimeout, time.Second*10).Should(gomega.BeTrue()) + + // Get backup details via CLI + describeBackup := lib.DescribeBackupViaCLI(backupName) + ginkgo.GinkgoWriter.Println(describeBackup) + + backupLogs, err := lib.BackupLogsViaCLI(backupName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + backupErrorLogs := lib.BackupErrorLogsViaCLI(backupName) + accumulatedTestLogs = append(accumulatedTestLogs, describeBackup, backupLogs) + + if !brCase.SkipVerifyLogs { + gomega.Expect(backupErrorLogs).Should(gomega.Equal([]string{})) + } + + // Check if backup succeeded + succeeded, err := lib.IsBackupCompletedSuccessfullyViaCLI(backupName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(succeeded).To(gomega.Equal(true)) + + log.Printf("Backup for case %s succeeded via CLI", brCase.Name) + + return nsRequiresResticDCWorkaround +} + +// CLI-specific restore execution +func runRestoreViaCLI(brCase BackupRestoreCase, backupName, restoreName string, nsRequiresResticDCWorkaround bool) { + log.Printf("Creating restore %s for case %s via CLI", restoreName, brCase.Name) + err := lib.CreateRestoreFromBackupViaCLI(backupName, restoreName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(lib.IsRestoreDoneViaCLI(restoreName), time.Minute*60, time.Second*10).Should(gomega.BeTrue()) + + // Get restore details via CLI + describeRestore := lib.DescribeRestoreViaCLI(restoreName) + ginkgo.GinkgoWriter.Println(describeRestore) + + restoreLogs, err := lib.RestoreLogsViaCLI(restoreName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + restoreErrorLogs := lib.RestoreErrorLogsViaCLI(restoreName) + accumulatedTestLogs = append(accumulatedTestLogs, describeRestore, restoreLogs) + + if !brCase.SkipVerifyLogs { + gomega.Expect(restoreErrorLogs).Should(gomega.Equal([]string{})) + } + + // Check if restore succeeded + succeeded, err := lib.IsRestoreCompletedSuccessfullyViaCLI(restoreName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(succeeded).To(gomega.Equal(true)) + + if nsRequiresResticDCWorkaround { + log.Printf("Running dc-post-restore.sh script.") + err = lib.RunDcPostRestoreScript(restoreName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } +} + +// CLI-specific application backup and restore +func runApplicationBackupAndRestoreViaCLI(brCase ApplicationBackupRestoreCase, updateLastBRcase func(brCase ApplicationBackupRestoreCase), updateLastInstallTime func()) { + updateLastBRcase(brCase) + + // create DPA (still using K8s client for setup) + backupName, restoreName := prepareBackupAndRestore(brCase.BackupRestoreCase, updateLastInstallTime) + + // Ensure that an existing backup repository is deleted (still using K8s client) + brerr := lib.DeleteBackupRepositories(runTimeClientForSuiteRun, namespace) + gomega.Expect(brerr).ToNot(gomega.HaveOccurred()) + + // install app (still using K8s client for setup) + updateLastInstallTime() + log.Printf("Installing application for case %s", brCase.Name) + err := lib.InstallApplication(dpaCR.Client, brCase.ApplicationTemplate) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + if brCase.BackupRestoreType == lib.CSI || brCase.BackupRestoreType == lib.CSIDataMover { + log.Printf("Creating pvc for case %s", brCase.Name) + var pvcName string + var pvcPath string + + pvcName = provider + if brCase.PvcSuffixName != "" { + pvcName += brCase.PvcSuffixName + } + + pvcPathFormat := "./sample-applications/%s/pvc/%s.yaml" + if strings.Contains(brCase.Name, "twovol") { + pvcPathFormat = "./sample-applications/%s/pvc-twoVol/%s.yaml" + } + + pvcPath = fmt.Sprintf(pvcPathFormat, brCase.Namespace, pvcName) + + err = lib.InstallApplication(dpaCR.Client, pvcPath) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + // Run optional custom verification + if brCase.PreBackupVerify != nil { + log.Printf("Running pre-backup custom function for case %s", brCase.Name) + err := brCase.PreBackupVerify(dpaCR.Client, brCase.Namespace) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + // do the backup via CLI + nsRequiredResticDCWorkaround := runBackupViaCLI(brCase.BackupRestoreCase, backupName) + + // uninstall app (still using K8s client) + log.Printf("Uninstalling app for case %s", brCase.Name) + err = lib.UninstallApplication(dpaCR.Client, brCase.ApplicationTemplate) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Wait for namespace to be deleted + gomega.Eventually(lib.IsNamespaceDeleted(kubernetesClientForSuiteRun, brCase.Namespace), time.Minute*4, time.Second*5).Should(gomega.BeTrue()) + + updateLastInstallTime() + + // run restore via CLI + runRestoreViaCLI(brCase.BackupRestoreCase, backupName, restoreName, nsRequiredResticDCWorkaround) + + // Run optional custom verification + if brCase.PostRestoreVerify != nil { + log.Printf("Running post-restore custom function for case %s", brCase.Name) + err = brCase.PostRestoreVerify(dpaCR.Client, brCase.Namespace) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } +} + +// CLI teardown function +func tearDownBackupAndRestoreViaCLI(brCase BackupRestoreCase, installTime time.Time, report ginkgo.SpecReport) { + log.Println("Post backup and restore state (CLI): ", report.State.String()) + + if report.Failed() { + knownFlake = lib.CheckIfFlakeOccurred(accumulatedTestLogs) + accumulatedTestLogs = nil + getFailedTestLogs(namespace, brCase.Namespace, installTime, report) + } + + if brCase.BackupRestoreType == lib.CSI || brCase.BackupRestoreType == lib.CSIDataMover { + log.Printf("Deleting VolumeSnapshot for CSI backuprestore of %s", brCase.Name) + snapshotClassPath := fmt.Sprintf("./sample-applications/snapclass-csi/%s.yaml", provider) + err := lib.UninstallApplication(dpaCR.Client, snapshotClassPath) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + err := dpaCR.Delete() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + err = lib.DeleteNamespace(kubernetesClientForSuiteRun, brCase.Namespace) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Eventually(lib.IsNamespaceDeleted(kubernetesClientForSuiteRun, brCase.Namespace), time.Minute*5, time.Second*5).Should(gomega.BeTrue()) +} + +// CLI Test Suite +var _ = ginkgo.Describe("Backup and restore tests via OADP CLI", ginkgo.Label("cli"), ginkgo.Ordered, func() { + var lastBRCase ApplicationBackupRestoreCase + var lastInstallTime time.Time + updateLastBRcase := func(brCase ApplicationBackupRestoreCase) { + lastBRCase = brCase + } + updateLastInstallTime := func() { + lastInstallTime = time.Now() + } + + ginkgo.BeforeAll(func() { + // Verify OADP CLI is available (should be installed in Docker image) + log.Print("Verifying OADP CLI is available...") + cmd := exec.Command("kubectl", "oadp", "version") + output, err := cmd.CombinedOutput() + if err != nil { + ginkgo.Fail(fmt.Sprintf("OADP CLI not available: %v, output: %s", err, string(output))) + } + log.Printf("OADP CLI available. Version: %s", string(output)) + }) + + var _ = ginkgo.AfterEach(func(ctx ginkgo.SpecContext) { + tearDownBackupAndRestoreViaCLI(lastBRCase.BackupRestoreCase, lastInstallTime, ctx.SpecReport()) + }) + + var _ = ginkgo.AfterAll(func() { + // Same cleanup as original + waitOADPReadiness(lib.KOPIA) + + log.Printf("Creating real DataProtectionTest before must-gather") + bsls, err := dpaCR.ListBSLs() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + bslName := bsls.Items[0].Name + err = lib.CreateUploadTestOnlyDPT(dpaCR.Client, dpaCR.Namespace, bslName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + log.Printf("skipMustGather: %v", skipMustGather) + if !skipMustGather { + log.Printf("Running OADP must-gather") + err = lib.RunMustGather(artifact_dir, dpaCR.Client) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + err = dpaCR.Delete() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + ginkgo.DescribeTable("Backup and restore applications via OADP CLI", + func(brCase ApplicationBackupRestoreCase, expectedErr error) { + if ginkgo.CurrentSpecReport().NumAttempts > 1 && !knownFlake { + ginkgo.Fail("No known FLAKE found in a previous run, marking test as failed.") + } + runApplicationBackupAndRestoreViaCLI(brCase, updateLastBRcase, updateLastInstallTime) + }, + ginkgo.Entry("MySQL application CSI via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-csi.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mysql-persistent", + Name: "mysql-csi-cli-e2e", + BackupRestoreType: lib.CSI, + PreBackupVerify: todoListReady(true, false, "mysql"), + PostRestoreVerify: todoListReady(false, false, "mysql"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("Mongo application CSI via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent-csi.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mongo-persistent", + Name: "mongo-csi-cli-e2e", + BackupRestoreType: lib.CSI, + PreBackupVerify: todoListReady(true, false, "mongo"), + PostRestoreVerify: todoListReady(false, false, "mongo"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("MySQL application two Vol CSI via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-twovol-csi.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mysql-persistent", + Name: "mysql-twovol-csi-cli-e2e", + BackupRestoreType: lib.CSI, + PreBackupVerify: todoListReady(true, true, "mysql"), + PostRestoreVerify: todoListReady(false, true, "mysql"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("Mongo application RESTIC via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mongo-persistent", + Name: "mongo-restic-cli-e2e", + BackupRestoreType: lib.RESTIC, + PreBackupVerify: todoListReady(true, false, "mongo"), + PostRestoreVerify: todoListReady(false, false, "mongo"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("MySQL application RESTIC via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mysql-persistent", + Name: "mysql-restic-cli-e2e", + BackupRestoreType: lib.RESTIC, + PreBackupVerify: todoListReady(true, false, "mysql"), + PostRestoreVerify: todoListReady(false, false, "mysql"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("Mongo application KOPIA via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mongo-persistent", + Name: "mongo-kopia-cli-e2e", + BackupRestoreType: lib.KOPIA, + PreBackupVerify: todoListReady(true, false, "mongo"), + PostRestoreVerify: todoListReady(false, false, "mongo"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("MySQL application KOPIA via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mysql-persistent", + Name: "mysql-kopia-cli-e2e", + BackupRestoreType: lib.KOPIA, + PreBackupVerify: todoListReady(true, false, "mysql"), + PostRestoreVerify: todoListReady(false, false, "mysql"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("Mongo application DATAMOVER via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent-csi.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mongo-persistent", + Name: "mongo-datamover-cli-e2e", + BackupRestoreType: lib.CSIDataMover, + PreBackupVerify: todoListReady(true, false, "mongo"), + PostRestoreVerify: todoListReady(false, false, "mongo"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("MySQL application DATAMOVER via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-csi.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mysql-persistent", + Name: "mysql-datamover-cli-e2e", + BackupRestoreType: lib.CSIDataMover, + PreBackupVerify: todoListReady(true, false, "mysql"), + PostRestoreVerify: todoListReady(false, false, "mysql"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("Mongo application BlockDevice DATAMOVER via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent-block.yaml", + PvcSuffixName: "-block-mode", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mongo-persistent", + Name: "mongo-blockdevice-cli-e2e", + BackupRestoreType: lib.CSIDataMover, + PreBackupVerify: todoListReady(true, false, "mongo"), + PostRestoreVerify: todoListReady(false, false, "mongo"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("MySQL application Native-Snapshots via CLI", ginkgo.FlakeAttempts(flakeAttempts), ginkgo.Label("aws", "azure", "gcp"), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mysql-persistent", + Name: "mysql-native-snapshots-cli-e2e", + BackupRestoreType: lib.NativeSnapshots, + PreBackupVerify: todoListReady(true, false, "mysql"), + PostRestoreVerify: todoListReady(false, false, "mysql"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("Mongo application Native-Snapshots via CLI", ginkgo.FlakeAttempts(flakeAttempts), ginkgo.Label("aws", "azure", "gcp"), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mongo-persistent", + Name: "mongo-native-snapshots-cli-e2e", + BackupRestoreType: lib.NativeSnapshots, + PreBackupVerify: todoListReady(true, false, "mongo"), + PostRestoreVerify: todoListReady(false, false, "mongo"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ) +}) diff --git a/tests/e2e/lib/backup_cli.go b/tests/e2e/lib/backup_cli.go new file mode 100644 index 00000000000..217470474b0 --- /dev/null +++ b/tests/e2e/lib/backup_cli.go @@ -0,0 +1,261 @@ +package lib + +import ( + "encoding/json" + "fmt" + "log" + "strings" + + velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +// CreateBackupForNamespacesViaCLI creates a backup using the OADP CLI +func CreateBackupForNamespacesViaCLI(backupName string, namespaces []string, defaultVolumesToFsBackup bool, snapshotMoveData bool) error { + var options []string + + // Add included namespaces (comma-separated) + if len(namespaces) > 0 { + options = append(options, "--include-namespaces", strings.Join(namespaces, ",")) + } + + // Add volume backup options + if defaultVolumesToFsBackup { + options = append(options, "--default-volumes-to-fs-backup") + } + + // Add snapshot move data option + if snapshotMoveData { + options = append(options, "--snapshot-move-data") + } + + // Execute CLI command + cmd := &CLICommand{ + Resource: "backup", + Action: "create", + Name: backupName, + Options: options, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Errorf("failed to create backup via CLI: %v, output: %s", err, string(output)) + } + + log.Printf("Backup created via CLI: %s", backupName) + return nil +} + +// CreateCustomBackupForNamespacesViaCLI creates a custom backup using the OADP CLI +func CreateCustomBackupForNamespacesViaCLI(backupName string, namespaces []string, includedResources, excludedResources []string, defaultVolumesToFsBackup bool, snapshotMoveData bool) error { + var options []string + + // Add included namespaces (comma-separated) + if len(namespaces) > 0 { + options = append(options, "--include-namespaces", strings.Join(namespaces, ",")) + } + + // Add included resources + if len(includedResources) > 0 { + options = append(options, "--include-resources", strings.Join(includedResources, ",")) + } + + // Add excluded resources + if len(excludedResources) > 0 { + options = append(options, "--exclude-resources", strings.Join(excludedResources, ",")) + } + + // Add volume backup options + if defaultVolumesToFsBackup { + options = append(options, "--default-volumes-to-fs-backup") + } + + // Add snapshot move data option + if snapshotMoveData { + options = append(options, "--snapshot-move-data") + } + + // Execute CLI command + cmd := &CLICommand{ + Resource: "backup", + Action: "create", + Name: backupName, + Options: options, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Errorf("failed to create custom backup via CLI: %v, output: %s", err, string(output)) + } + + log.Printf("Custom backup created via CLI: %s", backupName) + return nil +} + +// GetBackupViaCLI gets backup details using the OADP CLI +func GetBackupViaCLI(name string) (*velero.Backup, error) { + if name == "" { + return nil, fmt.Errorf("backup name cannot be empty") + } + + // Use CLI to get backup details in JSON format + cmd := &CLICommand{ + Resource: "backup", + Action: "get", + Name: name, + Options: []string{"-o", "json"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return nil, fmt.Errorf("failed to get backup via CLI: %v", err) + } + + // Parse the JSON output back to velero.Backup struct + var backup velero.Backup + if err := json.Unmarshal(output, &backup); err != nil { + return nil, fmt.Errorf("failed to parse backup JSON: %v", err) + } + + return &backup, nil +} + +// IsBackupDoneViaCLI checks if backup is done using the OADP CLI +func IsBackupDoneViaCLI(name string) wait.ConditionFunc { + return func() (bool, error) { + // Use CLI to get backup status + cmd := &CLICommand{ + Resource: "backup", + Action: "get", + Name: name, + Options: []string{"-o", "yaml"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return false, fmt.Errorf("failed to get backup status via CLI: %v", err) + } + + // Parse phase from YAML output + phase := ParsePhaseFromYAML(string(output)) + + if len(phase) > 0 { + log.Printf("backup phase: %s", phase) + } + + var phasesNotDone = []string{ + string(velero.BackupPhaseNew), + string(velero.BackupPhaseInProgress), + string(velero.BackupPhaseWaitingForPluginOperations), + string(velero.BackupPhaseWaitingForPluginOperationsPartiallyFailed), + string(velero.BackupPhaseFinalizing), + string(velero.BackupPhaseFinalizingPartiallyFailed), + "", + } + + for _, notDonePhase := range phasesNotDone { + if phase == notDonePhase { + return false, nil + } + } + return true, nil + } +} + +// IsBackupCompletedSuccessfullyViaCLI checks if backup completed successfully using the OADP CLI +func IsBackupCompletedSuccessfullyViaCLI(name string) (bool, error) { + // Use CLI to get backup status + cmd := &CLICommand{ + Resource: "backup", + Action: "get", + Name: name, + Options: []string{"-o", "yaml"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return false, fmt.Errorf("failed to get backup status via CLI: %v", err) + } + + // Parse phase from YAML output + phase := ParsePhaseFromYAML(string(output)) + + if phase == string(velero.BackupPhaseCompleted) { + return true, nil + } + + // Get additional failure information using CLI + backupLogs, logsErr := BackupLogsViaCLI(name) + if logsErr != nil { + backupLogs = fmt.Sprintf("Failed to get logs: %v", logsErr) + } + + return false, fmt.Errorf( + "backup phase is: %s; expected: %s\nvelero failure logs: %v", + phase, string(velero.BackupPhaseCompleted), backupLogs, + ) +} + +// DescribeBackupViaCLI describes backup using the OADP CLI +func DescribeBackupViaCLI(name string) (backupDescription string) { + // Use CLI to describe backup + cmd := &CLICommand{ + Resource: "backup", + Action: "describe", + Name: name, + Options: []string{"--details"}, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Sprintf("could not describe backup via CLI: %v, output: %s", err, string(output)) + } + + return string(output) +} + +// BackupLogsViaCLI gets backup logs using the OADP CLI +func BackupLogsViaCLI(name string) (backupLogs string, err error) { + if name == "" { + return "", fmt.Errorf("backup name cannot be empty") + } + + // Use CLI to get backup logs + cmd := &CLICommand{ + Resource: "backup", + Action: "logs", + Name: name, + Options: []string{}, + } + output, cmdErr := cmd.ExecuteOutput() + if cmdErr != nil { + return "", fmt.Errorf("failed to get backup logs via CLI: %v", cmdErr) + } + + return string(output), nil +} + +// BackupErrorLogsViaCLI gets backup error logs using the OADP CLI +func BackupErrorLogsViaCLI(name string) []string { + bl, err := BackupLogsViaCLI(name) + if err != nil { + return []string{err.Error()} + } + return errorLogsExcludingIgnored(bl) +} + +// DeleteBackupViaCLI deletes a backup using the OADP CLI +func DeleteBackupViaCLI(name string) error { + if name == "" { + return fmt.Errorf("backup name cannot be empty") + } + + // Use CLI to delete backup + cmd := &CLICommand{ + Resource: "backup", + Action: "delete", + Name: name, + Options: []string{}, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Errorf("failed to delete backup via CLI: %v, output: %s", err, string(output)) + } + + log.Printf("Backup deleted via CLI: %s", name) + return nil +} diff --git a/tests/e2e/lib/cli_common.go b/tests/e2e/lib/cli_common.go new file mode 100644 index 00000000000..0ececbaf7b7 --- /dev/null +++ b/tests/e2e/lib/cli_common.go @@ -0,0 +1,58 @@ +package lib + +import ( + "log" + "os/exec" + "strings" +) + +type CLICommand struct { + Resource string // "backup" or "restore" + Action string // "create", "get", "delete", etc. + Name string + Options []string +} + +func (c *CLICommand) Execute() ([]byte, error) { + args := []string{"oadp", c.Resource, c.Action} + if c.Name != "" { + args = append(args, c.Name) + } + args = append(args, c.Options...) + + c.LogCLICommand() + cmd := exec.Command("kubectl", args...) + return cmd.CombinedOutput() +} + +func (c *CLICommand) ExecuteOutput() ([]byte, error) { + args := []string{"oadp", c.Resource, c.Action} + if c.Name != "" { + args = append(args, c.Name) + } + args = append(args, c.Options...) + + c.LogCLICommand() + cmd := exec.Command("kubectl", args...) + return cmd.Output() +} + +func (c *CLICommand) LogCLICommand() { + args := []string{"kubectl", "oadp", c.Resource, c.Action} + if c.Name != "" { + args = append(args, c.Name) + } + args = append(args, c.Options...) + log.Printf("Executing CLI command: %s", strings.Join(args, " ")) +} + +func ParsePhaseFromYAML(yamlOutput string) string { + lines := strings.Split(yamlOutput, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "phase:") { + return strings.TrimSpace(strings.TrimPrefix(line, "phase:")) + } + } + return "" +} diff --git a/tests/e2e/lib/restore_cli.go b/tests/e2e/lib/restore_cli.go new file mode 100644 index 00000000000..ae70afdc793 --- /dev/null +++ b/tests/e2e/lib/restore_cli.go @@ -0,0 +1,263 @@ +package lib + +import ( + "encoding/json" + "fmt" + "log" + "strings" + + velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +// CreateRestoreFromBackupViaCLI creates a restore using the OADP CLI +func CreateRestoreFromBackupViaCLI(backupName, restoreName string) error { + options := []string{"--from-backup", backupName} + + // Execute CLI command + cmd := &CLICommand{ + Resource: "restore", + Action: "create", + Name: restoreName, + Options: options, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Errorf("failed to create restore via CLI: %v, output: %s", err, string(output)) + } + + log.Printf("Restore created via CLI: %s", restoreName) + return nil +} + +// CreateRestoreFromBackupWithOptionsViaCLI creates a restore with options using the OADP CLI +func CreateRestoreFromBackupWithOptionsViaCLI(backupName, restoreName string, restoreVolumes bool, includeNamespaces, excludeNamespaces []string) error { + options := []string{"--from-backup", backupName} + + // Add restore volumes option + if restoreVolumes { + options = append(options, "--restore-volumes") + } + + // Add included namespaces + if len(includeNamespaces) > 0 { + options = append(options, "--include-namespaces", strings.Join(includeNamespaces, ",")) + } + + // Add excluded namespaces + if len(excludeNamespaces) > 0 { + options = append(options, "--exclude-namespaces", strings.Join(excludeNamespaces, ",")) + } + + // Execute CLI command + cmd := &CLICommand{ + Resource: "restore", + Action: "create", + Name: restoreName, + Options: options, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Errorf("failed to create restore with options via CLI: %v, output: %s", err, string(output)) + } + + log.Printf("Restore with options created via CLI: %s", restoreName) + return nil +} + +// GetRestoreViaCLI gets restore details using the OADP CLI +func GetRestoreViaCLI(name string) (*velero.Restore, error) { + if name == "" { + return nil, fmt.Errorf("restore name cannot be empty") + } + + // Use CLI to get restore details in JSON format + cmd := &CLICommand{ + Resource: "restore", + Action: "get", + Name: name, + Options: []string{"-o", "json"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return nil, fmt.Errorf("failed to get restore via CLI: %v", err) + } + + // Parse the JSON output back to velero.Restore struct + var restore velero.Restore + if err := json.Unmarshal(output, &restore); err != nil { + return nil, fmt.Errorf("failed to parse restore JSON: %v", err) + } + + return &restore, nil +} + +// IsRestoreDoneViaCLI checks if restore is done using the OADP CLI +func IsRestoreDoneViaCLI(name string) wait.ConditionFunc { + return func() (bool, error) { + // Use CLI to get restore status + cmd := &CLICommand{ + Resource: "restore", + Action: "get", + Name: name, + Options: []string{"-o", "yaml"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return false, fmt.Errorf("failed to get restore status via CLI: %v", err) + } + + // Parse phase from YAML output + phase := ParsePhaseFromYAML(string(output)) + + if len(phase) > 0 { + log.Printf("restore phase: %s", phase) + } + + var phasesNotDone = []string{ + string(velero.RestorePhaseNew), + string(velero.RestorePhaseInProgress), + string(velero.RestorePhaseWaitingForPluginOperations), + string(velero.RestorePhaseWaitingForPluginOperationsPartiallyFailed), + string(velero.RestorePhaseFinalizing), + string(velero.RestorePhaseFinalizingPartiallyFailed), + "", + } + + for _, notDonePhase := range phasesNotDone { + if phase == notDonePhase { + return false, nil + } + } + return true, nil + } +} + +// IsRestoreCompletedSuccessfullyViaCLI checks if restore completed successfully using the OADP CLI +func IsRestoreCompletedSuccessfullyViaCLI(name string) (bool, error) { + // Use CLI to get restore status + cmd := &CLICommand{ + Resource: "restore", + Action: "get", + Name: name, + Options: []string{"-o", "yaml"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return false, fmt.Errorf("failed to get restore status via CLI: %v", err) + } + + // Parse phase from YAML output + phase := ParsePhaseFromYAML(string(output)) + + if phase == string(velero.RestorePhaseCompleted) { + return true, nil + } + + // Get additional failure information using CLI + restoreLogs, logsErr := RestoreLogsViaCLI(name) + if logsErr != nil { + restoreLogs = fmt.Sprintf("Failed to get logs: %v", logsErr) + } + + return false, fmt.Errorf( + "restore phase is: %s; expected: %s\nvelero failure logs: %v", + phase, string(velero.RestorePhaseCompleted), restoreLogs, + ) +} + +// DescribeRestoreViaCLI describes restore using the OADP CLI +func DescribeRestoreViaCLI(name string) string { + // Use CLI to describe restore + cmd := &CLICommand{ + Resource: "restore", + Action: "describe", + Name: name, + Options: []string{"--details"}, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Sprintf("could not describe restore via CLI: %v, output: %s", err, string(output)) + } + + return string(output) +} + +// RestoreLogsViaCLI gets restore logs using the OADP CLI +func RestoreLogsViaCLI(name string) (restoreLogs string, err error) { + if name == "" { + return "", fmt.Errorf("restore name cannot be empty") + } + + // Use CLI to get restore logs + cmd := &CLICommand{ + Resource: "restore", + Action: "logs", + Name: name, + Options: []string{}, + } + output, cmdErr := cmd.ExecuteOutput() + if cmdErr != nil { + return "", fmt.Errorf("failed to get restore logs via CLI: %v", cmdErr) + } + + return string(output), nil +} + +// RestoreErrorLogsViaCLI gets restore error logs using the OADP CLI +func RestoreErrorLogsViaCLI(name string) []string { + rl, err := RestoreLogsViaCLI(name) + if err != nil { + return []string{err.Error()} + } + return errorLogsExcludingIgnored(rl) +} + +// DeleteRestoreViaCLI deletes a restore using the OADP CLI +func DeleteRestoreViaCLI(name string) error { + if name == "" { + return fmt.Errorf("restore name cannot be empty") + } + + // Use CLI to delete restore + cmd := &CLICommand{ + Resource: "restore", + Action: "delete", + Name: name, + Options: []string{}, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Errorf("failed to delete restore via CLI: %v, output: %s", err, string(output)) + } + + log.Printf("Restore deleted via CLI: %s", name) + return nil +} + +// ListRestoresViaCLI lists all restores using the OADP CLI +func ListRestoresViaCLI() ([]string, error) { + // Use CLI to list restores + cmd := &CLICommand{ + Resource: "restore", + Action: "list", + Name: "", // list commands don't require a name + Options: []string{"-o", "name"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return nil, fmt.Errorf("failed to list restores via CLI: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + var restores []string + for _, line := range lines { + if line != "" { + // Remove "restore.velero.io/" prefix if present + restore := strings.TrimPrefix(line, "restore.velero.io/") + restores = append(restores, restore) + } + } + + return restores, nil +} From b7f5b439d7bf2d1ebb4a7a2be0ee0b09abdf1439 Mon Sep 17 00:00:00 2001 From: Joseph Antony Vaikath Date: Thu, 14 Aug 2025 18:25:56 -0700 Subject: [PATCH 2/6] ci change to avoid interactive prompt (#1899) * Test Refactor OADP CLI installation in CI Dockerfile and e2e tests - Removed direct installation of oadp-cli from the Dockerfile. - Introduced a new CLISetup struct to handle the cloning, building, and installation of oadp-cli in a more structured manner. - Updated e2e tests to utilize the new CLISetup for verifying OADP CLI availability and configuration. * Debug * Manual build and push to path * Cleanup * Refactor CLI binary installation process in e2e tests - Changed the installation method from moving to copying the kubectl-oadp binary to /usr/local/bin. - Updated log messages to reflect the change in operation. - Improved error messages for better clarity on failure points. * Update --- tests/e2e/backup_restore_cli_suite_test.go | 12 +- tests/e2e/lib/cli_common.go | 171 +++++++++++++++++++++ 2 files changed, 175 insertions(+), 8 deletions(-) diff --git a/tests/e2e/backup_restore_cli_suite_test.go b/tests/e2e/backup_restore_cli_suite_test.go index b3bfb01f642..1de1187a204 100644 --- a/tests/e2e/backup_restore_cli_suite_test.go +++ b/tests/e2e/backup_restore_cli_suite_test.go @@ -3,7 +3,6 @@ package e2e_test import ( "fmt" "log" - "os/exec" "strings" "time" @@ -196,14 +195,11 @@ var _ = ginkgo.Describe("Backup and restore tests via OADP CLI", ginkgo.Label("c } ginkgo.BeforeAll(func() { - // Verify OADP CLI is available (should be installed in Docker image) - log.Print("Verifying OADP CLI is available...") - cmd := exec.Command("kubectl", "oadp", "version") - output, err := cmd.CombinedOutput() - if err != nil { - ginkgo.Fail(fmt.Sprintf("OADP CLI not available: %v, output: %s", err, string(output))) + + cliSetup := lib.NewOADPCLISetup() + if err := cliSetup.Install(); err != nil { + ginkgo.Fail(fmt.Sprintf("OADP CLI setup failed: %v", err)) } - log.Printf("OADP CLI available. Version: %s", string(output)) }) var _ = ginkgo.AfterEach(func(ctx ginkgo.SpecContext) { diff --git a/tests/e2e/lib/cli_common.go b/tests/e2e/lib/cli_common.go index 0ececbaf7b7..eb54aa274fa 100644 --- a/tests/e2e/lib/cli_common.go +++ b/tests/e2e/lib/cli_common.go @@ -1,8 +1,11 @@ package lib import ( + "fmt" "log" + "os" "os/exec" + "path/filepath" "strings" ) @@ -56,3 +59,171 @@ func ParsePhaseFromYAML(yamlOutput string) string { } return "" } + +type CLISetup struct { + repoURL string + installArgs []string + namespace string +} + +func NewOADPCLISetup() *CLISetup { + return &CLISetup{ + repoURL: "https://github.com/migtools/oadp-cli.git", + installArgs: []string{"build"}, + namespace: "openshift-adp", + } +} + +func (c *CLISetup) Install() error { + tmpDir, err := c.createTempDir() + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + cloneDir := filepath.Join(tmpDir, "oadp-cli") + + steps := []struct { + name string + fn func() error + }{ + {"Cloning repository", func() error { return c.cloneRepo(cloneDir) }}, + {"Building and installing", func() error { return c.buildAndInstall(cloneDir) }}, + {"Verifying installation", func() error { return c.verifyInstallation() }}, + {"Configuring namespace", func() error { return c.configureNamespace() }}, + } + + for _, step := range steps { + log.Printf("OADP CLI Setup: %s...", step.name) + if err := step.fn(); err != nil { + return fmt.Errorf("%s failed: %w", step.name, err) + } + } + + log.Print("OADP CLI setup completed successfully") + return nil +} + +func (c *CLISetup) createTempDir() (string, error) { + tmpDir, err := os.MkdirTemp("", "oadp-cli-*") + if err != nil { + return "", fmt.Errorf("failed to create temp directory: %w", err) + } + return tmpDir, nil +} + +func (c *CLISetup) cloneRepo(cloneDir string) error { + return runCommand("git", []string{"clone", c.repoURL, cloneDir}, "") +} + +func (c *CLISetup) buildAndInstall(cloneDir string) error { + // Build the binary + log.Print("Building OADP CLI...") + if err := runCommand("make", c.installArgs, cloneDir); err != nil { + return fmt.Errorf("build failed: %w", err) + } + + // Verify the binary was created + binaryPath := filepath.Join(cloneDir, "kubectl-oadp") + if _, err := os.Stat(binaryPath); err != nil { + return fmt.Errorf("kubectl-oadp binary not found at %s: %w", binaryPath, err) + } + + // Try multiple target locations, starting with user-writable paths + targetPaths := []string{ + fmt.Sprintf("%s/bin/kubectl-oadp", os.Getenv("HOME")), + "/usr/local/bin/kubectl-oadp", + } + + var targetPath string + var moveErr error + + for _, tp := range targetPaths { + targetPath = tp + log.Printf("Attempting to move binary from %s to %s", binaryPath, targetPath) + + // Create directory if it doesn't exist (for ~/bin) + if targetPath == fmt.Sprintf("%s/bin/kubectl-oadp", os.Getenv("HOME")) { + binDir := filepath.Dir(targetPath) + if err := os.MkdirAll(binDir, 0755); err != nil { + log.Printf("Failed to create directory %s: %v", binDir, err) + continue + } + } + + if err := runCommand("mv", []string{binaryPath, targetPath}, ""); err != nil { + log.Printf("Failed to move to %s: %v", targetPath, err) + moveErr = err + continue + } + + // Success! + moveErr = nil + break + } + + if moveErr != nil { + return fmt.Errorf("failed to move binary to any location: %w", moveErr) + } + + // Make it executable + if err := runCommand("chmod", []string{"+x", targetPath}, ""); err != nil { + return fmt.Errorf("failed to make binary executable: %w", err) + } + + // If we installed to ~/bin, ensure it's in PATH + if targetPath == fmt.Sprintf("%s/bin/kubectl-oadp", os.Getenv("HOME")) { + homeBin := fmt.Sprintf("%s/bin", os.Getenv("HOME")) + currentPath := os.Getenv("PATH") + if !strings.Contains(currentPath, homeBin) { + newPath := fmt.Sprintf("%s:%s", homeBin, currentPath) + os.Setenv("PATH", newPath) + log.Printf("Added %s to PATH", homeBin) + } + } + + log.Printf("Successfully installed kubectl-oadp to %s", targetPath) + return nil +} + +func (c *CLISetup) verifyInstallation() error { + // Check current PATH + cmd := exec.Command("bash", "-c", "echo $PATH") + if output, err := cmd.CombinedOutput(); err == nil { + log.Printf("Current PATH: %s", string(output)) + } + + // Try to find kubectl-oadp binary + cmd = exec.Command("which", "kubectl-oadp") + if output, err := cmd.CombinedOutput(); err == nil { + log.Printf("kubectl-oadp found at: %s", string(output)) + } else { + log.Printf("kubectl-oadp not found in PATH: %v", err) + } + + // List kubectl plugins + cmd = exec.Command("kubectl", "plugin", "list") + if output, err := cmd.CombinedOutput(); err == nil { + log.Printf("Available kubectl plugins: %s", string(output)) + } + + return runCommand("kubectl", []string{"oadp", "version"}, "") +} + +func (c *CLISetup) configureNamespace() error { + return runCommand("kubectl", []string{"oadp", "client", "config", "set", fmt.Sprintf("namespace=%s", c.namespace)}, "") +} + +func runCommand(name string, args []string, dir string) error { + cmd := exec.Command(name, args...) + if dir != "" { + cmd.Dir = dir + } + + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("command '%s %s' failed: %v\nOutput: %s", + name, strings.Join(args, " "), err, string(output)) + } + return nil +} From 368580e99da5e3fc784c30aeaeb89fcf2821fb18 Mon Sep 17 00:00:00 2001 From: Michal Pryc Date: Tue, 27 Jan 2026 18:46:19 +0100 Subject: [PATCH 3/6] Add timeout and retry to CLI log/describe commands in e2e tests (#2049) Prevents tests from hanging indefinitely when streaming logs from object storage. Uses 5-minute timeout with 3 retries. Signed-off-by: Michal Pryc Co-authored-by: Claude --- tests/e2e/lib/backup_cli.go | 47 +++++++++++++-- tests/e2e/lib/cli_common.go | 109 +++++++++++++++++++++++++++++++++++ tests/e2e/lib/restore_cli.go | 47 +++++++++++++-- 3 files changed, 193 insertions(+), 10 deletions(-) diff --git a/tests/e2e/lib/backup_cli.go b/tests/e2e/lib/backup_cli.go index 217470474b0..003e5307712 100644 --- a/tests/e2e/lib/backup_cli.go +++ b/tests/e2e/lib/backup_cli.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "strings" + "time" velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -191,8 +192,14 @@ func IsBackupCompletedSuccessfullyViaCLI(name string) (bool, error) { ) } -// DescribeBackupViaCLI describes backup using the OADP CLI +// DescribeBackupViaCLI describes backup using the OADP CLI with default timeout and retry. +// The timeout prevents the command from hanging when retrieving backup details from object storage. func DescribeBackupViaCLI(name string) (backupDescription string) { + return DescribeBackupViaCLIWithOptions(name, DefaultCLITimeout, DefaultCLIRetries, DefaultCLIRetryDelay) +} + +// DescribeBackupViaCLIWithOptions describes backup using the OADP CLI with specified timeout and retry options. +func DescribeBackupViaCLIWithOptions(name string, timeout time.Duration, maxRetries int, retryDelay time.Duration) (backupDescription string) { // Use CLI to describe backup cmd := &CLICommand{ Resource: "backup", @@ -200,7 +207,16 @@ func DescribeBackupViaCLI(name string) (backupDescription string) { Name: name, Options: []string{"--details"}, } - output, err := cmd.Execute() + + var output []byte + var err error + + if maxRetries > 1 { + output, err = cmd.ExecuteWithTimeoutAndRetry(timeout, maxRetries, retryDelay) + } else { + output, err = cmd.ExecuteWithTimeout(timeout) + } + if err != nil { return fmt.Sprintf("could not describe backup via CLI: %v, output: %s", err, string(output)) } @@ -208,20 +224,41 @@ func DescribeBackupViaCLI(name string) (backupDescription string) { return string(output) } -// BackupLogsViaCLI gets backup logs using the OADP CLI +// BackupLogsViaCLI gets backup logs using the OADP CLI with default timeout and retry. +// The timeout prevents the command from hanging indefinitely when streaming logs from object storage. +// Retry logic helps handle transient network issues. func BackupLogsViaCLI(name string) (backupLogs string, err error) { + return BackupLogsViaCLIWithOptions(name, DefaultCLITimeout, DefaultCLIRetries, DefaultCLIRetryDelay) +} + +// BackupLogsViaCLIWithTimeout gets backup logs using the OADP CLI with a specified timeout (no retry). +func BackupLogsViaCLIWithTimeout(name string, timeout time.Duration) (backupLogs string, err error) { + return BackupLogsViaCLIWithOptions(name, timeout, 1, 0) +} + +// BackupLogsViaCLIWithOptions gets backup logs using the OADP CLI with specified timeout and retry options. +func BackupLogsViaCLIWithOptions(name string, timeout time.Duration, maxRetries int, retryDelay time.Duration) (backupLogs string, err error) { if name == "" { return "", fmt.Errorf("backup name cannot be empty") } - // Use CLI to get backup logs + // Use CLI to get backup logs with timeout and retry to prevent hanging cmd := &CLICommand{ Resource: "backup", Action: "logs", Name: name, Options: []string{}, } - output, cmdErr := cmd.ExecuteOutput() + + var output []byte + var cmdErr error + + if maxRetries > 1 { + output, cmdErr = cmd.ExecuteOutputWithTimeoutAndRetry(timeout, maxRetries, retryDelay) + } else { + output, cmdErr = cmd.ExecuteOutputWithTimeout(timeout) + } + if cmdErr != nil { return "", fmt.Errorf("failed to get backup logs via CLI: %v", cmdErr) } diff --git a/tests/e2e/lib/cli_common.go b/tests/e2e/lib/cli_common.go index eb54aa274fa..eb19741776d 100644 --- a/tests/e2e/lib/cli_common.go +++ b/tests/e2e/lib/cli_common.go @@ -1,12 +1,23 @@ package lib import ( + "context" "fmt" "log" "os" "os/exec" "path/filepath" "strings" + "time" +) + +// Default timeout for CLI commands that may hang (e.g., log streaming) +const DefaultCLITimeout = 5 * time.Minute + +// Default retry settings for CLI commands +const ( + DefaultCLIRetries = 3 + DefaultCLIRetryDelay = 10 * time.Second ) type CLICommand struct { @@ -14,6 +25,7 @@ type CLICommand struct { Action string // "create", "get", "delete", etc. Name string Options []string + Timeout time.Duration // Optional timeout for commands that may hang } func (c *CLICommand) Execute() ([]byte, error) { @@ -40,6 +52,103 @@ func (c *CLICommand) ExecuteOutput() ([]byte, error) { return cmd.Output() } +// ExecuteWithTimeout executes the CLI command with a timeout. +// If the timeout is exceeded, the command is killed and an error is returned. +func (c *CLICommand) ExecuteWithTimeout(timeout time.Duration) ([]byte, error) { + args := []string{"oadp", c.Resource, c.Action} + if c.Name != "" { + args = append(args, c.Name) + } + args = append(args, c.Options...) + + c.LogCLICommand() + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, "kubectl", args...) + output, err := cmd.CombinedOutput() + + if ctx.Err() == context.DeadlineExceeded { + return output, fmt.Errorf("command timed out after %v: kubectl %s", timeout, strings.Join(args, " ")) + } + + return output, err +} + +// ExecuteOutputWithTimeout executes the CLI command with a timeout and returns stdout only. +// If the timeout is exceeded, the command is killed and an error is returned. +func (c *CLICommand) ExecuteOutputWithTimeout(timeout time.Duration) ([]byte, error) { + args := []string{"oadp", c.Resource, c.Action} + if c.Name != "" { + args = append(args, c.Name) + } + args = append(args, c.Options...) + + c.LogCLICommand() + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, "kubectl", args...) + output, err := cmd.Output() + + if ctx.Err() == context.DeadlineExceeded { + return output, fmt.Errorf("command timed out after %v: kubectl %s", timeout, strings.Join(args, " ")) + } + + return output, err +} + +// ExecuteOutputWithTimeoutAndRetry executes the CLI command with a timeout and retry logic. +// It retries the command up to maxRetries times with a delay between attempts. +// This is useful for commands that may fail due to transient issues (e.g., network problems). +func (c *CLICommand) ExecuteOutputWithTimeoutAndRetry(timeout time.Duration, maxRetries int, retryDelay time.Duration) ([]byte, error) { + var lastErr error + var lastOutput []byte + + for attempt := 1; attempt <= maxRetries; attempt++ { + output, err := c.ExecuteOutputWithTimeout(timeout) + if err == nil { + return output, nil + } + + lastErr = err + lastOutput = output + + if attempt < maxRetries { + log.Printf("CLI command failed (attempt %d/%d): %v. Retrying in %v...", attempt, maxRetries, err, retryDelay) + time.Sleep(retryDelay) + } + } + + return lastOutput, fmt.Errorf("CLI command failed after %d attempts: %v", maxRetries, lastErr) +} + +// ExecuteWithTimeoutAndRetry executes the CLI command with a timeout and retry logic. +// It retries the command up to maxRetries times with a delay between attempts. +func (c *CLICommand) ExecuteWithTimeoutAndRetry(timeout time.Duration, maxRetries int, retryDelay time.Duration) ([]byte, error) { + var lastErr error + var lastOutput []byte + + for attempt := 1; attempt <= maxRetries; attempt++ { + output, err := c.ExecuteWithTimeout(timeout) + if err == nil { + return output, nil + } + + lastErr = err + lastOutput = output + + if attempt < maxRetries { + log.Printf("CLI command failed (attempt %d/%d): %v. Retrying in %v...", attempt, maxRetries, err, retryDelay) + time.Sleep(retryDelay) + } + } + + return lastOutput, fmt.Errorf("CLI command failed after %d attempts: %v", maxRetries, lastErr) +} + func (c *CLICommand) LogCLICommand() { args := []string{"kubectl", "oadp", c.Resource, c.Action} if c.Name != "" { diff --git a/tests/e2e/lib/restore_cli.go b/tests/e2e/lib/restore_cli.go index ae70afdc793..0c252d3e87c 100644 --- a/tests/e2e/lib/restore_cli.go +++ b/tests/e2e/lib/restore_cli.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "strings" + "time" velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -166,8 +167,14 @@ func IsRestoreCompletedSuccessfullyViaCLI(name string) (bool, error) { ) } -// DescribeRestoreViaCLI describes restore using the OADP CLI +// DescribeRestoreViaCLI describes restore using the OADP CLI with default timeout and retry. +// The timeout prevents the command from hanging when retrieving restore details from object storage. func DescribeRestoreViaCLI(name string) string { + return DescribeRestoreViaCLIWithOptions(name, DefaultCLITimeout, DefaultCLIRetries, DefaultCLIRetryDelay) +} + +// DescribeRestoreViaCLIWithOptions describes restore using the OADP CLI with specified timeout and retry options. +func DescribeRestoreViaCLIWithOptions(name string, timeout time.Duration, maxRetries int, retryDelay time.Duration) string { // Use CLI to describe restore cmd := &CLICommand{ Resource: "restore", @@ -175,7 +182,16 @@ func DescribeRestoreViaCLI(name string) string { Name: name, Options: []string{"--details"}, } - output, err := cmd.Execute() + + var output []byte + var err error + + if maxRetries > 1 { + output, err = cmd.ExecuteWithTimeoutAndRetry(timeout, maxRetries, retryDelay) + } else { + output, err = cmd.ExecuteWithTimeout(timeout) + } + if err != nil { return fmt.Sprintf("could not describe restore via CLI: %v, output: %s", err, string(output)) } @@ -183,20 +199,41 @@ func DescribeRestoreViaCLI(name string) string { return string(output) } -// RestoreLogsViaCLI gets restore logs using the OADP CLI +// RestoreLogsViaCLI gets restore logs using the OADP CLI with default timeout and retry. +// The timeout prevents the command from hanging indefinitely when streaming logs from object storage. +// Retry logic helps handle transient network issues. func RestoreLogsViaCLI(name string) (restoreLogs string, err error) { + return RestoreLogsViaCLIWithOptions(name, DefaultCLITimeout, DefaultCLIRetries, DefaultCLIRetryDelay) +} + +// RestoreLogsViaCLIWithTimeout gets restore logs using the OADP CLI with a specified timeout (no retry). +func RestoreLogsViaCLIWithTimeout(name string, timeout time.Duration) (restoreLogs string, err error) { + return RestoreLogsViaCLIWithOptions(name, timeout, 1, 0) +} + +// RestoreLogsViaCLIWithOptions gets restore logs using the OADP CLI with specified timeout and retry options. +func RestoreLogsViaCLIWithOptions(name string, timeout time.Duration, maxRetries int, retryDelay time.Duration) (restoreLogs string, err error) { if name == "" { return "", fmt.Errorf("restore name cannot be empty") } - // Use CLI to get restore logs + // Use CLI to get restore logs with timeout and retry to prevent hanging cmd := &CLICommand{ Resource: "restore", Action: "logs", Name: name, Options: []string{}, } - output, cmdErr := cmd.ExecuteOutput() + + var output []byte + var cmdErr error + + if maxRetries > 1 { + output, cmdErr = cmd.ExecuteOutputWithTimeoutAndRetry(timeout, maxRetries, retryDelay) + } else { + output, cmdErr = cmd.ExecuteOutputWithTimeout(timeout) + } + if cmdErr != nil { return "", fmt.Errorf("failed to get restore logs via CLI: %v", cmdErr) } From 18c1fd65d7227f7e0e9ad584312b695a08eb5c99 Mon Sep 17 00:00:00 2001 From: OpenShift Cherrypick Robot Date: Mon, 23 Mar 2026 20:19:38 +0100 Subject: [PATCH 4/6] Fix CLI E2E tests missing ReadyToStart backup/restore phase (#2129) The CLI backup and restore "done" checks were missing the ReadyToStart phase, causing tests to incorrectly treat backups/restores as completed before they had started. This led to false PartiallyFailed results. Extract shared phase lists and helpers (IsBackupPhaseNotDone, IsRestorePhaseNotDone) to eliminate duplication between CLI and non-CLI code paths and prevent future phase drift. Co-authored-by: Joseph Co-authored-by: Claude Opus 4.6 (1M context) --- tests/e2e/lib/backup.go | 39 ++++++++++++++++++++++-------------- tests/e2e/lib/backup_cli.go | 19 +----------------- tests/e2e/lib/restore.go | 38 +++++++++++++++++++++-------------- tests/e2e/lib/restore_cli.go | 19 +----------------- 4 files changed, 49 insertions(+), 66 deletions(-) diff --git a/tests/e2e/lib/backup.go b/tests/e2e/lib/backup.go index 2836b5e98b5..d1f62214051 100755 --- a/tests/e2e/lib/backup.go +++ b/tests/e2e/lib/backup.go @@ -62,6 +62,29 @@ func GetBackup(c client.Client, namespace string, name string) (*velero.Backup, return &backup, nil } +// backupPhasesNotDone is the shared list of backup phases that indicate a backup is still in progress. +var backupPhasesNotDone = []velero.BackupPhase{ + velero.BackupPhaseNew, + velero.BackupPhaseQueued, + velero.BackupPhaseReadyToStart, + velero.BackupPhaseInProgress, + velero.BackupPhaseWaitingForPluginOperations, + velero.BackupPhaseWaitingForPluginOperationsPartiallyFailed, + velero.BackupPhaseFinalizing, + velero.BackupPhaseFinalizingPartiallyFailed, + "", +} + +// IsBackupPhaseNotDone returns true if the given phase string represents a backup still in progress. +func IsBackupPhaseNotDone(phase string) bool { + for _, notDonePhase := range backupPhasesNotDone { + if phase == string(notDonePhase) { + return true + } + } + return false +} + func IsBackupDone(ocClient client.Client, veleroNamespace, name string) wait.ConditionFunc { return func() (bool, error) { backup, err := GetBackup(ocClient, veleroNamespace, name) @@ -71,21 +94,7 @@ func IsBackupDone(ocClient client.Client, veleroNamespace, name string) wait.Con if len(backup.Status.Phase) > 0 { log.Printf("backup phase: %s", backup.Status.Phase) } - var phasesNotDone = []velero.BackupPhase{ - velero.BackupPhaseNew, - velero.BackupPhaseInProgress, - velero.BackupPhaseWaitingForPluginOperations, - velero.BackupPhaseWaitingForPluginOperationsPartiallyFailed, - velero.BackupPhaseFinalizing, - velero.BackupPhaseFinalizingPartiallyFailed, - "", - } - for _, notDonePhase := range phasesNotDone { - if backup.Status.Phase == notDonePhase { - return false, nil - } - } - return true, nil + return !IsBackupPhaseNotDone(string(backup.Status.Phase)), nil } } diff --git a/tests/e2e/lib/backup_cli.go b/tests/e2e/lib/backup_cli.go index 003e5307712..50da9f6d1cd 100644 --- a/tests/e2e/lib/backup_cli.go +++ b/tests/e2e/lib/backup_cli.go @@ -121,7 +121,6 @@ func GetBackupViaCLI(name string) (*velero.Backup, error) { // IsBackupDoneViaCLI checks if backup is done using the OADP CLI func IsBackupDoneViaCLI(name string) wait.ConditionFunc { return func() (bool, error) { - // Use CLI to get backup status cmd := &CLICommand{ Resource: "backup", Action: "get", @@ -133,29 +132,13 @@ func IsBackupDoneViaCLI(name string) wait.ConditionFunc { return false, fmt.Errorf("failed to get backup status via CLI: %v", err) } - // Parse phase from YAML output phase := ParsePhaseFromYAML(string(output)) if len(phase) > 0 { log.Printf("backup phase: %s", phase) } - var phasesNotDone = []string{ - string(velero.BackupPhaseNew), - string(velero.BackupPhaseInProgress), - string(velero.BackupPhaseWaitingForPluginOperations), - string(velero.BackupPhaseWaitingForPluginOperationsPartiallyFailed), - string(velero.BackupPhaseFinalizing), - string(velero.BackupPhaseFinalizingPartiallyFailed), - "", - } - - for _, notDonePhase := range phasesNotDone { - if phase == notDonePhase { - return false, nil - } - } - return true, nil + return !IsBackupPhaseNotDone(phase), nil } } diff --git a/tests/e2e/lib/restore.go b/tests/e2e/lib/restore.go index b3a80bcaf9e..d8ce1309ccd 100755 --- a/tests/e2e/lib/restore.go +++ b/tests/e2e/lib/restore.go @@ -42,6 +42,28 @@ func GetRestore(c client.Client, namespace string, name string) (*velero.Restore return &restore, nil } +// restorePhasesNotDone is the shared list of restore phases that indicate a restore is still in progress. +var restorePhasesNotDone = []velero.RestorePhase{ + velero.RestorePhaseNew, + "ReadyToStart", + velero.RestorePhaseInProgress, + velero.RestorePhaseWaitingForPluginOperations, + velero.RestorePhaseWaitingForPluginOperationsPartiallyFailed, + velero.RestorePhaseFinalizing, + velero.RestorePhaseFinalizingPartiallyFailed, + "", +} + +// IsRestorePhaseNotDone returns true if the given phase string represents a restore still in progress. +func IsRestorePhaseNotDone(phase string) bool { + for _, notDonePhase := range restorePhasesNotDone { + if phase == string(notDonePhase) { + return true + } + } + return false +} + func IsRestoreDone(ocClient client.Client, veleroNamespace, name string) wait.ConditionFunc { return func() (bool, error) { restore, err := GetRestore(ocClient, veleroNamespace, name) @@ -51,21 +73,7 @@ func IsRestoreDone(ocClient client.Client, veleroNamespace, name string) wait.Co if len(restore.Status.Phase) > 0 { log.Printf("restore phase: %s", restore.Status.Phase) } - var phasesNotDone = []velero.RestorePhase{ - velero.RestorePhaseNew, - velero.RestorePhaseInProgress, - velero.RestorePhaseWaitingForPluginOperations, - velero.RestorePhaseWaitingForPluginOperationsPartiallyFailed, - velero.RestorePhaseFinalizing, - velero.RestorePhaseFinalizingPartiallyFailed, - "", - } - for _, notDonePhase := range phasesNotDone { - if restore.Status.Phase == notDonePhase { - return false, nil - } - } - return true, nil + return !IsRestorePhaseNotDone(string(restore.Status.Phase)), nil } } diff --git a/tests/e2e/lib/restore_cli.go b/tests/e2e/lib/restore_cli.go index 0c252d3e87c..0439a3db3bd 100644 --- a/tests/e2e/lib/restore_cli.go +++ b/tests/e2e/lib/restore_cli.go @@ -96,7 +96,6 @@ func GetRestoreViaCLI(name string) (*velero.Restore, error) { // IsRestoreDoneViaCLI checks if restore is done using the OADP CLI func IsRestoreDoneViaCLI(name string) wait.ConditionFunc { return func() (bool, error) { - // Use CLI to get restore status cmd := &CLICommand{ Resource: "restore", Action: "get", @@ -108,29 +107,13 @@ func IsRestoreDoneViaCLI(name string) wait.ConditionFunc { return false, fmt.Errorf("failed to get restore status via CLI: %v", err) } - // Parse phase from YAML output phase := ParsePhaseFromYAML(string(output)) if len(phase) > 0 { log.Printf("restore phase: %s", phase) } - var phasesNotDone = []string{ - string(velero.RestorePhaseNew), - string(velero.RestorePhaseInProgress), - string(velero.RestorePhaseWaitingForPluginOperations), - string(velero.RestorePhaseWaitingForPluginOperationsPartiallyFailed), - string(velero.RestorePhaseFinalizing), - string(velero.RestorePhaseFinalizingPartiallyFailed), - "", - } - - for _, notDonePhase := range phasesNotDone { - if phase == notDonePhase { - return false, nil - } - } - return true, nil + return !IsRestorePhaseNotDone(phase), nil } } From c3583a8f77c66fffe91bd12b189e0f3698c7df97 Mon Sep 17 00:00:00 2001 From: Nicholas Yancey Date: Wed, 8 Jul 2026 15:27:54 -0400 Subject: [PATCH 5/6] Used string literals and pin oadp-cli to oadp-1.4 --- tests/e2e/lib/backup.go | 4 ++-- tests/e2e/lib/cli_common.go | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/e2e/lib/backup.go b/tests/e2e/lib/backup.go index d1f62214051..3a1d70df294 100755 --- a/tests/e2e/lib/backup.go +++ b/tests/e2e/lib/backup.go @@ -65,8 +65,8 @@ func GetBackup(c client.Client, namespace string, name string) (*velero.Backup, // backupPhasesNotDone is the shared list of backup phases that indicate a backup is still in progress. var backupPhasesNotDone = []velero.BackupPhase{ velero.BackupPhaseNew, - velero.BackupPhaseQueued, - velero.BackupPhaseReadyToStart, + "Queued", + "ReadyToStart", velero.BackupPhaseInProgress, velero.BackupPhaseWaitingForPluginOperations, velero.BackupPhaseWaitingForPluginOperationsPartiallyFailed, diff --git a/tests/e2e/lib/cli_common.go b/tests/e2e/lib/cli_common.go index eb19741776d..cec5bdf1501 100644 --- a/tests/e2e/lib/cli_common.go +++ b/tests/e2e/lib/cli_common.go @@ -171,6 +171,7 @@ func ParsePhaseFromYAML(yamlOutput string) string { type CLISetup struct { repoURL string + repoBranch string installArgs []string namespace string } @@ -178,6 +179,7 @@ type CLISetup struct { func NewOADPCLISetup() *CLISetup { return &CLISetup{ repoURL: "https://github.com/migtools/oadp-cli.git", + repoBranch: "oadp-1.4", installArgs: []string{"build"}, namespace: "openshift-adp", } @@ -222,7 +224,7 @@ func (c *CLISetup) createTempDir() (string, error) { } func (c *CLISetup) cloneRepo(cloneDir string) error { - return runCommand("git", []string{"clone", c.repoURL, cloneDir}, "") + return runCommand("git", []string{"clone", "--branch", c.repoBranch, "--depth", "1", c.repoURL, cloneDir}, "") } func (c *CLISetup) buildAndInstall(cloneDir string) error { From 95cd27e5c81b1b974f1c042f6af7b9a93449988c Mon Sep 17 00:00:00 2001 From: Nicholas Yancey Date: Wed, 8 Jul 2026 15:33:05 -0400 Subject: [PATCH 6/6] make test error fix --- tests/e2e/backup_restore_cli_suite_test.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/e2e/backup_restore_cli_suite_test.go b/tests/e2e/backup_restore_cli_suite_test.go index 1de1187a204..1ca697d11ef 100644 --- a/tests/e2e/backup_restore_cli_suite_test.go +++ b/tests/e2e/backup_restore_cli_suite_test.go @@ -210,14 +210,7 @@ var _ = ginkgo.Describe("Backup and restore tests via OADP CLI", ginkgo.Label("c // Same cleanup as original waitOADPReadiness(lib.KOPIA) - log.Printf("Creating real DataProtectionTest before must-gather") - bsls, err := dpaCR.ListBSLs() - gomega.Expect(err).ToNot(gomega.HaveOccurred()) - - bslName := bsls.Items[0].Name - err = lib.CreateUploadTestOnlyDPT(dpaCR.Client, dpaCR.Namespace, bslName) - gomega.Expect(err).ToNot(gomega.HaveOccurred()) - + var err error log.Printf("skipMustGather: %v", skipMustGather) if !skipMustGather { log.Printf("Running OADP must-gather")