diff --git a/Makefile b/Makefile index 5f2eefff984..8d3dabb8d18 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,8 @@ OC_CLI = $(shell which oc) TEST_VIRT ?= false TEST_HCP ?= false TEST_UPGRADE ?= false +TEST_CLI ?= false +SKIP_MUST_GATHER ?= false # TOOL VERSIONS # All version-related variables are defined here for easy maintenance @@ -615,6 +617,11 @@ ifeq ($(TEST_HCP),true) else TEST_FILTER += && (! hcp) endif +ifeq ($(TEST_CLI),true) + TEST_FILTER += && (cli) +else + TEST_FILTER += && (! cli) +endif SETTINGS_TMP=/tmp/test-settings .PHONY: test-e2e-setup diff --git a/build/ci-Dockerfile b/build/ci-Dockerfile index 8208510d941..9339479d7e6 100644 --- a/build/ci-Dockerfile +++ b/build/ci-Dockerfile @@ -6,7 +6,21 @@ WORKDIR /go/src/github.com/openshift/oadp-operator COPY ./ . RUN go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo -RUN go mod download -RUN chmod -R 777 ./ -RUN chmod -R 777 $(go env GOPATH) -RUN mkdir -p $(go env GOCACHE) && chmod -R 777 $(go env GOCACHE) + +# Clone and install oadp-cli pinned to oadp-1.4 +RUN git clone --depth=1 --branch oadp-1.4 https://github.com/migtools/oadp-cli.git /tmp/oadp-cli && \ + cd /tmp/oadp-cli && \ + make build && \ + cp kubectl-oadp /usr/local/bin/ && \ + chmod +x /usr/local/bin/kubectl-oadp && \ + rm -rf /tmp/oadp-cli + +# Install kubectl (multi-arch) +RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') && \ + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl" && \ + chmod +x kubectl && \ + mv kubectl /usr/local/bin/ + +RUN go mod download && \ + mkdir -p $(go env GOCACHE) && \ + chmod -R 777 ./ $(go env GOCACHE) $(go env GOPATH) diff --git a/docs/developer/testing/TESTING.md b/docs/developer/testing/TESTING.md index 649f62751b9..fbf529ac9b0 100644 --- a/docs/developer/testing/TESTING.md +++ b/docs/developer/testing/TESTING.md @@ -25,6 +25,9 @@ To get started, you need to provide the following **required** environment varia | `MUST_GATHER_IMAGE` | Container image to use for must-gather collection via `oc adm must-gather` | `quay.io/konveyor/oadp-must-gather:oadp-1.4` | false | | `MUST_GATHER_REPO` | GitHub repo (e.g., `openshift/oadp-must-gather`) to build must-gather from source. Sets `MUST_GATHER_IMAGE` to a ttl.sh image automatically | - | false | | `MUST_GATHER_BRANCH` | Branch to use when building from `MUST_GATHER_REPO` | `oadp-1.4` | false | +| `TEST_UPGRADE` | Exclusively run upgrade tests. Need to first run `make catalog-test-upgrade`, if testing non production operator | `false` | false | +| `TEST_CLI` | Exclusively run CLI-based backup/restore testing | `false` | false | +| `SKIP_MUST_GATHER` | must-gather is compiled locally in the Makefile, may cause issue if local and cluster arch do not match| `false` | false | The expected format for `OADP_CRED_FILE` and `CI_CRED_FILE` files is: ``` diff --git a/tests/e2e/backup_restore_cli_suite_test.go b/tests/e2e/backup_restore_cli_suite_test.go new file mode 100644 index 00000000000..1ca697d11ef --- /dev/null +++ b/tests/e2e/backup_restore_cli_suite_test.go @@ -0,0 +1,366 @@ +package e2e_test + +import ( + "fmt" + "log" + "strings" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + "github.com/openshift/oadp-operator/tests/e2e/lib" +) + +// CLI-specific backup execution +func runBackupViaCLI(brCase BackupRestoreCase, backupName string) bool { + nsRequiresResticDCWorkaround, err := lib.NamespaceRequiresResticDCWorkaround(dpaCR.Client, brCase.Namespace) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + if strings.Contains(brCase.Name, "twovol") { + volumeSyncDelay := 30 * time.Second + log.Printf("Sleeping for %v to allow volume to be in sync with /tmp/log/ for case %s", volumeSyncDelay, brCase.Name) + time.Sleep(volumeSyncDelay) + } + + // Create backup via CLI + log.Printf("Creating backup %s for case %s via CLI", backupName, brCase.Name) + err = lib.CreateBackupForNamespacesViaCLI(backupName, []string{brCase.Namespace}, brCase.BackupRestoreType == lib.RESTIC || brCase.BackupRestoreType == lib.KOPIA, brCase.BackupRestoreType == lib.CSIDataMover) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Wait for backup via CLI + gomega.Eventually(lib.IsBackupDoneViaCLI(backupName), brCase.BackupTimeout, time.Second*10).Should(gomega.BeTrue()) + + // Get backup details via CLI + describeBackup := lib.DescribeBackupViaCLI(backupName) + ginkgo.GinkgoWriter.Println(describeBackup) + + backupLogs, err := lib.BackupLogsViaCLI(backupName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + backupErrorLogs := lib.BackupErrorLogsViaCLI(backupName) + accumulatedTestLogs = append(accumulatedTestLogs, describeBackup, backupLogs) + + if !brCase.SkipVerifyLogs { + gomega.Expect(backupErrorLogs).Should(gomega.Equal([]string{})) + } + + // Check if backup succeeded + succeeded, err := lib.IsBackupCompletedSuccessfullyViaCLI(backupName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(succeeded).To(gomega.Equal(true)) + + log.Printf("Backup for case %s succeeded via CLI", brCase.Name) + + return nsRequiresResticDCWorkaround +} + +// CLI-specific restore execution +func runRestoreViaCLI(brCase BackupRestoreCase, backupName, restoreName string, nsRequiresResticDCWorkaround bool) { + log.Printf("Creating restore %s for case %s via CLI", restoreName, brCase.Name) + err := lib.CreateRestoreFromBackupViaCLI(backupName, restoreName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + gomega.Eventually(lib.IsRestoreDoneViaCLI(restoreName), time.Minute*60, time.Second*10).Should(gomega.BeTrue()) + + // Get restore details via CLI + describeRestore := lib.DescribeRestoreViaCLI(restoreName) + ginkgo.GinkgoWriter.Println(describeRestore) + + restoreLogs, err := lib.RestoreLogsViaCLI(restoreName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + restoreErrorLogs := lib.RestoreErrorLogsViaCLI(restoreName) + accumulatedTestLogs = append(accumulatedTestLogs, describeRestore, restoreLogs) + + if !brCase.SkipVerifyLogs { + gomega.Expect(restoreErrorLogs).Should(gomega.Equal([]string{})) + } + + // Check if restore succeeded + succeeded, err := lib.IsRestoreCompletedSuccessfullyViaCLI(restoreName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(succeeded).To(gomega.Equal(true)) + + if nsRequiresResticDCWorkaround { + log.Printf("Running dc-post-restore.sh script.") + err = lib.RunDcPostRestoreScript(restoreName) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } +} + +// CLI-specific application backup and restore +func runApplicationBackupAndRestoreViaCLI(brCase ApplicationBackupRestoreCase, updateLastBRcase func(brCase ApplicationBackupRestoreCase), updateLastInstallTime func()) { + updateLastBRcase(brCase) + + // create DPA (still using K8s client for setup) + backupName, restoreName := prepareBackupAndRestore(brCase.BackupRestoreCase, updateLastInstallTime) + + // Ensure that an existing backup repository is deleted (still using K8s client) + brerr := lib.DeleteBackupRepositories(runTimeClientForSuiteRun, namespace) + gomega.Expect(brerr).ToNot(gomega.HaveOccurred()) + + // install app (still using K8s client for setup) + updateLastInstallTime() + log.Printf("Installing application for case %s", brCase.Name) + err := lib.InstallApplication(dpaCR.Client, brCase.ApplicationTemplate) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + if brCase.BackupRestoreType == lib.CSI || brCase.BackupRestoreType == lib.CSIDataMover { + log.Printf("Creating pvc for case %s", brCase.Name) + var pvcName string + var pvcPath string + + pvcName = provider + if brCase.PvcSuffixName != "" { + pvcName += brCase.PvcSuffixName + } + + pvcPathFormat := "./sample-applications/%s/pvc/%s.yaml" + if strings.Contains(brCase.Name, "twovol") { + pvcPathFormat = "./sample-applications/%s/pvc-twoVol/%s.yaml" + } + + pvcPath = fmt.Sprintf(pvcPathFormat, brCase.Namespace, pvcName) + + err = lib.InstallApplication(dpaCR.Client, pvcPath) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + // Run optional custom verification + if brCase.PreBackupVerify != nil { + log.Printf("Running pre-backup custom function for case %s", brCase.Name) + err := brCase.PreBackupVerify(dpaCR.Client, brCase.Namespace) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + // do the backup via CLI + nsRequiredResticDCWorkaround := runBackupViaCLI(brCase.BackupRestoreCase, backupName) + + // uninstall app (still using K8s client) + log.Printf("Uninstalling app for case %s", brCase.Name) + err = lib.UninstallApplication(dpaCR.Client, brCase.ApplicationTemplate) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // Wait for namespace to be deleted + gomega.Eventually(lib.IsNamespaceDeleted(kubernetesClientForSuiteRun, brCase.Namespace), time.Minute*4, time.Second*5).Should(gomega.BeTrue()) + + updateLastInstallTime() + + // run restore via CLI + runRestoreViaCLI(brCase.BackupRestoreCase, backupName, restoreName, nsRequiredResticDCWorkaround) + + // Run optional custom verification + if brCase.PostRestoreVerify != nil { + log.Printf("Running post-restore custom function for case %s", brCase.Name) + err = brCase.PostRestoreVerify(dpaCR.Client, brCase.Namespace) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } +} + +// CLI teardown function +func tearDownBackupAndRestoreViaCLI(brCase BackupRestoreCase, installTime time.Time, report ginkgo.SpecReport) { + log.Println("Post backup and restore state (CLI): ", report.State.String()) + + if report.Failed() { + knownFlake = lib.CheckIfFlakeOccurred(accumulatedTestLogs) + accumulatedTestLogs = nil + getFailedTestLogs(namespace, brCase.Namespace, installTime, report) + } + + if brCase.BackupRestoreType == lib.CSI || brCase.BackupRestoreType == lib.CSIDataMover { + log.Printf("Deleting VolumeSnapshot for CSI backuprestore of %s", brCase.Name) + snapshotClassPath := fmt.Sprintf("./sample-applications/snapclass-csi/%s.yaml", provider) + err := lib.UninstallApplication(dpaCR.Client, snapshotClassPath) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + err := dpaCR.Delete() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + err = lib.DeleteNamespace(kubernetesClientForSuiteRun, brCase.Namespace) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Eventually(lib.IsNamespaceDeleted(kubernetesClientForSuiteRun, brCase.Namespace), time.Minute*5, time.Second*5).Should(gomega.BeTrue()) +} + +// CLI Test Suite +var _ = ginkgo.Describe("Backup and restore tests via OADP CLI", ginkgo.Label("cli"), ginkgo.Ordered, func() { + var lastBRCase ApplicationBackupRestoreCase + var lastInstallTime time.Time + updateLastBRcase := func(brCase ApplicationBackupRestoreCase) { + lastBRCase = brCase + } + updateLastInstallTime := func() { + lastInstallTime = time.Now() + } + + ginkgo.BeforeAll(func() { + + cliSetup := lib.NewOADPCLISetup() + if err := cliSetup.Install(); err != nil { + ginkgo.Fail(fmt.Sprintf("OADP CLI setup failed: %v", err)) + } + }) + + var _ = ginkgo.AfterEach(func(ctx ginkgo.SpecContext) { + tearDownBackupAndRestoreViaCLI(lastBRCase.BackupRestoreCase, lastInstallTime, ctx.SpecReport()) + }) + + var _ = ginkgo.AfterAll(func() { + // Same cleanup as original + waitOADPReadiness(lib.KOPIA) + + var err error + log.Printf("skipMustGather: %v", skipMustGather) + if !skipMustGather { + log.Printf("Running OADP must-gather") + err = lib.RunMustGather(artifact_dir, dpaCR.Client) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + err = dpaCR.Delete() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) + + ginkgo.DescribeTable("Backup and restore applications via OADP CLI", + func(brCase ApplicationBackupRestoreCase, expectedErr error) { + if ginkgo.CurrentSpecReport().NumAttempts > 1 && !knownFlake { + ginkgo.Fail("No known FLAKE found in a previous run, marking test as failed.") + } + runApplicationBackupAndRestoreViaCLI(brCase, updateLastBRcase, updateLastInstallTime) + }, + ginkgo.Entry("MySQL application CSI via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-csi.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mysql-persistent", + Name: "mysql-csi-cli-e2e", + BackupRestoreType: lib.CSI, + PreBackupVerify: todoListReady(true, false, "mysql"), + PostRestoreVerify: todoListReady(false, false, "mysql"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("Mongo application CSI via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent-csi.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mongo-persistent", + Name: "mongo-csi-cli-e2e", + BackupRestoreType: lib.CSI, + PreBackupVerify: todoListReady(true, false, "mongo"), + PostRestoreVerify: todoListReady(false, false, "mongo"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("MySQL application two Vol CSI via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-twovol-csi.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mysql-persistent", + Name: "mysql-twovol-csi-cli-e2e", + BackupRestoreType: lib.CSI, + PreBackupVerify: todoListReady(true, true, "mysql"), + PostRestoreVerify: todoListReady(false, true, "mysql"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("Mongo application RESTIC via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mongo-persistent", + Name: "mongo-restic-cli-e2e", + BackupRestoreType: lib.RESTIC, + PreBackupVerify: todoListReady(true, false, "mongo"), + PostRestoreVerify: todoListReady(false, false, "mongo"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("MySQL application RESTIC via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mysql-persistent", + Name: "mysql-restic-cli-e2e", + BackupRestoreType: lib.RESTIC, + PreBackupVerify: todoListReady(true, false, "mysql"), + PostRestoreVerify: todoListReady(false, false, "mysql"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("Mongo application KOPIA via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mongo-persistent", + Name: "mongo-kopia-cli-e2e", + BackupRestoreType: lib.KOPIA, + PreBackupVerify: todoListReady(true, false, "mongo"), + PostRestoreVerify: todoListReady(false, false, "mongo"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("MySQL application KOPIA via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mysql-persistent", + Name: "mysql-kopia-cli-e2e", + BackupRestoreType: lib.KOPIA, + PreBackupVerify: todoListReady(true, false, "mysql"), + PostRestoreVerify: todoListReady(false, false, "mysql"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("Mongo application DATAMOVER via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent-csi.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mongo-persistent", + Name: "mongo-datamover-cli-e2e", + BackupRestoreType: lib.CSIDataMover, + PreBackupVerify: todoListReady(true, false, "mongo"), + PostRestoreVerify: todoListReady(false, false, "mongo"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("MySQL application DATAMOVER via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent-csi.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mysql-persistent", + Name: "mysql-datamover-cli-e2e", + BackupRestoreType: lib.CSIDataMover, + PreBackupVerify: todoListReady(true, false, "mysql"), + PostRestoreVerify: todoListReady(false, false, "mysql"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("Mongo application BlockDevice DATAMOVER via CLI", ginkgo.FlakeAttempts(flakeAttempts), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent-block.yaml", + PvcSuffixName: "-block-mode", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mongo-persistent", + Name: "mongo-blockdevice-cli-e2e", + BackupRestoreType: lib.CSIDataMover, + PreBackupVerify: todoListReady(true, false, "mongo"), + PostRestoreVerify: todoListReady(false, false, "mongo"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("MySQL application Native-Snapshots via CLI", ginkgo.FlakeAttempts(flakeAttempts), ginkgo.Label("aws", "azure", "gcp"), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mysql-persistent", + Name: "mysql-native-snapshots-cli-e2e", + BackupRestoreType: lib.NativeSnapshots, + PreBackupVerify: todoListReady(true, false, "mysql"), + PostRestoreVerify: todoListReady(false, false, "mysql"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ginkgo.Entry("Mongo application Native-Snapshots via CLI", ginkgo.FlakeAttempts(flakeAttempts), ginkgo.Label("aws", "azure", "gcp"), ApplicationBackupRestoreCase{ + ApplicationTemplate: "./sample-applications/mongo-persistent/mongo-persistent.yaml", + BackupRestoreCase: BackupRestoreCase{ + Namespace: "mongo-persistent", + Name: "mongo-native-snapshots-cli-e2e", + BackupRestoreType: lib.NativeSnapshots, + PreBackupVerify: todoListReady(true, false, "mongo"), + PostRestoreVerify: todoListReady(false, false, "mongo"), + BackupTimeout: 20 * time.Minute, + }, + }, nil), + ) +}) diff --git a/tests/e2e/lib/backup.go b/tests/e2e/lib/backup.go index 2836b5e98b5..3a1d70df294 100755 --- a/tests/e2e/lib/backup.go +++ b/tests/e2e/lib/backup.go @@ -62,6 +62,29 @@ func GetBackup(c client.Client, namespace string, name string) (*velero.Backup, return &backup, nil } +// backupPhasesNotDone is the shared list of backup phases that indicate a backup is still in progress. +var backupPhasesNotDone = []velero.BackupPhase{ + velero.BackupPhaseNew, + "Queued", + "ReadyToStart", + velero.BackupPhaseInProgress, + velero.BackupPhaseWaitingForPluginOperations, + velero.BackupPhaseWaitingForPluginOperationsPartiallyFailed, + velero.BackupPhaseFinalizing, + velero.BackupPhaseFinalizingPartiallyFailed, + "", +} + +// IsBackupPhaseNotDone returns true if the given phase string represents a backup still in progress. +func IsBackupPhaseNotDone(phase string) bool { + for _, notDonePhase := range backupPhasesNotDone { + if phase == string(notDonePhase) { + return true + } + } + return false +} + func IsBackupDone(ocClient client.Client, veleroNamespace, name string) wait.ConditionFunc { return func() (bool, error) { backup, err := GetBackup(ocClient, veleroNamespace, name) @@ -71,21 +94,7 @@ func IsBackupDone(ocClient client.Client, veleroNamespace, name string) wait.Con if len(backup.Status.Phase) > 0 { log.Printf("backup phase: %s", backup.Status.Phase) } - var phasesNotDone = []velero.BackupPhase{ - velero.BackupPhaseNew, - velero.BackupPhaseInProgress, - velero.BackupPhaseWaitingForPluginOperations, - velero.BackupPhaseWaitingForPluginOperationsPartiallyFailed, - velero.BackupPhaseFinalizing, - velero.BackupPhaseFinalizingPartiallyFailed, - "", - } - for _, notDonePhase := range phasesNotDone { - if backup.Status.Phase == notDonePhase { - return false, nil - } - } - return true, nil + return !IsBackupPhaseNotDone(string(backup.Status.Phase)), nil } } diff --git a/tests/e2e/lib/backup_cli.go b/tests/e2e/lib/backup_cli.go new file mode 100644 index 00000000000..50da9f6d1cd --- /dev/null +++ b/tests/e2e/lib/backup_cli.go @@ -0,0 +1,281 @@ +package lib + +import ( + "encoding/json" + "fmt" + "log" + "strings" + "time" + + velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +// CreateBackupForNamespacesViaCLI creates a backup using the OADP CLI +func CreateBackupForNamespacesViaCLI(backupName string, namespaces []string, defaultVolumesToFsBackup bool, snapshotMoveData bool) error { + var options []string + + // Add included namespaces (comma-separated) + if len(namespaces) > 0 { + options = append(options, "--include-namespaces", strings.Join(namespaces, ",")) + } + + // Add volume backup options + if defaultVolumesToFsBackup { + options = append(options, "--default-volumes-to-fs-backup") + } + + // Add snapshot move data option + if snapshotMoveData { + options = append(options, "--snapshot-move-data") + } + + // Execute CLI command + cmd := &CLICommand{ + Resource: "backup", + Action: "create", + Name: backupName, + Options: options, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Errorf("failed to create backup via CLI: %v, output: %s", err, string(output)) + } + + log.Printf("Backup created via CLI: %s", backupName) + return nil +} + +// CreateCustomBackupForNamespacesViaCLI creates a custom backup using the OADP CLI +func CreateCustomBackupForNamespacesViaCLI(backupName string, namespaces []string, includedResources, excludedResources []string, defaultVolumesToFsBackup bool, snapshotMoveData bool) error { + var options []string + + // Add included namespaces (comma-separated) + if len(namespaces) > 0 { + options = append(options, "--include-namespaces", strings.Join(namespaces, ",")) + } + + // Add included resources + if len(includedResources) > 0 { + options = append(options, "--include-resources", strings.Join(includedResources, ",")) + } + + // Add excluded resources + if len(excludedResources) > 0 { + options = append(options, "--exclude-resources", strings.Join(excludedResources, ",")) + } + + // Add volume backup options + if defaultVolumesToFsBackup { + options = append(options, "--default-volumes-to-fs-backup") + } + + // Add snapshot move data option + if snapshotMoveData { + options = append(options, "--snapshot-move-data") + } + + // Execute CLI command + cmd := &CLICommand{ + Resource: "backup", + Action: "create", + Name: backupName, + Options: options, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Errorf("failed to create custom backup via CLI: %v, output: %s", err, string(output)) + } + + log.Printf("Custom backup created via CLI: %s", backupName) + return nil +} + +// GetBackupViaCLI gets backup details using the OADP CLI +func GetBackupViaCLI(name string) (*velero.Backup, error) { + if name == "" { + return nil, fmt.Errorf("backup name cannot be empty") + } + + // Use CLI to get backup details in JSON format + cmd := &CLICommand{ + Resource: "backup", + Action: "get", + Name: name, + Options: []string{"-o", "json"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return nil, fmt.Errorf("failed to get backup via CLI: %v", err) + } + + // Parse the JSON output back to velero.Backup struct + var backup velero.Backup + if err := json.Unmarshal(output, &backup); err != nil { + return nil, fmt.Errorf("failed to parse backup JSON: %v", err) + } + + return &backup, nil +} + +// IsBackupDoneViaCLI checks if backup is done using the OADP CLI +func IsBackupDoneViaCLI(name string) wait.ConditionFunc { + return func() (bool, error) { + cmd := &CLICommand{ + Resource: "backup", + Action: "get", + Name: name, + Options: []string{"-o", "yaml"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return false, fmt.Errorf("failed to get backup status via CLI: %v", err) + } + + phase := ParsePhaseFromYAML(string(output)) + + if len(phase) > 0 { + log.Printf("backup phase: %s", phase) + } + + return !IsBackupPhaseNotDone(phase), nil + } +} + +// IsBackupCompletedSuccessfullyViaCLI checks if backup completed successfully using the OADP CLI +func IsBackupCompletedSuccessfullyViaCLI(name string) (bool, error) { + // Use CLI to get backup status + cmd := &CLICommand{ + Resource: "backup", + Action: "get", + Name: name, + Options: []string{"-o", "yaml"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return false, fmt.Errorf("failed to get backup status via CLI: %v", err) + } + + // Parse phase from YAML output + phase := ParsePhaseFromYAML(string(output)) + + if phase == string(velero.BackupPhaseCompleted) { + return true, nil + } + + // Get additional failure information using CLI + backupLogs, logsErr := BackupLogsViaCLI(name) + if logsErr != nil { + backupLogs = fmt.Sprintf("Failed to get logs: %v", logsErr) + } + + return false, fmt.Errorf( + "backup phase is: %s; expected: %s\nvelero failure logs: %v", + phase, string(velero.BackupPhaseCompleted), backupLogs, + ) +} + +// DescribeBackupViaCLI describes backup using the OADP CLI with default timeout and retry. +// The timeout prevents the command from hanging when retrieving backup details from object storage. +func DescribeBackupViaCLI(name string) (backupDescription string) { + return DescribeBackupViaCLIWithOptions(name, DefaultCLITimeout, DefaultCLIRetries, DefaultCLIRetryDelay) +} + +// DescribeBackupViaCLIWithOptions describes backup using the OADP CLI with specified timeout and retry options. +func DescribeBackupViaCLIWithOptions(name string, timeout time.Duration, maxRetries int, retryDelay time.Duration) (backupDescription string) { + // Use CLI to describe backup + cmd := &CLICommand{ + Resource: "backup", + Action: "describe", + Name: name, + Options: []string{"--details"}, + } + + var output []byte + var err error + + if maxRetries > 1 { + output, err = cmd.ExecuteWithTimeoutAndRetry(timeout, maxRetries, retryDelay) + } else { + output, err = cmd.ExecuteWithTimeout(timeout) + } + + if err != nil { + return fmt.Sprintf("could not describe backup via CLI: %v, output: %s", err, string(output)) + } + + return string(output) +} + +// BackupLogsViaCLI gets backup logs using the OADP CLI with default timeout and retry. +// The timeout prevents the command from hanging indefinitely when streaming logs from object storage. +// Retry logic helps handle transient network issues. +func BackupLogsViaCLI(name string) (backupLogs string, err error) { + return BackupLogsViaCLIWithOptions(name, DefaultCLITimeout, DefaultCLIRetries, DefaultCLIRetryDelay) +} + +// BackupLogsViaCLIWithTimeout gets backup logs using the OADP CLI with a specified timeout (no retry). +func BackupLogsViaCLIWithTimeout(name string, timeout time.Duration) (backupLogs string, err error) { + return BackupLogsViaCLIWithOptions(name, timeout, 1, 0) +} + +// BackupLogsViaCLIWithOptions gets backup logs using the OADP CLI with specified timeout and retry options. +func BackupLogsViaCLIWithOptions(name string, timeout time.Duration, maxRetries int, retryDelay time.Duration) (backupLogs string, err error) { + if name == "" { + return "", fmt.Errorf("backup name cannot be empty") + } + + // Use CLI to get backup logs with timeout and retry to prevent hanging + cmd := &CLICommand{ + Resource: "backup", + Action: "logs", + Name: name, + Options: []string{}, + } + + var output []byte + var cmdErr error + + if maxRetries > 1 { + output, cmdErr = cmd.ExecuteOutputWithTimeoutAndRetry(timeout, maxRetries, retryDelay) + } else { + output, cmdErr = cmd.ExecuteOutputWithTimeout(timeout) + } + + if cmdErr != nil { + return "", fmt.Errorf("failed to get backup logs via CLI: %v", cmdErr) + } + + return string(output), nil +} + +// BackupErrorLogsViaCLI gets backup error logs using the OADP CLI +func BackupErrorLogsViaCLI(name string) []string { + bl, err := BackupLogsViaCLI(name) + if err != nil { + return []string{err.Error()} + } + return errorLogsExcludingIgnored(bl) +} + +// DeleteBackupViaCLI deletes a backup using the OADP CLI +func DeleteBackupViaCLI(name string) error { + if name == "" { + return fmt.Errorf("backup name cannot be empty") + } + + // Use CLI to delete backup + cmd := &CLICommand{ + Resource: "backup", + Action: "delete", + Name: name, + Options: []string{}, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Errorf("failed to delete backup via CLI: %v, output: %s", err, string(output)) + } + + log.Printf("Backup deleted via CLI: %s", name) + return nil +} diff --git a/tests/e2e/lib/cli_common.go b/tests/e2e/lib/cli_common.go new file mode 100644 index 00000000000..cec5bdf1501 --- /dev/null +++ b/tests/e2e/lib/cli_common.go @@ -0,0 +1,340 @@ +package lib + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// Default timeout for CLI commands that may hang (e.g., log streaming) +const DefaultCLITimeout = 5 * time.Minute + +// Default retry settings for CLI commands +const ( + DefaultCLIRetries = 3 + DefaultCLIRetryDelay = 10 * time.Second +) + +type CLICommand struct { + Resource string // "backup" or "restore" + Action string // "create", "get", "delete", etc. + Name string + Options []string + Timeout time.Duration // Optional timeout for commands that may hang +} + +func (c *CLICommand) Execute() ([]byte, error) { + args := []string{"oadp", c.Resource, c.Action} + if c.Name != "" { + args = append(args, c.Name) + } + args = append(args, c.Options...) + + c.LogCLICommand() + cmd := exec.Command("kubectl", args...) + return cmd.CombinedOutput() +} + +func (c *CLICommand) ExecuteOutput() ([]byte, error) { + args := []string{"oadp", c.Resource, c.Action} + if c.Name != "" { + args = append(args, c.Name) + } + args = append(args, c.Options...) + + c.LogCLICommand() + cmd := exec.Command("kubectl", args...) + return cmd.Output() +} + +// ExecuteWithTimeout executes the CLI command with a timeout. +// If the timeout is exceeded, the command is killed and an error is returned. +func (c *CLICommand) ExecuteWithTimeout(timeout time.Duration) ([]byte, error) { + args := []string{"oadp", c.Resource, c.Action} + if c.Name != "" { + args = append(args, c.Name) + } + args = append(args, c.Options...) + + c.LogCLICommand() + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, "kubectl", args...) + output, err := cmd.CombinedOutput() + + if ctx.Err() == context.DeadlineExceeded { + return output, fmt.Errorf("command timed out after %v: kubectl %s", timeout, strings.Join(args, " ")) + } + + return output, err +} + +// ExecuteOutputWithTimeout executes the CLI command with a timeout and returns stdout only. +// If the timeout is exceeded, the command is killed and an error is returned. +func (c *CLICommand) ExecuteOutputWithTimeout(timeout time.Duration) ([]byte, error) { + args := []string{"oadp", c.Resource, c.Action} + if c.Name != "" { + args = append(args, c.Name) + } + args = append(args, c.Options...) + + c.LogCLICommand() + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, "kubectl", args...) + output, err := cmd.Output() + + if ctx.Err() == context.DeadlineExceeded { + return output, fmt.Errorf("command timed out after %v: kubectl %s", timeout, strings.Join(args, " ")) + } + + return output, err +} + +// ExecuteOutputWithTimeoutAndRetry executes the CLI command with a timeout and retry logic. +// It retries the command up to maxRetries times with a delay between attempts. +// This is useful for commands that may fail due to transient issues (e.g., network problems). +func (c *CLICommand) ExecuteOutputWithTimeoutAndRetry(timeout time.Duration, maxRetries int, retryDelay time.Duration) ([]byte, error) { + var lastErr error + var lastOutput []byte + + for attempt := 1; attempt <= maxRetries; attempt++ { + output, err := c.ExecuteOutputWithTimeout(timeout) + if err == nil { + return output, nil + } + + lastErr = err + lastOutput = output + + if attempt < maxRetries { + log.Printf("CLI command failed (attempt %d/%d): %v. Retrying in %v...", attempt, maxRetries, err, retryDelay) + time.Sleep(retryDelay) + } + } + + return lastOutput, fmt.Errorf("CLI command failed after %d attempts: %v", maxRetries, lastErr) +} + +// ExecuteWithTimeoutAndRetry executes the CLI command with a timeout and retry logic. +// It retries the command up to maxRetries times with a delay between attempts. +func (c *CLICommand) ExecuteWithTimeoutAndRetry(timeout time.Duration, maxRetries int, retryDelay time.Duration) ([]byte, error) { + var lastErr error + var lastOutput []byte + + for attempt := 1; attempt <= maxRetries; attempt++ { + output, err := c.ExecuteWithTimeout(timeout) + if err == nil { + return output, nil + } + + lastErr = err + lastOutput = output + + if attempt < maxRetries { + log.Printf("CLI command failed (attempt %d/%d): %v. Retrying in %v...", attempt, maxRetries, err, retryDelay) + time.Sleep(retryDelay) + } + } + + return lastOutput, fmt.Errorf("CLI command failed after %d attempts: %v", maxRetries, lastErr) +} + +func (c *CLICommand) LogCLICommand() { + args := []string{"kubectl", "oadp", c.Resource, c.Action} + if c.Name != "" { + args = append(args, c.Name) + } + args = append(args, c.Options...) + log.Printf("Executing CLI command: %s", strings.Join(args, " ")) +} + +func ParsePhaseFromYAML(yamlOutput string) string { + lines := strings.Split(yamlOutput, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "phase:") { + return strings.TrimSpace(strings.TrimPrefix(line, "phase:")) + } + } + return "" +} + +type CLISetup struct { + repoURL string + repoBranch string + installArgs []string + namespace string +} + +func NewOADPCLISetup() *CLISetup { + return &CLISetup{ + repoURL: "https://github.com/migtools/oadp-cli.git", + repoBranch: "oadp-1.4", + installArgs: []string{"build"}, + namespace: "openshift-adp", + } +} + +func (c *CLISetup) Install() error { + tmpDir, err := c.createTempDir() + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + cloneDir := filepath.Join(tmpDir, "oadp-cli") + + steps := []struct { + name string + fn func() error + }{ + {"Cloning repository", func() error { return c.cloneRepo(cloneDir) }}, + {"Building and installing", func() error { return c.buildAndInstall(cloneDir) }}, + {"Verifying installation", func() error { return c.verifyInstallation() }}, + {"Configuring namespace", func() error { return c.configureNamespace() }}, + } + + for _, step := range steps { + log.Printf("OADP CLI Setup: %s...", step.name) + if err := step.fn(); err != nil { + return fmt.Errorf("%s failed: %w", step.name, err) + } + } + + log.Print("OADP CLI setup completed successfully") + return nil +} + +func (c *CLISetup) createTempDir() (string, error) { + tmpDir, err := os.MkdirTemp("", "oadp-cli-*") + if err != nil { + return "", fmt.Errorf("failed to create temp directory: %w", err) + } + return tmpDir, nil +} + +func (c *CLISetup) cloneRepo(cloneDir string) error { + return runCommand("git", []string{"clone", "--branch", c.repoBranch, "--depth", "1", c.repoURL, cloneDir}, "") +} + +func (c *CLISetup) buildAndInstall(cloneDir string) error { + // Build the binary + log.Print("Building OADP CLI...") + if err := runCommand("make", c.installArgs, cloneDir); err != nil { + return fmt.Errorf("build failed: %w", err) + } + + // Verify the binary was created + binaryPath := filepath.Join(cloneDir, "kubectl-oadp") + if _, err := os.Stat(binaryPath); err != nil { + return fmt.Errorf("kubectl-oadp binary not found at %s: %w", binaryPath, err) + } + + // Try multiple target locations, starting with user-writable paths + targetPaths := []string{ + fmt.Sprintf("%s/bin/kubectl-oadp", os.Getenv("HOME")), + "/usr/local/bin/kubectl-oadp", + } + + var targetPath string + var moveErr error + + for _, tp := range targetPaths { + targetPath = tp + log.Printf("Attempting to move binary from %s to %s", binaryPath, targetPath) + + // Create directory if it doesn't exist (for ~/bin) + if targetPath == fmt.Sprintf("%s/bin/kubectl-oadp", os.Getenv("HOME")) { + binDir := filepath.Dir(targetPath) + if err := os.MkdirAll(binDir, 0755); err != nil { + log.Printf("Failed to create directory %s: %v", binDir, err) + continue + } + } + + if err := runCommand("mv", []string{binaryPath, targetPath}, ""); err != nil { + log.Printf("Failed to move to %s: %v", targetPath, err) + moveErr = err + continue + } + + // Success! + moveErr = nil + break + } + + if moveErr != nil { + return fmt.Errorf("failed to move binary to any location: %w", moveErr) + } + + // Make it executable + if err := runCommand("chmod", []string{"+x", targetPath}, ""); err != nil { + return fmt.Errorf("failed to make binary executable: %w", err) + } + + // If we installed to ~/bin, ensure it's in PATH + if targetPath == fmt.Sprintf("%s/bin/kubectl-oadp", os.Getenv("HOME")) { + homeBin := fmt.Sprintf("%s/bin", os.Getenv("HOME")) + currentPath := os.Getenv("PATH") + if !strings.Contains(currentPath, homeBin) { + newPath := fmt.Sprintf("%s:%s", homeBin, currentPath) + os.Setenv("PATH", newPath) + log.Printf("Added %s to PATH", homeBin) + } + } + + log.Printf("Successfully installed kubectl-oadp to %s", targetPath) + return nil +} + +func (c *CLISetup) verifyInstallation() error { + // Check current PATH + cmd := exec.Command("bash", "-c", "echo $PATH") + if output, err := cmd.CombinedOutput(); err == nil { + log.Printf("Current PATH: %s", string(output)) + } + + // Try to find kubectl-oadp binary + cmd = exec.Command("which", "kubectl-oadp") + if output, err := cmd.CombinedOutput(); err == nil { + log.Printf("kubectl-oadp found at: %s", string(output)) + } else { + log.Printf("kubectl-oadp not found in PATH: %v", err) + } + + // List kubectl plugins + cmd = exec.Command("kubectl", "plugin", "list") + if output, err := cmd.CombinedOutput(); err == nil { + log.Printf("Available kubectl plugins: %s", string(output)) + } + + return runCommand("kubectl", []string{"oadp", "version"}, "") +} + +func (c *CLISetup) configureNamespace() error { + return runCommand("kubectl", []string{"oadp", "client", "config", "set", fmt.Sprintf("namespace=%s", c.namespace)}, "") +} + +func runCommand(name string, args []string, dir string) error { + cmd := exec.Command(name, args...) + if dir != "" { + cmd.Dir = dir + } + + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("command '%s %s' failed: %v\nOutput: %s", + name, strings.Join(args, " "), err, string(output)) + } + return nil +} diff --git a/tests/e2e/lib/restore.go b/tests/e2e/lib/restore.go index b3a80bcaf9e..d8ce1309ccd 100755 --- a/tests/e2e/lib/restore.go +++ b/tests/e2e/lib/restore.go @@ -42,6 +42,28 @@ func GetRestore(c client.Client, namespace string, name string) (*velero.Restore return &restore, nil } +// restorePhasesNotDone is the shared list of restore phases that indicate a restore is still in progress. +var restorePhasesNotDone = []velero.RestorePhase{ + velero.RestorePhaseNew, + "ReadyToStart", + velero.RestorePhaseInProgress, + velero.RestorePhaseWaitingForPluginOperations, + velero.RestorePhaseWaitingForPluginOperationsPartiallyFailed, + velero.RestorePhaseFinalizing, + velero.RestorePhaseFinalizingPartiallyFailed, + "", +} + +// IsRestorePhaseNotDone returns true if the given phase string represents a restore still in progress. +func IsRestorePhaseNotDone(phase string) bool { + for _, notDonePhase := range restorePhasesNotDone { + if phase == string(notDonePhase) { + return true + } + } + return false +} + func IsRestoreDone(ocClient client.Client, veleroNamespace, name string) wait.ConditionFunc { return func() (bool, error) { restore, err := GetRestore(ocClient, veleroNamespace, name) @@ -51,21 +73,7 @@ func IsRestoreDone(ocClient client.Client, veleroNamespace, name string) wait.Co if len(restore.Status.Phase) > 0 { log.Printf("restore phase: %s", restore.Status.Phase) } - var phasesNotDone = []velero.RestorePhase{ - velero.RestorePhaseNew, - velero.RestorePhaseInProgress, - velero.RestorePhaseWaitingForPluginOperations, - velero.RestorePhaseWaitingForPluginOperationsPartiallyFailed, - velero.RestorePhaseFinalizing, - velero.RestorePhaseFinalizingPartiallyFailed, - "", - } - for _, notDonePhase := range phasesNotDone { - if restore.Status.Phase == notDonePhase { - return false, nil - } - } - return true, nil + return !IsRestorePhaseNotDone(string(restore.Status.Phase)), nil } } diff --git a/tests/e2e/lib/restore_cli.go b/tests/e2e/lib/restore_cli.go new file mode 100644 index 00000000000..0439a3db3bd --- /dev/null +++ b/tests/e2e/lib/restore_cli.go @@ -0,0 +1,283 @@ +package lib + +import ( + "encoding/json" + "fmt" + "log" + "strings" + "time" + + velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +// CreateRestoreFromBackupViaCLI creates a restore using the OADP CLI +func CreateRestoreFromBackupViaCLI(backupName, restoreName string) error { + options := []string{"--from-backup", backupName} + + // Execute CLI command + cmd := &CLICommand{ + Resource: "restore", + Action: "create", + Name: restoreName, + Options: options, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Errorf("failed to create restore via CLI: %v, output: %s", err, string(output)) + } + + log.Printf("Restore created via CLI: %s", restoreName) + return nil +} + +// CreateRestoreFromBackupWithOptionsViaCLI creates a restore with options using the OADP CLI +func CreateRestoreFromBackupWithOptionsViaCLI(backupName, restoreName string, restoreVolumes bool, includeNamespaces, excludeNamespaces []string) error { + options := []string{"--from-backup", backupName} + + // Add restore volumes option + if restoreVolumes { + options = append(options, "--restore-volumes") + } + + // Add included namespaces + if len(includeNamespaces) > 0 { + options = append(options, "--include-namespaces", strings.Join(includeNamespaces, ",")) + } + + // Add excluded namespaces + if len(excludeNamespaces) > 0 { + options = append(options, "--exclude-namespaces", strings.Join(excludeNamespaces, ",")) + } + + // Execute CLI command + cmd := &CLICommand{ + Resource: "restore", + Action: "create", + Name: restoreName, + Options: options, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Errorf("failed to create restore with options via CLI: %v, output: %s", err, string(output)) + } + + log.Printf("Restore with options created via CLI: %s", restoreName) + return nil +} + +// GetRestoreViaCLI gets restore details using the OADP CLI +func GetRestoreViaCLI(name string) (*velero.Restore, error) { + if name == "" { + return nil, fmt.Errorf("restore name cannot be empty") + } + + // Use CLI to get restore details in JSON format + cmd := &CLICommand{ + Resource: "restore", + Action: "get", + Name: name, + Options: []string{"-o", "json"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return nil, fmt.Errorf("failed to get restore via CLI: %v", err) + } + + // Parse the JSON output back to velero.Restore struct + var restore velero.Restore + if err := json.Unmarshal(output, &restore); err != nil { + return nil, fmt.Errorf("failed to parse restore JSON: %v", err) + } + + return &restore, nil +} + +// IsRestoreDoneViaCLI checks if restore is done using the OADP CLI +func IsRestoreDoneViaCLI(name string) wait.ConditionFunc { + return func() (bool, error) { + cmd := &CLICommand{ + Resource: "restore", + Action: "get", + Name: name, + Options: []string{"-o", "yaml"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return false, fmt.Errorf("failed to get restore status via CLI: %v", err) + } + + phase := ParsePhaseFromYAML(string(output)) + + if len(phase) > 0 { + log.Printf("restore phase: %s", phase) + } + + return !IsRestorePhaseNotDone(phase), nil + } +} + +// IsRestoreCompletedSuccessfullyViaCLI checks if restore completed successfully using the OADP CLI +func IsRestoreCompletedSuccessfullyViaCLI(name string) (bool, error) { + // Use CLI to get restore status + cmd := &CLICommand{ + Resource: "restore", + Action: "get", + Name: name, + Options: []string{"-o", "yaml"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return false, fmt.Errorf("failed to get restore status via CLI: %v", err) + } + + // Parse phase from YAML output + phase := ParsePhaseFromYAML(string(output)) + + if phase == string(velero.RestorePhaseCompleted) { + return true, nil + } + + // Get additional failure information using CLI + restoreLogs, logsErr := RestoreLogsViaCLI(name) + if logsErr != nil { + restoreLogs = fmt.Sprintf("Failed to get logs: %v", logsErr) + } + + return false, fmt.Errorf( + "restore phase is: %s; expected: %s\nvelero failure logs: %v", + phase, string(velero.RestorePhaseCompleted), restoreLogs, + ) +} + +// DescribeRestoreViaCLI describes restore using the OADP CLI with default timeout and retry. +// The timeout prevents the command from hanging when retrieving restore details from object storage. +func DescribeRestoreViaCLI(name string) string { + return DescribeRestoreViaCLIWithOptions(name, DefaultCLITimeout, DefaultCLIRetries, DefaultCLIRetryDelay) +} + +// DescribeRestoreViaCLIWithOptions describes restore using the OADP CLI with specified timeout and retry options. +func DescribeRestoreViaCLIWithOptions(name string, timeout time.Duration, maxRetries int, retryDelay time.Duration) string { + // Use CLI to describe restore + cmd := &CLICommand{ + Resource: "restore", + Action: "describe", + Name: name, + Options: []string{"--details"}, + } + + var output []byte + var err error + + if maxRetries > 1 { + output, err = cmd.ExecuteWithTimeoutAndRetry(timeout, maxRetries, retryDelay) + } else { + output, err = cmd.ExecuteWithTimeout(timeout) + } + + if err != nil { + return fmt.Sprintf("could not describe restore via CLI: %v, output: %s", err, string(output)) + } + + return string(output) +} + +// RestoreLogsViaCLI gets restore logs using the OADP CLI with default timeout and retry. +// The timeout prevents the command from hanging indefinitely when streaming logs from object storage. +// Retry logic helps handle transient network issues. +func RestoreLogsViaCLI(name string) (restoreLogs string, err error) { + return RestoreLogsViaCLIWithOptions(name, DefaultCLITimeout, DefaultCLIRetries, DefaultCLIRetryDelay) +} + +// RestoreLogsViaCLIWithTimeout gets restore logs using the OADP CLI with a specified timeout (no retry). +func RestoreLogsViaCLIWithTimeout(name string, timeout time.Duration) (restoreLogs string, err error) { + return RestoreLogsViaCLIWithOptions(name, timeout, 1, 0) +} + +// RestoreLogsViaCLIWithOptions gets restore logs using the OADP CLI with specified timeout and retry options. +func RestoreLogsViaCLIWithOptions(name string, timeout time.Duration, maxRetries int, retryDelay time.Duration) (restoreLogs string, err error) { + if name == "" { + return "", fmt.Errorf("restore name cannot be empty") + } + + // Use CLI to get restore logs with timeout and retry to prevent hanging + cmd := &CLICommand{ + Resource: "restore", + Action: "logs", + Name: name, + Options: []string{}, + } + + var output []byte + var cmdErr error + + if maxRetries > 1 { + output, cmdErr = cmd.ExecuteOutputWithTimeoutAndRetry(timeout, maxRetries, retryDelay) + } else { + output, cmdErr = cmd.ExecuteOutputWithTimeout(timeout) + } + + if cmdErr != nil { + return "", fmt.Errorf("failed to get restore logs via CLI: %v", cmdErr) + } + + return string(output), nil +} + +// RestoreErrorLogsViaCLI gets restore error logs using the OADP CLI +func RestoreErrorLogsViaCLI(name string) []string { + rl, err := RestoreLogsViaCLI(name) + if err != nil { + return []string{err.Error()} + } + return errorLogsExcludingIgnored(rl) +} + +// DeleteRestoreViaCLI deletes a restore using the OADP CLI +func DeleteRestoreViaCLI(name string) error { + if name == "" { + return fmt.Errorf("restore name cannot be empty") + } + + // Use CLI to delete restore + cmd := &CLICommand{ + Resource: "restore", + Action: "delete", + Name: name, + Options: []string{}, + } + output, err := cmd.Execute() + if err != nil { + return fmt.Errorf("failed to delete restore via CLI: %v, output: %s", err, string(output)) + } + + log.Printf("Restore deleted via CLI: %s", name) + return nil +} + +// ListRestoresViaCLI lists all restores using the OADP CLI +func ListRestoresViaCLI() ([]string, error) { + // Use CLI to list restores + cmd := &CLICommand{ + Resource: "restore", + Action: "list", + Name: "", // list commands don't require a name + Options: []string{"-o", "name"}, + } + output, err := cmd.ExecuteOutput() + if err != nil { + return nil, fmt.Errorf("failed to list restores via CLI: %v", err) + } + + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + var restores []string + for _, line := range lines { + if line != "" { + // Remove "restore.velero.io/" prefix if present + restore := strings.TrimPrefix(line, "restore.velero.io/") + restores = append(restores, restore) + } + } + + return restores, nil +}