bundle migrate: handle added/removed resources gracefully

NEXT_CHANGELOG.md

@@ -9,6 +9,7 @@
 ### Bundles
 * Set the default `data_security_mode` to `DATA_SECURITY_MODE_AUTO` in bundle templates ([#5452](https://github.com/databricks/cli/pull/5452)).
 * Mark vector search index index_subtype as backend_default to prevent drift after deployment ([#5454](https://github.com/databricks/cli/pull/5454)).
+* `bundle deployment migrate`: handle resources added to or removed from `databricks.yml` since the last Terraform deploy ([#5463](https://github.com/databricks/cli/pull/5463)).
 
 ### Dependency updates
 

acceptance/bundle/migrate/added/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: migrate-added-test
+
+resources:
+  jobs:
+    job_a: {name: "Job A"}
+    #job_b: {name: "Job B"}

acceptance/bundle/migrate/added/output.txt

@@ -0,0 +1,61 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-added-test/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> musterr [CLI] bundle deployment migrate
+Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
+create jobs.job_b
+
+Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged
+Error: 'databricks bundle plan' shows actions planned, aborting migration. Please run 'databricks bundle deploy' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option
+
+>>> [CLI] bundle deployment migrate --noplancheck
+Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json
+
+Validate the migration by running "databricks bundle plan", there should be no actions planned.
+
+The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy".
+
+To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/default/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate
+
+
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
+create jobs.job_b
+
+Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-added-test/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //jobs/create
+{
+  "headers": {
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat"
+    ]
+  },
+  "method": "POST",
+  "path": "/api/2.2/jobs/create",
+  "body": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-added-test/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "Job B",
+    "queue": {
+      "enabled": true
+    }
+  }
+}
+
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged

acceptance/bundle/migrate/added/script

@@ -0,0 +1,26 @@
+export DATABRICKS_BUNDLE_ENGINE=terraform
+
+# Deploy with terraform (only job_a; job_b is commented out)
+trace $CLI bundle deploy
+
+# Uncomment job_b (add it to config without deploying)
+update_file.py databricks.yml "#job_b" "job_b"
+
+# Should fail at plan check: job_b is "1 to add"
+trace musterr $CLI bundle deployment migrate
+
+# Should succeed: job_b skipped, will be created on next deploy
+trace $CLI bundle deployment migrate --noplancheck
+
+# After migration: plan shows job_b as "to add"
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 to add"
+
+# Deploy creates job_b; verify via recorded requests
+rm out.requests.txt
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy
+trace print_requests.py //jobs/create
+
+# No further actions planned
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "2 unchanged"
+
+rm out.requests.txt

acceptance/bundle/migrate/removed/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: migrate-removed-test
+
+resources:
+  jobs:
+    job_a: {name: "Job A"}
+    job_b: {name: "Job B"}

acceptance/bundle/migrate/removed/output.txt

@@ -0,0 +1,51 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-removed-test/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> musterr [CLI] bundle deployment migrate
+Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:
+delete jobs.job_b
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+Error: 'databricks bundle plan' shows actions planned, aborting migration. Please run 'databricks bundle deploy' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option
+
+>>> [CLI] bundle deployment migrate --noplancheck
+Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json
+
+Validate the migration by running "databricks bundle plan", there should be no actions planned.
+
+The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy".
+
+To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/default/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate
+
+
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
+delete jobs.job_b
+
+Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged
+
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/migrate-removed-test/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //jobs/delete
+{
+  "headers": {
+    "User-Agent": [
+      "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat"
+    ]
+  },
+  "method": "POST",
+  "path": "/api/2.2/jobs/delete",
+  "body": {
+    "job_id": [NUMID]
+  }
+}
+
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged

acceptance/bundle/migrate/removed/script

@@ -0,0 +1,26 @@
+export DATABRICKS_BUNDLE_ENGINE=terraform
+
+# Deploy with terraform (both job_a and job_b)
+trace $CLI bundle deploy
+
+# Remove job_b from config without deploying the deletion
+grep -v job_b databricks.yml > databricks_tmp.yml && mv databricks_tmp.yml databricks.yml
+
+# Should fail at plan check: job_b is "1 to delete"
+trace musterr $CLI bundle deployment migrate
+
+# Should succeed: job_b's ID preserved in direct state for deletion on next deploy
+trace $CLI bundle deployment migrate --noplancheck
+
+# After migration: plan shows job_b as "to delete"
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 to delete"
+
+# Deploy deletes job_b; verify via recorded requests
+rm out.requests.txt
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy
+trace print_requests.py //jobs/delete
+
+# No further actions planned
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 unchanged"
+
+rm out.requests.txt

acceptance/bundle/migrate/test.toml

@@ -4,4 +4,4 @@ IncludeRequestHeaders = ["User-Agent"]
 Ignore = [".databricks"]
 
 # All tests explicitly set DATABRICKS_BUNDLE_ENGINE, so there is no need for matrix testing
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = []

bundle/direct/bundle_apply.go

@@ -83,8 +83,18 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa
 
 		if action == deployplan.Delete {
 			if migrateMode {
-				logdiag.LogError(ctx, fmt.Errorf("%s: Unexpected delete action during migration", errorPrefix))
-				return false
+				// Resource is in terraform state but not in config. Preserve its ID in
+				// direct state so the next direct deploy will plan and execute deletion.
+				id := b.StateDB.GetResourceID(resourceKey)
+				if id == "" {
+					logdiag.LogError(ctx, fmt.Errorf("%s: internal error: no ID in state", errorPrefix))
+					return false
+				}
+				if err = b.StateDB.SaveState(resourceKey, id, json.RawMessage("{}"), entry.DependsOn); err != nil {
+					logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err))
+					return false
+				}
+				return true
 			}
 			err = d.Destroy(ctx, &b.StateDB)
 			if err != nil {

cmd/bundle/deployment/migrate.go

@@ -251,8 +251,20 @@ To start using direct engine, set "engine: direct" under bundle in your databric
 		}
 
 		for _, entry := range plan.Plan {
-			// Force all actions to be "update" so that deploym below goes through every resource
-			entry.Action = deployplan.Update
+			switch entry.Action {
+			case deployplan.Create:
+				// Resource is in config but not in terraform state; skip it during migration.
+				// It will be created on the first direct deploy.
+				entry.Action = deployplan.Skip
+			case deployplan.Delete:
+				// Resource is in terraform state but not in config. Keep as Delete so the
+				// apply migrate path can preserve its ID in direct state, allowing the next
+				// direct deploy to remove it.
+			default:
+				// Force existing resources to Update so migration reads their remote state
+				// and writes a full config snapshot.
+				entry.Action = deployplan.Update
+			}
 		}
 
 		// We need to copy ETag into new state.