From 99b7b80c2c6f2c581b45845601d7d14a0b1edd27 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Jun 2026 15:56:04 +0200 Subject: [PATCH 01/16] bundle migrate: add no-API-call migration from TF to direct engine Implement a new `bundle migrate` command that creates the direct state file from local config + Terraform state attributes, without making any API calls. Unlike `bundle deployment migrate` which calls DoRead for each resource, this command reads resolved field values directly from the local TF state file. Cross-resource references (e.g. ${resources.jobs.src.git_source[0].branch}) are resolved using two independent methods and the results reconciled: - Method A: look up the field in the TF state of the resource that contains the reference (e.g. read name from databricks_job.dst TF attributes). - Method B: evaluate the template by reading each ${resources.*} reference from the TF state of the referenced resource and interpolating. If both methods agree, the value is used silently. If only one succeeds, that value is used. If both succeed but disagree, the longer string is used with a warning. If both fail, an error is returned. The bundle/migrate package provides: - ParseTFStateAttrs: parses the full TF state file (all resource attributes) - LookupTFField: looks up a field value using DABsPathToTerraform translation - ResolveFieldRef: reconciles Methods A and B for a single field Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 6 + bundle/migrate/resolve.go | 91 ++++++++ bundle/migrate/tf_state.go | 107 +++++++++ cmd/bundle/bundle.go | 1 + cmd/bundle/deployment/migrate.go | 12 + cmd/bundle/migrate.go | 372 +++++++++++++++++++++++++++++++ 6 files changed, 589 insertions(+) create mode 100644 bundle/migrate/resolve.go create mode 100644 bundle/migrate/tf_state.go create mode 100644 cmd/bundle/migrate.go diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 5591626cd75..79e0b05648a 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -931,6 +931,12 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root return p, nil } +// ExtractReferences extracts all variable references from the config subtree rooted at node. +// Returns a map from structpath string (field path within the resource) to template string. +func ExtractReferences(root dyn.Value, node string) (map[string]string, error) { + return extractReferences(root, node) +} + func extractReferences(root dyn.Value, node string) (map[string]string, error) { nodeType := config.GetResourceTypeFromKey(node) refs := make(map[string]string) diff --git a/bundle/migrate/resolve.go b/bundle/migrate/resolve.go new file mode 100644 index 00000000000..8d5f6bc6b35 --- /dev/null +++ b/bundle/migrate/resolve.go @@ -0,0 +1,91 @@ +package migrate + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" +) + +// evaluateTemplate evaluates a template string like "${resources.pipelines.bar.cluster[0].label}" +// by looking up each ${...} reference from TF state. +func evaluateTemplate(state TFStateAttrs, template string) (string, error) { + ref, ok := dynvar.NewRef(dyn.V(template)) + if !ok { + return template, nil + } + + result := template + for _, pathString := range ref.References() { + path, err := structpath.ParsePath(pathString) + if err != nil { + return "", fmt.Errorf("cannot parse reference path %q: %w", pathString, err) + } + // Expect resources... + if path.Len() < 4 { + return "", fmt.Errorf("unexpected reference format (too short): %q", pathString) + } + // Check first component is "resources" + firstNode := path.Prefix(1) + if firstNode.String() != "resources" { + return "", fmt.Errorf("unexpected reference format (expected resources.*): %q", pathString) + } + + group := path.SkipPrefix(1).Prefix(1).String() + name := path.SkipPrefix(2).Prefix(1).String() + fieldPath := path.SkipPrefix(3) + + value, err := LookupTFField(state, group, name, fieldPath) + if err != nil { + return "", fmt.Errorf("cannot look up %q: %w", pathString, err) + } + + result = strings.ReplaceAll(result, "${"+pathString+"}", fmt.Sprintf("%v", value)) + } + return result, nil +} + +// ResolveFieldRef resolves a single reference for a field in resource (srcGroup, srcName). +// fieldPath is the path of the field within the source resource (in DABs naming, from sv.Refs key). +// refTemplate is the template string for that field, e.g. "${resources.pipelines.bar.cluster[0].label}". +// +// Two methods are tried: +// - Method A: read the field from the source resource's own TF state. +// - Method B: evaluate the template by reading each referenced field from TF state. +// +// Returns the reconciled value or an error if both methods fail. +func ResolveFieldRef(ctx context.Context, state TFStateAttrs, srcGroup, srcName string, fieldPath *structpath.PathNode, refTemplate string) (any, error) { + // Method A: read field from source resource's TF state. + valueA, errA := LookupTFField(state, srcGroup, srcName, fieldPath) + + // Method B: evaluate the template by looking up each reference. + valueB, errB := evaluateTemplate(state, refTemplate) + + switch { + case errA == nil && errB == nil: + aStr := fmt.Sprintf("%v", valueA) + if aStr == valueB { + return valueA, nil + } + // Both succeeded but disagree: prefer longer string and warn. + if len(valueB) > len(aStr) { + log.Warnf(ctx, "resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method B)", + srcGroup, srcName, fieldPath, aStr, valueB) + return valueB, nil + } + log.Warnf(ctx, "resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method A)", + srcGroup, srcName, fieldPath, aStr, valueB) + return valueA, nil + case errA == nil: + return valueA, nil + case errB == nil: + return valueB, nil + default: + return nil, fmt.Errorf("%s.%s field %s: method A: %w; method B: %w", + srcGroup, srcName, fieldPath, errA, errB) + } +} diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go new file mode 100644 index 00000000000..642f7a26b74 --- /dev/null +++ b/bundle/migrate/tf_state.go @@ -0,0 +1,107 @@ +package migrate + +import ( + "encoding/json" + "fmt" + "os" + "reflect" + "strings" + "sync" + + "github.com/databricks/cli/bundle/deploy/terraform" + tfschema "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/bundle/terraform_dabs_map" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + tfjson "github.com/hashicorp/terraform-json" +) + +// TFStateAttrs maps (tfResourceType → resourceName → raw JSON attributes). +type TFStateAttrs map[string]map[string]json.RawMessage + +// ParseTFStateAttrs parses the terraform state file returning full attribute JSON per resource. +func ParseTFStateAttrs(path string) (TFStateAttrs, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var state struct { + Version int `json:"version"` + Resources []struct { + Type string `json:"type"` + Name string `json:"name"` + Mode tfjson.ResourceMode `json:"mode"` + Instances []struct { + Attributes json.RawMessage `json:"attributes"` + } `json:"instances"` + } `json:"resources"` + } + + if err := json.Unmarshal(raw, &state); err != nil { + return nil, err + } + + result := make(TFStateAttrs) + for _, r := range state.Resources { + if r.Mode != tfjson.ManagedResourceMode || len(r.Instances) == 0 { + continue + } + if result[r.Type] == nil { + result[r.Type] = make(map[string]json.RawMessage) + } + result[r.Type][r.Name] = r.Instances[0].Attributes + } + return result, nil +} + +// tfSchemaTypeMap maps TF resource type name → schema struct type (via AllResources json tags). +var tfSchemaTypeMap = sync.OnceValue(func() map[string]reflect.Type { + t := reflect.TypeOf(tfschema.AllResources{}) + m := make(map[string]reflect.Type, t.NumField()) + for i := range t.NumField() { + f := t.Field(i) + tag := strings.Split(f.Tag.Get("json"), ",")[0] + if tag != "" && tag != "-" { + m[tag] = f.Type + } + } + return m +}) + +// LookupTFField looks up a field from TF state attributes for a bundle resource. +// group is the DABs group (e.g. "pipelines"), name is the resource name. +// fieldPath is the path to the field (may be in DABs or TF naming; both handled by DABsPathToTerraform). +// Returns (nil, nil) for empty/zero fields, error if the resource or field is not found. +func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath.PathNode) (any, error) { + tfType, ok := terraform.GroupToTerraformName[group] + if !ok { + return nil, fmt.Errorf("unknown resource group %q", group) + } + + // Translate field path to TF naming. + // DABsPathToTerraform handles both DABs names (renames) and TF names (pass-through for unknowns). + // Returns error for known DABs-only fields that have no TF equivalent. + tfFieldPath, err := terraform_dabs_map.DABsPathToTerraform(group, fieldPath) + if err != nil { + return nil, err + } + + attrsJSON, ok := state[tfType][name] + if !ok { + return nil, fmt.Errorf("%s.%s not found in TF state", tfType, name) + } + + schemaType, ok := tfSchemaTypeMap()[tfType] + if !ok { + return nil, fmt.Errorf("no schema type registered for %q", tfType) + } + + // Unmarshal attributes into a new instance of the schema struct. + ptr := reflect.New(schemaType) + if err := json.Unmarshal(attrsJSON, ptr.Interface()); err != nil { + return nil, fmt.Errorf("cannot parse TF state for %s.%s: %w", tfType, name, err) + } + + return structaccess.Get(ptr.Interface(), tfFieldPath) +} diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index 7189b1d431d..84666eec689 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -38,6 +38,7 @@ Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.htm cmd.AddCommand(newDebugCommand()) cmd.AddCommand(newOpenCommand()) cmd.AddCommand(newPlanCommand()) + cmd.AddCommand(newMigrateCommand()) cmd.AddCommand(newConfigRemoteSyncCommand()) // Bundle Metadata Service (DMS) read-only command groups. Only `get` diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 801e46f7918..aa22e5118ac 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -30,6 +30,12 @@ import ( const backupSuffix = ".backup" +// RunPlanCheck runs bundle plan and checks if there are any actions planned. +// Returns error if plan fails or if there are actions planned. +func RunPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { + return runPlanCheck(cmd, extraArgs, extraArgsStr) +} + // runPlanCheck runs bundle plan and checks if there are any actions planned. // Returns error if plan fails or if there are actions planned. func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { @@ -78,6 +84,12 @@ func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) e return nil } +// GetCommonArgs extracts common flags (target, profile, var) from the command into +// argument slices suitable for forwarding to a subprocess. +func GetCommonArgs(cmd *cobra.Command) ([]string, string) { + return getCommonArgs(cmd) +} + func getCommonArgs(cmd *cobra.Command) ([]string, string) { var args []string var quotedArgs []string diff --git a/cmd/bundle/migrate.go b/cmd/bundle/migrate.go new file mode 100644 index 00000000000..3579658d3fe --- /dev/null +++ b/cmd/bundle/migrate.go @@ -0,0 +1,372 @@ +package bundle + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/migrate" + "github.com/databricks/cli/cmd/bundle/deployment" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/spf13/cobra" +) + +func newMigrateCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "migrate", + Short: "Migrate from Terraform to Direct deployment engine (no API calls)", + Long: `Creates a Direct deployment state file from the local config and Terraform state, +without making API calls. Cross-resource references are resolved from TF state.`, + Args: root.NoArgs, + } + + var noPlanCheck bool + cmd.Flags().BoolVar(&noPlanCheck, "noplancheck", false, "Skip running bundle plan before migration.") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + // Clear engine env var: we read TF state and produce a direct state. + cmd.SetContext(env.Set(cmd.Context(), engine.EnvVar, "")) + + opts := utils.ProcessOptions{ + AlwaysPull: true, + FastValidate: true, + Build: true, + PostInitFunc: func(_ context.Context, b *bundle.Bundle) error { + if b.Config.Bundle.Engine == engine.EngineTerraform { + return fmt.Errorf("bundle.engine is set to %q; migration requires \"engine: direct\" or no engine setting", engine.EngineTerraform) + } + return nil + }, + } + + b, stateDesc, err := utils.ProcessBundleRet(cmd, opts) + if err != nil { + return err + } + ctx := cmd.Context() + + if stateDesc.Lineage == "" { + cmdio.LogString(ctx, `Error: no existing state found. To start fresh with direct engine, set "engine: direct".`) + return root.ErrAlreadyPrinted + } + if stateDesc.Engine.IsDirect() { + return fmt.Errorf("already using direct engine: %s", stateDesc.String()) + } + + _, localTerraformPath := b.StateFilenameTerraform(ctx) + if _, err := os.Stat(localTerraformPath); err != nil { + return fmt.Errorf("reading %s: %w", localTerraformPath, err) + } + + // Run plan check unless --noplancheck is set. + if !noPlanCheck { + cmdio.LogString(ctx, "Note: Migration should be done after a full deploy. Running plan now to verify:") + extraArgs, extraArgsStr := deployment.GetCommonArgs(cmd) + if err := deployment.RunPlanCheck(cmd, extraArgs, extraArgsStr); err != nil { + return err + } + } + + // Parse TF state: IDs (for state entries) and full attributes (for ref resolution). + tfResourceIDs, err := terraform.ParseResourcesState(ctx, b) + if err != nil { + return fmt.Errorf("failed to parse terraform state: %w", err) + } + for key, entry := range tfResourceIDs { + if entry.ID == "" { + return fmt.Errorf("missing ID for %s in terraform state", key) + } + } + + cacheDir, err := terraform.Dir(ctx, b) + if err != nil { + return err + } + tfStateFilename, _ := b.StateFilenameTerraform(ctx) + tfStateFullPath := filepath.Join(cacheDir, tfStateFilename) + + tfAttrs, err := migrate.ParseTFStateAttrs(tfStateFullPath) + if err != nil { + return fmt.Errorf("failed to read terraform state attributes: %w", err) + } + + _, localPath := b.StateFilenameDirect(ctx) + tempPath := localPath + ".temp-migration" + + if _, err := os.Stat(tempPath); err == nil { + return fmt.Errorf("temporary state file %s already exists, another migration is in progress or was interrupted. In the latter case, delete the file", tempPath) + } + if _, err := os.Stat(localPath); err == nil { + return fmt.Errorf("state file %s already exists", localPath) + } + + // Apply SecretScopeFixups so the config matches what the direct engine expects. + // This adds MANAGE ACL for the current user to all secret scopes, ensuring + // the migrated state and config agree on .permissions entries. + bundle.ApplyContext(ctx, b, resourcemutator.SecretScopeFixups(engine.EngineDirect)) + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + // Build initial state with IDs and optional ETags. + etags := map[string]string{} + state := make(map[string]dstate.ResourceEntry) + for key, resourceEntry := range tfResourceIDs { + state[key] = dstate.ResourceEntry{ + ID: resourceEntry.ID, + State: json.RawMessage("{}"), + } + if resourceEntry.ETag != "" { + etags[key] = resourceEntry.ETag + } + } + + migratedDB := dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1) + migratedDB.State = state + + var stateDB dstate.DeploymentState + stateDB.OpenWithData(tempPath, migratedDB) + + removeTempPath := true + defer func() { + if removeTempPath { + _ = os.Remove(tempPath) + } + }() + + // Initialize adapters. + adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) + if err != nil { + return err + } + + if err := stateDB.UpgradeToWrite(); err != nil { + return fmt.Errorf("upgrading state for write: %w", err) + } + + // Process each resource: prepare state, resolve refs from TF state, save. + if err := buildStateFromTF(ctx, b, adapters, &stateDB, tfAttrs, tfResourceIDs, etags); err != nil { + return err + } + + if _, err := stateDB.Finalize(ctx); err != nil { + return fmt.Errorf("finalizing state: %w", err) + } + if logdiag.HasError(ctx) { + return errors.New("migration encountered errors") + } + + if err := os.Rename(tempPath, localPath); err != nil { + return fmt.Errorf("renaming %s to %s: %w", tempPath, localPath, err) + } + removeTempPath = false + + localTerraformBackupPath := localTerraformPath + ".backup" + err = os.Rename(localTerraformPath, localTerraformBackupPath) + if err != nil { + // Not fatal since we've already incremented the serial. + logdiag.LogError(ctx, err) + } + + extraArgsStr := "" + if flag := cmd.Flag("target"); flag != nil && flag.Changed { + extraArgsStr = " -t " + flag.Value.String() + } + + cmdio.LogString(ctx, fmt.Sprintf(`Success! Migrated %d resources to direct engine state file: %s + +Validate the migration by running "databricks bundle plan%s", there should be no actions planned. + +The state file is not synchronized to the workspace yet. To finalize the migration, run "bundle deploy%s". + +To undo the migration, remove %s and rename %s to %s +`, len(state), localPath, extraArgsStr, extraArgsStr, localPath, localTerraformBackupPath, localTerraformPath)) + return nil + } + + return cmd +} + +// buildStateFromTF iterates over bundle resources, resolves cross-resource +// references using TF state attributes, and writes each resource's state entry. +func buildStateFromTF( + ctx context.Context, + b *bundle.Bundle, + adapters map[string]*dresources.Adapter, + stateDB *dstate.DeploymentState, + tfAttrs migrate.TFStateAttrs, + tfIDs terraform.ExportedResourcesMap, + etags map[string]string, +) error { + configRoot := &b.Config + + // Collect all resource nodes (same patterns as makePlan). + var nodes []string + patterns := []dyn.Pattern{ + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + } + for _, pat := range patterns { + _, err := dyn.MapByPattern( + configRoot.Value(), + pat, + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + nodes = append(nodes, p.String()) + return dyn.InvalidValue, nil + }, + ) + if err != nil { + return err + } + } + + for _, node := range nodes { + idEntry, ok := tfIDs[node] + if !ok { + // Resource is in config but not in TF state (new resource); skip. + continue + } + + group := config.GetResourceTypeFromKey(node) + if group == "" { + return fmt.Errorf("cannot determine resource type for %q", node) + } + + adapter, ok := adapters[group] + if !ok { + log.Warnf(ctx, "unsupported resource type %q for %s, skipping", group, node) + continue + } + + inputConfig, err := configRoot.GetResourceConfig(node) + if err != nil { + return fmt.Errorf("%s: getting config: %w", node, err) + } + + baseRefs := map[string]string{} + + switch { + case strings.HasSuffix(node, ".permissions"): + var sv *structvar.StructVar + if strings.HasPrefix(node, "resources.secret_scopes.") { + typedConfig, ok := inputConfig.(*[]resources.SecretScopePermission) + if !ok { + return fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) + } + sv, err = dresources.PrepareSecretScopeAclsInputConfig(*typedConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) + } + } else { + sv, err = dresources.PreparePermissionsInputConfig(inputConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing permissions config: %w", node, err) + } + } + inputConfig = sv.Value + baseRefs = sv.Refs + + case strings.HasSuffix(node, ".grants"): + sv, err := dresources.PrepareGrantsInputConfig(inputConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing grants config: %w", node, err) + } + inputConfig = sv.Value + baseRefs = sv.Refs + } + + newStateValue, err := adapter.PrepareState(inputConfig) + if err != nil { + return fmt.Errorf("%s: PrepareState: %w", node, err) + } + + refs, err := direct.ExtractReferences(configRoot.Value(), node) + if err != nil { + return fmt.Errorf("%s: extracting references: %w", node, err) + } + maps.Copy(refs, baseRefs) + + sv := structvar.NewStructVar(newStateValue, refs) + + // Resolve each reference using TF state. + // We need to extract the resource name for Method A (looking up in the source resource's TF state). + parts := strings.SplitN(node, ".", 4) + // node format: "resources.." or "resources...permissions" + var srcGroup, srcName string + if len(parts) >= 3 { + srcGroup = parts[1] + srcName = parts[2] + } + + // Collect all field paths that need resolution (avoid modifying map during iteration). + type refEntry struct { + fieldPathStr string + refTemplate string + } + var pendingRefs []refEntry + for fieldPathStr, refTemplate := range sv.Refs { + pendingRefs = append(pendingRefs, refEntry{fieldPathStr, refTemplate}) + } + + for _, pending := range pendingRefs { + fieldPath, err := structpath.ParsePath(pending.fieldPathStr) + if err != nil { + return fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) + } + + // ResolveFieldRef returns the fully resolved value for this field, + // using either Method A (TF state lookup) or Method B (template evaluation). + value, err := migrate.ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate) + if err != nil { + return fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) + } + + // Set the resolved value directly and remove the ref entry. + if err := structaccess.Set(sv.Value, fieldPath, value); err != nil { + return fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) + } + delete(sv.Refs, pending.fieldPathStr) + } + + if len(sv.Refs) > 0 { + return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) + } + + // Handle etag for dashboards. + if etag := etags[node]; etag != "" { + if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { + return fmt.Errorf("%s: cannot set etag: %w", node, err) + } + } + + if err := stateDB.SaveState(node, idEntry.ID, sv.Value, nil); err != nil { + return fmt.Errorf("%s: SaveState: %w", node, err) + } + } + + return nil +} From a496febf0a88f1fe543d7d903c31b194c1bf1a24 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Jun 2026 16:12:10 +0200 Subject: [PATCH 02/16] bundle migrate: remove plan check, make --noplancheck a no-op The new migrate command reads only from the local TF state file and never invokes the Terraform engine, so a pre-migration plan check has no place here. The --noplancheck flag is kept but ignored to avoid breaking callers. Co-authored-by: Isaac --- cmd/bundle/migrate.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/cmd/bundle/migrate.go b/cmd/bundle/migrate.go index 3579658d3fe..ffc79ddda18 100644 --- a/cmd/bundle/migrate.go +++ b/cmd/bundle/migrate.go @@ -20,7 +20,6 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/migrate" - "github.com/databricks/cli/cmd/bundle/deployment" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" @@ -43,8 +42,9 @@ without making API calls. Cross-resource references are resolved from TF state.` Args: root.NoArgs, } - var noPlanCheck bool - cmd.Flags().BoolVar(&noPlanCheck, "noplancheck", false, "Skip running bundle plan before migration.") + // --noplancheck is kept for compatibility but has no effect: this command reads + // only from the local TF state file and never invokes the Terraform engine. + cmd.Flags().Bool("noplancheck", false, "No-op (kept for compatibility).") cmd.RunE = func(cmd *cobra.Command, args []string) error { // Clear engine env var: we read TF state and produce a direct state. @@ -81,15 +81,6 @@ without making API calls. Cross-resource references are resolved from TF state.` return fmt.Errorf("reading %s: %w", localTerraformPath, err) } - // Run plan check unless --noplancheck is set. - if !noPlanCheck { - cmdio.LogString(ctx, "Note: Migration should be done after a full deploy. Running plan now to verify:") - extraArgs, extraArgsStr := deployment.GetCommonArgs(cmd) - if err := deployment.RunPlanCheck(cmd, extraArgs, extraArgsStr); err != nil { - return err - } - } - // Parse TF state: IDs (for state entries) and full attributes (for ref resolution). tfResourceIDs, err := terraform.ParseResourcesState(ctx, b) if err != nil { From 94ed42d7071d581e689705cf093137b9f8727ad1 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Jun 2026 16:31:10 +0200 Subject: [PATCH 03/16] bundle/direct: drop MigrateMode, consolidate state-building in bundle/migrate MigrateMode was a bool parameter on Apply that forked between "save state without deploying" and "actually deploy". The two MigrateMode(true) callers (bundle deployment migrate and upload_state_for_yaml_sync) now use migrate.BuildStateFromTF directly, reading from the local TF state file without any API calls. Changes: - New bundle/migrate/build_state.go: public BuildStateFromTF extracted from cmd/bundle/migrate.go, taking *config.Root so callers can pass an un-interpolated config (needed by upload_state_for_yaml_sync). - bundle/direct/bundle_apply.go: drop MigrateMode type and parameter; Apply now only handles real deployments. - bundle/phases/{deploy,destroy}.go: drop MigrateMode(false) argument. - upload_state_for_yaml_sync.go: replace CalculatePlan+Apply with BuildStateFromTF; keep reverseInterpolate since config is TF-interpolated. - cmd/bundle/deployment/migrate.go: replace CalculatePlan+Apply with BuildStateFromTF; drop exported RunPlanCheck/GetCommonArgs wrappers. - cmd/bundle/migrate.go: delegate to migrate.BuildStateFromTF. Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 25 +-- bundle/migrate/build_state.go | 179 ++++++++++++++++++ bundle/phases/deploy.go | 3 +- bundle/phases/destroy.go | 3 +- .../statemgmt/upload_state_for_yaml_sync.go | 45 ++--- cmd/bundle/deployment/migrate.go | 62 ++---- cmd/bundle/migrate.go | 173 +---------------- 7 files changed, 216 insertions(+), 274 deletions(-) create mode 100644 bundle/migrate/build_state.go diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 16b145f7af8..a5849ffbd1a 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -15,9 +15,7 @@ import ( "github.com/databricks/databricks-sdk-go" ) -type MigrateMode bool - -func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan, migrateMode MigrateMode) { +func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) { if plan == nil { panic("Planning is not done") } @@ -52,9 +50,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa action := entry.Action errorPrefix := fmt.Sprintf("cannot %s %s", action, resourceKey) - if migrateMode { - errorPrefix = "cannot migrate " + resourceKey - } if action == deployplan.Undefined { logdiag.LogError(ctx, fmt.Errorf("cannot deploy %s: unknown action %q", resourceKey, action)) @@ -82,10 +77,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } if action == deployplan.Delete { - if migrateMode { - logdiag.LogError(ctx, fmt.Errorf("%s: Unexpected delete action during migration", errorPrefix)) - return false - } err = d.Destroy(ctx, &b.StateDB) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) @@ -113,18 +104,8 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } - if migrateMode { - // In migration mode we're reading resources in DAG order so that we have fully resolved config snapshots stored - id := b.StateDB.GetResourceID(resourceKey) - if id == "" { - logdiag.LogError(ctx, fmt.Errorf("state entry not found for %q", resourceKey)) - return false - } - err = b.StateDB.SaveState(resourceKey, id, sv.Value, entry.DependsOn) - } else { - // TODO: redo calcDiff to downgrade planned action if possible (?) - err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) - } + // TODO: redo calcDiff to downgrade planned action if possible (?) + err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go new file mode 100644 index 00000000000..af2b3449086 --- /dev/null +++ b/bundle/migrate/build_state.go @@ -0,0 +1,179 @@ +package migrate + +import ( + "context" + "fmt" + "maps" + "strings" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" +) + +// BuildStateFromTF iterates over bundle resources, resolves cross-resource +// references using TF state attributes, and writes each resource's state entry. +// configRoot should be an un-interpolated config (with ${resources.*} references). +func BuildStateFromTF( + ctx context.Context, + configRoot *config.Root, + adapters map[string]*dresources.Adapter, + stateDB *dstate.DeploymentState, + tfAttrs TFStateAttrs, + tfIDs terraform.ExportedResourcesMap, + etags map[string]string, +) error { + // Collect all resource nodes (same patterns as makePlan). + var nodes []string + patterns := []dyn.Pattern{ + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + } + for _, pat := range patterns { + _, err := dyn.MapByPattern( + configRoot.Value(), + pat, + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + nodes = append(nodes, p.String()) + return dyn.InvalidValue, nil + }, + ) + if err != nil { + return err + } + } + + for _, node := range nodes { + idEntry, ok := tfIDs[node] + if !ok { + // Resource is in config but not in TF state (new resource); skip. + continue + } + + group := config.GetResourceTypeFromKey(node) + if group == "" { + return fmt.Errorf("cannot determine resource type for %q", node) + } + + adapter, ok := adapters[group] + if !ok { + log.Warnf(ctx, "unsupported resource type %q for %s, skipping", group, node) + continue + } + + inputConfig, err := configRoot.GetResourceConfig(node) + if err != nil { + return fmt.Errorf("%s: getting config: %w", node, err) + } + + baseRefs := map[string]string{} + + switch { + case strings.HasSuffix(node, ".permissions"): + var sv *structvar.StructVar + if strings.HasPrefix(node, "resources.secret_scopes.") { + typedConfig, ok := inputConfig.(*[]resources.SecretScopePermission) + if !ok { + return fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) + } + sv, err = dresources.PrepareSecretScopeAclsInputConfig(*typedConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) + } + } else { + sv, err = dresources.PreparePermissionsInputConfig(inputConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing permissions config: %w", node, err) + } + } + inputConfig = sv.Value + baseRefs = sv.Refs + + case strings.HasSuffix(node, ".grants"): + sv, err := dresources.PrepareGrantsInputConfig(inputConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing grants config: %w", node, err) + } + inputConfig = sv.Value + baseRefs = sv.Refs + } + + newStateValue, err := adapter.PrepareState(inputConfig) + if err != nil { + return fmt.Errorf("%s: PrepareState: %w", node, err) + } + + refs, err := direct.ExtractReferences(configRoot.Value(), node) + if err != nil { + return fmt.Errorf("%s: extracting references: %w", node, err) + } + maps.Copy(refs, baseRefs) + + sv := structvar.NewStructVar(newStateValue, refs) + + // Resolve each reference using TF state. + // node format: "resources.." or "resources...permissions" + parts := strings.SplitN(node, ".", 4) + var srcGroup, srcName string + if len(parts) >= 3 { + srcGroup = parts[1] + srcName = parts[2] + } + + // Collect all field paths that need resolution (avoid modifying map during iteration). + type refEntry struct { + fieldPathStr string + refTemplate string + } + var pendingRefs []refEntry + for fieldPathStr, refTemplate := range sv.Refs { + pendingRefs = append(pendingRefs, refEntry{fieldPathStr, refTemplate}) + } + + for _, pending := range pendingRefs { + fieldPath, err := structpath.ParsePath(pending.fieldPathStr) + if err != nil { + return fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) + } + + // ResolveFieldRef returns the fully resolved value for this field, + // using either Method A (TF state lookup) or Method B (template evaluation). + value, err := ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate) + if err != nil { + return fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) + } + + // Set the resolved value directly and remove the ref entry. + if err := structaccess.Set(sv.Value, fieldPath, value); err != nil { + return fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) + } + delete(sv.Refs, pending.fieldPathStr) + } + + if len(sv.Refs) > 0 { + return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) + } + + // Handle etag for dashboards. + if etag := etags[node]; etag != "" { + if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { + return fmt.Errorf("%s: cannot set etag: %w", node, err) + } + } + + if err := stateDB.SaveState(node, idEntry.ID, sv.Value, nil); err != nil { + return fmt.Errorf("%s: SaveState: %w", node, err) + } + } + + return nil +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 3cac322f9e3..78a06ef732e 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -14,7 +14,6 @@ import ( "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/permissions" @@ -81,7 +80,7 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, ta err error ) if targetEngine.IsDirect() { - b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, direct.MigrateMode(false)) + b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) state, err = b.DeploymentBundle.StateDB.Finalize(ctx) } else { bundle.ApplyContext(ctx, b, terraform.Apply()) diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 95eec600dc2..e1196fbc7f4 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" @@ -76,7 +75,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) { if engine.IsDirect() { - b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, direct.MigrateMode(false)) + b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) } else { // Core destructive mutators for destroy. These require informed user consent. bundle.ApplyContext(ctx, b, terraform.Apply()) diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 0399c7b31ff..4a6f804789c 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -14,18 +14,16 @@ import ( "github.com/databricks/cli/bundle/config/mutator/resourcemutator" "github.com/databricks/cli/bundle/deploy" "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" - "github.com/databricks/cli/libs/structs/structaccess" - "github.com/databricks/cli/libs/structs/structpath" ) type uploadStateForYamlSync struct { @@ -117,6 +115,11 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("failed to read terraform state: %w", err) } + tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) + if err != nil { + return false, fmt.Errorf("failed to read terraform state attributes: %w", err) + } + state := make(map[string]dstate.ResourceEntry) etags := map[string]string{} @@ -141,8 +144,8 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun migratedDB := dstate.NewDatabase(tfState.Lineage, tfState.Serial+1) migratedDB.State = state - deploymentBundle := &direct.DeploymentBundle{} - deploymentBundle.StateDB.OpenWithData(snapshotPath, migratedDB) + var stateDB dstate.DeploymentState + stateDB.OpenWithData(snapshotPath, migratedDB) // Apply SecretScopeFixups so the config matches what the direct engine expects. // This adds MANAGE ACL for the current user to all secret scopes, ensuring @@ -152,9 +155,9 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, errors.New("failed to apply secret scope fixups") } - // Get the dynamic value from b.Config and reverse the interpolation. // b.Config has been modified by terraform.Interpolate which converts bundle-style // references (${resources.pipelines.x.id}) to terraform-style (${databricks_pipeline.x.id}). + // BuildStateFromTF expects ${resources.*} references, so reverse the interpolation first. interpolatedRoot := b.Config.Value() uninterpolatedRoot, err := reverseInterpolate(interpolatedRoot) if err != nil { @@ -169,36 +172,20 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("failed to create uninterpolated config: %w", err) } - plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &uninterpolatedConfig) + adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) if err != nil { return false, err } - for _, entry := range plan.Plan { - entry.Action = deployplan.Update - } - - for key := range plan.Plan { - etag := etags[key] - if etag == "" { - continue - } - sv, ok := deploymentBundle.StateCache.Load(key) - if !ok { - continue - } - err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag) - if err != nil { - log.Warnf(ctx, "Failed to set etag on %q: %v", key, err) - } + if err := stateDB.UpgradeToWrite(); err != nil { + return false, fmt.Errorf("upgrading state for apply: %w", err) } - if err := deploymentBundle.StateDB.UpgradeToWrite(); err != nil { - return false, fmt.Errorf("upgrading state for apply: %w", err) + if err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfAttrs, terraformResources, etags); err != nil { + return false, err } - deploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, direct.MigrateMode(true)) - if _, err := deploymentBundle.StateDB.Finalize(ctx); err != nil { + if _, err := stateDB.Finalize(ctx); err != nil { return false, err } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index aa22e5118ac..1c59616fcc9 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -14,28 +14,20 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/shellquote" - "github.com/databricks/cli/libs/structs/structaccess" - "github.com/databricks/cli/libs/structs/structpath" "github.com/spf13/cobra" ) const backupSuffix = ".backup" -// RunPlanCheck runs bundle plan and checks if there are any actions planned. -// Returns error if plan fails or if there are actions planned. -func RunPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { - return runPlanCheck(cmd, extraArgs, extraArgsStr) -} - // runPlanCheck runs bundle plan and checks if there are any actions planned. // Returns error if plan fails or if there are actions planned. func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { @@ -84,12 +76,6 @@ func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) e return nil } -// GetCommonArgs extracts common flags (target, profile, var) from the command into -// argument slices suitable for forwarding to a subprocess. -func GetCommonArgs(cmd *cobra.Command) ([]string, string) { - return getCommonArgs(cmd) -} - func getCommonArgs(cmd *cobra.Command) ([]string, string) { var args []string var quotedArgs []string @@ -221,6 +207,11 @@ To start using direct engine, set "engine: direct" under bundle in your databric } } + tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) + if err != nil { + return fmt.Errorf("failed to read terraform state attributes: %w", err) + } + etags := map[string]string{} state := make(map[string]dstate.ResourceEntry) @@ -238,8 +229,8 @@ To start using direct engine, set "engine: direct" under bundle in your databric migratedDB := dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1) migratedDB.State = state - deploymentBundle := &direct.DeploymentBundle{} - deploymentBundle.StateDB.OpenWithData(tempStatePath, migratedDB) + var stateDB dstate.DeploymentState + stateDB.OpenWithData(tempStatePath, migratedDB) tempStatePathAutoRemove := true @@ -257,43 +248,20 @@ To start using direct engine, set "engine: direct" under bundle in your databric return root.ErrAlreadyPrinted } - plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config) + adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) if err != nil { return err } - for _, entry := range plan.Plan { - // Force all actions to be "update" so that deploym below goes through every resource - entry.Action = deployplan.Update - } - - // We need to copy ETag into new state. - // For most resources state consists of fully resolved local config snapshot + id. - // Dashboards are special in that they also store "etag" in state which is not provided by user but - // comes from remote state. If we don't store "etag" in state, we won't detect remote drift, because - // local=nil, remote="" which will be classified as a backend default and skipped. - - for key := range plan.Plan { - etag := etags[key] - if etag == "" { - continue - } - sv, ok := deploymentBundle.StateCache.Load(key) - if !ok { - return fmt.Errorf("failed to read state for %q", key) - } - err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag) - if err != nil { - return fmt.Errorf("failed to set etag on %q: %w", key, err) - } + if err := stateDB.UpgradeToWrite(); err != nil { + return fmt.Errorf("upgrading state for apply: %w", err) } - if err := deploymentBundle.StateDB.UpgradeToWrite(); err != nil { - return fmt.Errorf("upgrading state for apply: %w", err) + if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, terraformResources, etags); err != nil { + return err } - deploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, direct.MigrateMode(true)) - if _, err := deploymentBundle.StateDB.Finalize(ctx); err != nil { + if _, err := stateDB.Finalize(ctx); err != nil { logdiag.LogError(ctx, err) } if logdiag.HasError(ctx) { diff --git a/cmd/bundle/migrate.go b/cmd/bundle/migrate.go index ffc79ddda18..8a33fe85f55 100644 --- a/cmd/bundle/migrate.go +++ b/cmd/bundle/migrate.go @@ -5,31 +5,21 @@ import ( "encoding/json" "errors" "fmt" - "maps" "os" "path/filepath" - "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/env" - "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" - "github.com/databricks/cli/libs/structs/structaccess" - "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/structs/structvar" "github.com/spf13/cobra" ) @@ -159,7 +149,7 @@ without making API calls. Cross-resource references are resolved from TF state.` } // Process each resource: prepare state, resolve refs from TF state, save. - if err := buildStateFromTF(ctx, b, adapters, &stateDB, tfAttrs, tfResourceIDs, etags); err != nil { + if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, tfResourceIDs, etags); err != nil { return err } @@ -200,164 +190,3 @@ To undo the migration, remove %s and rename %s to %s return cmd } - -// buildStateFromTF iterates over bundle resources, resolves cross-resource -// references using TF state attributes, and writes each resource's state entry. -func buildStateFromTF( - ctx context.Context, - b *bundle.Bundle, - adapters map[string]*dresources.Adapter, - stateDB *dstate.DeploymentState, - tfAttrs migrate.TFStateAttrs, - tfIDs terraform.ExportedResourcesMap, - etags map[string]string, -) error { - configRoot := &b.Config - - // Collect all resource nodes (same patterns as makePlan). - var nodes []string - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), - } - for _, pat := range patterns { - _, err := dyn.MapByPattern( - configRoot.Value(), - pat, - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - nodes = append(nodes, p.String()) - return dyn.InvalidValue, nil - }, - ) - if err != nil { - return err - } - } - - for _, node := range nodes { - idEntry, ok := tfIDs[node] - if !ok { - // Resource is in config but not in TF state (new resource); skip. - continue - } - - group := config.GetResourceTypeFromKey(node) - if group == "" { - return fmt.Errorf("cannot determine resource type for %q", node) - } - - adapter, ok := adapters[group] - if !ok { - log.Warnf(ctx, "unsupported resource type %q for %s, skipping", group, node) - continue - } - - inputConfig, err := configRoot.GetResourceConfig(node) - if err != nil { - return fmt.Errorf("%s: getting config: %w", node, err) - } - - baseRefs := map[string]string{} - - switch { - case strings.HasSuffix(node, ".permissions"): - var sv *structvar.StructVar - if strings.HasPrefix(node, "resources.secret_scopes.") { - typedConfig, ok := inputConfig.(*[]resources.SecretScopePermission) - if !ok { - return fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) - } - sv, err = dresources.PrepareSecretScopeAclsInputConfig(*typedConfig, node) - if err != nil { - return fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) - } - } else { - sv, err = dresources.PreparePermissionsInputConfig(inputConfig, node) - if err != nil { - return fmt.Errorf("%s: preparing permissions config: %w", node, err) - } - } - inputConfig = sv.Value - baseRefs = sv.Refs - - case strings.HasSuffix(node, ".grants"): - sv, err := dresources.PrepareGrantsInputConfig(inputConfig, node) - if err != nil { - return fmt.Errorf("%s: preparing grants config: %w", node, err) - } - inputConfig = sv.Value - baseRefs = sv.Refs - } - - newStateValue, err := adapter.PrepareState(inputConfig) - if err != nil { - return fmt.Errorf("%s: PrepareState: %w", node, err) - } - - refs, err := direct.ExtractReferences(configRoot.Value(), node) - if err != nil { - return fmt.Errorf("%s: extracting references: %w", node, err) - } - maps.Copy(refs, baseRefs) - - sv := structvar.NewStructVar(newStateValue, refs) - - // Resolve each reference using TF state. - // We need to extract the resource name for Method A (looking up in the source resource's TF state). - parts := strings.SplitN(node, ".", 4) - // node format: "resources.." or "resources...permissions" - var srcGroup, srcName string - if len(parts) >= 3 { - srcGroup = parts[1] - srcName = parts[2] - } - - // Collect all field paths that need resolution (avoid modifying map during iteration). - type refEntry struct { - fieldPathStr string - refTemplate string - } - var pendingRefs []refEntry - for fieldPathStr, refTemplate := range sv.Refs { - pendingRefs = append(pendingRefs, refEntry{fieldPathStr, refTemplate}) - } - - for _, pending := range pendingRefs { - fieldPath, err := structpath.ParsePath(pending.fieldPathStr) - if err != nil { - return fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) - } - - // ResolveFieldRef returns the fully resolved value for this field, - // using either Method A (TF state lookup) or Method B (template evaluation). - value, err := migrate.ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate) - if err != nil { - return fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) - } - - // Set the resolved value directly and remove the ref entry. - if err := structaccess.Set(sv.Value, fieldPath, value); err != nil { - return fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) - } - delete(sv.Refs, pending.fieldPathStr) - } - - if len(sv.Refs) > 0 { - return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) - } - - // Handle etag for dashboards. - if etag := etags[node]; etag != "" { - if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { - return fmt.Errorf("%s: cannot set etag: %w", node, err) - } - } - - if err := stateDB.SaveState(node, idEntry.ID, sv.Value, nil); err != nil { - return fmt.Errorf("%s: SaveState: %w", node, err) - } - } - - return nil -} From cb34b4b8268ecc5efc6066cd439ceefa7fee37d4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Jun 2026 17:30:22 +0200 Subject: [PATCH 04/16] bundle migrate: remove new top-level command, keep deployment migrate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new approach (BuildStateFromTF, no API calls) has been folded into the existing 'bundle deployment migrate' command. The separate 'bundle migrate' command is removed — it was redundant. Also add resolve_test.go: verify that int and bool cross-resource references work correctly when Method B returns a string value. structaccess.Set already handles string→int and string→bool conversion via strconv, so no bug exists. Co-authored-by: Isaac --- bundle/migrate/resolve_test.go | 78 ++++++++++++++ cmd/bundle/bundle.go | 1 - cmd/bundle/migrate.go | 192 --------------------------------- 3 files changed, 78 insertions(+), 193 deletions(-) create mode 100644 bundle/migrate/resolve_test.go delete mode 100644 cmd/bundle/migrate.go diff --git a/bundle/migrate/resolve_test.go b/bundle/migrate/resolve_test.go new file mode 100644 index 00000000000..8a2ab06d13b --- /dev/null +++ b/bundle/migrate/resolve_test.go @@ -0,0 +1,78 @@ +package migrate_test + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/migrate" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// state with src job having int and bool fields set. +func testState() migrate.TFStateAttrs { + return migrate.TFStateAttrs{ + "databricks_job": { + "src": json.RawMessage(`{ + "id": "111", + "max_concurrent_runs": 4, + "always_running": true + }`), + "dst": json.RawMessage(`{ + "id": "222", + "max_concurrent_runs": 4, + "always_running": true + }`), + }, + } +} + +// TestResolveFieldRefInt proves that when Method B (template evaluation) wins for +// an int field, the returned string value is still usable: structaccess.Set must +// parse it back to int and not error. +func TestResolveFieldRefInt(t *testing.T) { + state := testState() + // Remove dst from state so Method A fails and Method B must be used. + delete(state["databricks_job"], "dst") + + ctx := t.Context() + fieldPath, err := structpath.ParsePath("max_concurrent_runs") + require.NoError(t, err) + + value, err := migrate.ResolveFieldRef(ctx, state, "jobs", "dst", fieldPath, + "${resources.jobs.src.max_concurrent_runs}") + require.NoError(t, err) + + // Method B succeeds: returns string "4". Verify Set converts it to int. + type target struct { + MaxConcurrentRuns int `json:"max_concurrent_runs"` + } + s := &target{} + err = structaccess.Set(s, fieldPath, value) + assert.NoError(t, err, "Set should parse string %q into int field", value) + assert.Equal(t, 4, s.MaxConcurrentRuns) +} + +// TestResolveFieldRefBool is the same for a bool field. +func TestResolveFieldRefBool(t *testing.T) { + state := testState() + delete(state["databricks_job"], "dst") + + ctx := t.Context() + fieldPath, err := structpath.ParsePath("always_running") + require.NoError(t, err) + + value, err := migrate.ResolveFieldRef(ctx, state, "jobs", "dst", fieldPath, + "${resources.jobs.src.always_running}") + require.NoError(t, err) + + type target struct { + AlwaysRunning bool `json:"always_running"` + } + s := &target{} + err = structaccess.Set(s, fieldPath, value) + assert.NoError(t, err, "Set should parse string %q into bool field", value) + assert.Equal(t, true, s.AlwaysRunning) +} diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index 84666eec689..7189b1d431d 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -38,7 +38,6 @@ Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.htm cmd.AddCommand(newDebugCommand()) cmd.AddCommand(newOpenCommand()) cmd.AddCommand(newPlanCommand()) - cmd.AddCommand(newMigrateCommand()) cmd.AddCommand(newConfigRemoteSyncCommand()) // Bundle Metadata Service (DMS) read-only command groups. Only `get` diff --git a/cmd/bundle/migrate.go b/cmd/bundle/migrate.go deleted file mode 100644 index 8a33fe85f55..00000000000 --- a/cmd/bundle/migrate.go +++ /dev/null @@ -1,192 +0,0 @@ -package bundle - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/direct/dresources" - "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/bundle/migrate" - "github.com/databricks/cli/cmd/bundle/utils" - "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/env" - "github.com/databricks/cli/libs/logdiag" - "github.com/spf13/cobra" -) - -func newMigrateCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "migrate", - Short: "Migrate from Terraform to Direct deployment engine (no API calls)", - Long: `Creates a Direct deployment state file from the local config and Terraform state, -without making API calls. Cross-resource references are resolved from TF state.`, - Args: root.NoArgs, - } - - // --noplancheck is kept for compatibility but has no effect: this command reads - // only from the local TF state file and never invokes the Terraform engine. - cmd.Flags().Bool("noplancheck", false, "No-op (kept for compatibility).") - - cmd.RunE = func(cmd *cobra.Command, args []string) error { - // Clear engine env var: we read TF state and produce a direct state. - cmd.SetContext(env.Set(cmd.Context(), engine.EnvVar, "")) - - opts := utils.ProcessOptions{ - AlwaysPull: true, - FastValidate: true, - Build: true, - PostInitFunc: func(_ context.Context, b *bundle.Bundle) error { - if b.Config.Bundle.Engine == engine.EngineTerraform { - return fmt.Errorf("bundle.engine is set to %q; migration requires \"engine: direct\" or no engine setting", engine.EngineTerraform) - } - return nil - }, - } - - b, stateDesc, err := utils.ProcessBundleRet(cmd, opts) - if err != nil { - return err - } - ctx := cmd.Context() - - if stateDesc.Lineage == "" { - cmdio.LogString(ctx, `Error: no existing state found. To start fresh with direct engine, set "engine: direct".`) - return root.ErrAlreadyPrinted - } - if stateDesc.Engine.IsDirect() { - return fmt.Errorf("already using direct engine: %s", stateDesc.String()) - } - - _, localTerraformPath := b.StateFilenameTerraform(ctx) - if _, err := os.Stat(localTerraformPath); err != nil { - return fmt.Errorf("reading %s: %w", localTerraformPath, err) - } - - // Parse TF state: IDs (for state entries) and full attributes (for ref resolution). - tfResourceIDs, err := terraform.ParseResourcesState(ctx, b) - if err != nil { - return fmt.Errorf("failed to parse terraform state: %w", err) - } - for key, entry := range tfResourceIDs { - if entry.ID == "" { - return fmt.Errorf("missing ID for %s in terraform state", key) - } - } - - cacheDir, err := terraform.Dir(ctx, b) - if err != nil { - return err - } - tfStateFilename, _ := b.StateFilenameTerraform(ctx) - tfStateFullPath := filepath.Join(cacheDir, tfStateFilename) - - tfAttrs, err := migrate.ParseTFStateAttrs(tfStateFullPath) - if err != nil { - return fmt.Errorf("failed to read terraform state attributes: %w", err) - } - - _, localPath := b.StateFilenameDirect(ctx) - tempPath := localPath + ".temp-migration" - - if _, err := os.Stat(tempPath); err == nil { - return fmt.Errorf("temporary state file %s already exists, another migration is in progress or was interrupted. In the latter case, delete the file", tempPath) - } - if _, err := os.Stat(localPath); err == nil { - return fmt.Errorf("state file %s already exists", localPath) - } - - // Apply SecretScopeFixups so the config matches what the direct engine expects. - // This adds MANAGE ACL for the current user to all secret scopes, ensuring - // the migrated state and config agree on .permissions entries. - bundle.ApplyContext(ctx, b, resourcemutator.SecretScopeFixups(engine.EngineDirect)) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted - } - - // Build initial state with IDs and optional ETags. - etags := map[string]string{} - state := make(map[string]dstate.ResourceEntry) - for key, resourceEntry := range tfResourceIDs { - state[key] = dstate.ResourceEntry{ - ID: resourceEntry.ID, - State: json.RawMessage("{}"), - } - if resourceEntry.ETag != "" { - etags[key] = resourceEntry.ETag - } - } - - migratedDB := dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1) - migratedDB.State = state - - var stateDB dstate.DeploymentState - stateDB.OpenWithData(tempPath, migratedDB) - - removeTempPath := true - defer func() { - if removeTempPath { - _ = os.Remove(tempPath) - } - }() - - // Initialize adapters. - adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) - if err != nil { - return err - } - - if err := stateDB.UpgradeToWrite(); err != nil { - return fmt.Errorf("upgrading state for write: %w", err) - } - - // Process each resource: prepare state, resolve refs from TF state, save. - if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, tfResourceIDs, etags); err != nil { - return err - } - - if _, err := stateDB.Finalize(ctx); err != nil { - return fmt.Errorf("finalizing state: %w", err) - } - if logdiag.HasError(ctx) { - return errors.New("migration encountered errors") - } - - if err := os.Rename(tempPath, localPath); err != nil { - return fmt.Errorf("renaming %s to %s: %w", tempPath, localPath, err) - } - removeTempPath = false - - localTerraformBackupPath := localTerraformPath + ".backup" - err = os.Rename(localTerraformPath, localTerraformBackupPath) - if err != nil { - // Not fatal since we've already incremented the serial. - logdiag.LogError(ctx, err) - } - - extraArgsStr := "" - if flag := cmd.Flag("target"); flag != nil && flag.Changed { - extraArgsStr = " -t " + flag.Value.String() - } - - cmdio.LogString(ctx, fmt.Sprintf(`Success! Migrated %d resources to direct engine state file: %s - -Validate the migration by running "databricks bundle plan%s", there should be no actions planned. - -The state file is not synchronized to the workspace yet. To finalize the migration, run "bundle deploy%s". - -To undo the migration, remove %s and rename %s to %s -`, len(state), localPath, extraArgsStr, extraArgsStr, localPath, localTerraformBackupPath, localTerraformPath)) - return nil - } - - return cmd -} From 7314ccea84e8899027b4feac951ef048345606b6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 2 Jun 2026 14:11:14 +0200 Subject: [PATCH 05/16] deployment migrate: drop plan check and --noplancheck flag The command now reads from the local TF state file without invoking the Terraform binary, so there is nothing for the plan check to verify. Also simplify getCommonArgs to return only the display string (the args slice was only needed to forward to the plan subprocess). Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 1 - bundle/migrate/resolve_test.go | 2 +- bundle/migrate/tf_state.go | 5 +- cmd/bundle/deployment/migrate.go | 99 ++++---------------------------- 4 files changed, 14 insertions(+), 93 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index a5849ffbd1a..a63d70aee13 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -106,7 +106,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) - if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false diff --git a/bundle/migrate/resolve_test.go b/bundle/migrate/resolve_test.go index 8a2ab06d13b..221c873a73d 100644 --- a/bundle/migrate/resolve_test.go +++ b/bundle/migrate/resolve_test.go @@ -74,5 +74,5 @@ func TestResolveFieldRefBool(t *testing.T) { s := &target{} err = structaccess.Set(s, fieldPath, value) assert.NoError(t, err, "Set should parse string %q into bool field", value) - assert.Equal(t, true, s.AlwaysRunning) + assert.True(t, s.AlwaysRunning) } diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 642f7a26b74..f4733c152eb 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -57,10 +57,9 @@ func ParseTFStateAttrs(path string) (TFStateAttrs, error) { // tfSchemaTypeMap maps TF resource type name → schema struct type (via AllResources json tags). var tfSchemaTypeMap = sync.OnceValue(func() map[string]reflect.Type { - t := reflect.TypeOf(tfschema.AllResources{}) + t := reflect.TypeFor[tfschema.AllResources]() m := make(map[string]reflect.Type, t.NumField()) - for i := range t.NumField() { - f := t.Field(i) + for f := range t.Fields() { tag := strings.Split(f.Tag.Get("json"), ",")[0] if tag != "" && tag != "-" { m[tag] = f.Type diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 1c59616fcc9..fb2179145d3 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -1,13 +1,11 @@ package deployment import ( - "bytes" "context" "encoding/json" "errors" "fmt" "os" - "os/exec" "strings" "github.com/databricks/cli/bundle" @@ -28,95 +26,31 @@ import ( const backupSuffix = ".backup" -// runPlanCheck runs bundle plan and checks if there are any actions planned. -// Returns error if plan fails or if there are actions planned. -func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { - ctx := cmd.Context() - - executable, err := os.Executable() - if err != nil { - return fmt.Errorf("failed to get executable path: %w", err) - } - - args := []string{"bundle", "plan"} - args = append(args, extraArgs...) - - planCmd := exec.CommandContext(ctx, executable, args...) - var stdout bytes.Buffer - planCmd.Stdout = &stdout - planCmd.Stderr = cmd.ErrOrStderr() - - // Use the engine encoded in the state - planCmd.Env = append(os.Environ(), "DATABRICKS_BUNDLE_ENGINE=terraform") - - err = planCmd.Run() - - // Output the plan stdout as is - output := stdout.String() - fmt.Fprint(cmd.OutOrStdout(), output) - - if err != nil { - msg := "" - if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { - msg = fmt.Sprintf("exit code %d", exitErr.ExitCode()) - } else { - msg = err.Error() - } - return fmt.Errorf("bundle plan failed with %s, aborting migration. To proceed with migration anyway, re-run the command with --noplancheck option", msg) - } - - if !strings.Contains(output, "Plan:") { - return fmt.Errorf("cannot parse 'databricks bundle plan%s' output, aborting migration. Skip plan check with --noplancheck option", extraArgsStr) - } - - if !strings.Contains(output, "Plan: 0 to add, 0 to change, 0 to delete") { - return fmt.Errorf("'databricks bundle plan%s' shows actions planned, aborting migration. Please run 'databricks bundle deploy%s' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option", extraArgsStr, extraArgsStr) - } - - return nil -} - -func getCommonArgs(cmd *cobra.Command) ([]string, string) { - var args []string +func getCommonArgs(cmd *cobra.Command) string { var quotedArgs []string if flag := cmd.Flag("target"); flag != nil && flag.Changed { - target := flag.Value.String() - if target != "" { - args = append(args, "-t") - args = append(args, target) - quotedArgs = append(quotedArgs, "-t") - quotedArgs = append(quotedArgs, shellquote.BashArg(target)) + if target := flag.Value.String(); target != "" { + quotedArgs = append(quotedArgs, "-t", shellquote.BashArg(target)) } } if flag := cmd.Flag("profile"); flag != nil && flag.Changed { - profile := flag.Value.String() - if profile != "" { - args = append(args, "-p") - args = append(args, profile) - quotedArgs = append(quotedArgs, "-p") - quotedArgs = append(quotedArgs, shellquote.BashArg(profile)) + if profile := flag.Value.String(); profile != "" { + quotedArgs = append(quotedArgs, "-p", shellquote.BashArg(profile)) } } if flag := cmd.Flag("var"); flag != nil && flag.Changed { - varValues, err := cmd.Flags().GetStringSlice("var") - if err == nil { + if varValues, err := cmd.Flags().GetStringSlice("var"); err == nil { for _, v := range varValues { - args = append(args, "--var") - args = append(args, v) - quotedArgs = append(quotedArgs, "--var") - quotedArgs = append(quotedArgs, shellquote.BashArg(v)) + quotedArgs = append(quotedArgs, "--var", shellquote.BashArg(v)) } } } - argsStr := "" - - if len(quotedArgs) > 0 { - argsStr = " " + strings.Join(quotedArgs, " ") + if len(quotedArgs) == 0 { + return "" } - - return args, argsStr + return " " + strings.Join(quotedArgs, " ") } func newMigrateCommand() *cobra.Command { @@ -134,11 +68,8 @@ to the workspace so that subsequent deploys of this bundle use direct deployment Args: root.NoArgs, } - var noPlanCheck bool - cmd.Flags().BoolVar(&noPlanCheck, "noplancheck", false, "Skip running bundle plan before migration.") - cmd.RunE = func(cmd *cobra.Command, args []string) error { - extraArgs, extraArgsStr := getCommonArgs(cmd) + extraArgsStr := getCommonArgs(cmd) // Clear the engine env var so migrate always uses terraform engine to read existing state, // regardless of what the user may have set in their environment. @@ -199,14 +130,6 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("state file %s already exists", localPath) } - // Run plan check unless --noplancheck is set - if !noPlanCheck { - cmdio.LogString(ctx, "Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:") - if err = runPlanCheck(cmd, extraArgs, extraArgsStr); err != nil { - return err - } - } - tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) if err != nil { return fmt.Errorf("failed to read terraform state attributes: %w", err) From b5d2c673ace3bc0f1b02474d5e88bc895f8b7559 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 11:50:57 +0200 Subject: [PATCH 06/16] acceptance: update bundle-deployment-migrate help output Remove --noplancheck from expected output after dropping that flag. Co-authored-by: Isaac --- acceptance/bundle/help/bundle-deployment-migrate/output.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/acceptance/bundle/help/bundle-deployment-migrate/output.txt b/acceptance/bundle/help/bundle-deployment-migrate/output.txt index da2f95707bc..36ee7d481e7 100644 --- a/acceptance/bundle/help/bundle-deployment-migrate/output.txt +++ b/acceptance/bundle/help/bundle-deployment-migrate/output.txt @@ -12,8 +12,7 @@ Usage: databricks bundle deployment migrate [flags] Flags: - -h, --help help for migrate - --noplancheck Skip running bundle plan before migration. + -h, --help help for migrate Global Flags: --debug enable debug logging From 95a5b1343598eb8784ec4381a749fabb24c5671f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 12:08:36 +0200 Subject: [PATCH 07/16] migrate: fix TF list-block unmarshaling, restore --noplancheck no-op, update outputs TF state stores single-block fields (e.g. continuous, deployment) as single-element arrays [{}], not plain objects. json.Unmarshal into the generated Go schema structs fails with type mismatch. Switch LookupTFField to navigate via map[string]any + custom navigateTFState that auto-unwraps single-element lists when a string-key step follows. Also: - Restore --noplancheck as a no-op flag (backward compat; used by invariant tests for job_with_depends_on config). - Remove the plan-check lines from acceptance test output.txt files. - Update help output to include --noplancheck. Co-authored-by: Isaac --- .../help/bundle-deployment-migrate/output.txt | 3 +- acceptance/bundle/migrate/basic/output.txt | 2 - .../bundle/migrate/dashboards/output.txt | 2 - .../bundle/migrate/default-python/output.txt | 1 - acceptance/bundle/migrate/grants/output.txt | 2 - .../bundle/migrate/permissions/output.txt | 2 - .../bundle/migrate/profile_arg/output.txt | 4 - acceptance/bundle/migrate/runas/output.txt | 1 - acceptance/bundle/migrate/var_arg/output.txt | 4 - bundle/migrate/tf_state.go | 83 ++++++++++++------- cmd/bundle/deployment/migrate.go | 4 + 11 files changed, 61 insertions(+), 47 deletions(-) diff --git a/acceptance/bundle/help/bundle-deployment-migrate/output.txt b/acceptance/bundle/help/bundle-deployment-migrate/output.txt index 36ee7d481e7..9312fb85ffc 100644 --- a/acceptance/bundle/help/bundle-deployment-migrate/output.txt +++ b/acceptance/bundle/help/bundle-deployment-migrate/output.txt @@ -12,7 +12,8 @@ Usage: databricks bundle deployment migrate [flags] Flags: - -h, --help help for migrate + -h, --help help for migrate + --noplancheck No-op (kept for compatibility). Global Flags: --debug enable debug logging diff --git a/acceptance/bundle/migrate/basic/output.txt b/acceptance/bundle/migrate/basic/output.txt index dafa3a4086e..cd45577a1ee 100644 --- a/acceptance/bundle/migrate/basic/output.txt +++ b/acceptance/bundle/migrate/basic/output.txt @@ -11,8 +11,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged Success! Migrated 3 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/dashboards/output.txt b/acceptance/bundle/migrate/dashboards/output.txt index 19a4f1c7bb5..cfda4350ceb 100644 --- a/acceptance/bundle/migrate/dashboards/output.txt +++ b/acceptance/bundle/migrate/dashboards/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/default-python/output.txt b/acceptance/bundle/migrate/default-python/output.txt index 15d49805b47..92efb3066d6 100644 --- a/acceptance/bundle/migrate/default-python/output.txt +++ b/acceptance/bundle/migrate/default-python/output.txt @@ -24,7 +24,6 @@ Deployment complete! >>> musterr [CLI] bundle deployment migrate Building python_artifact... -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: Building python_artifact... update jobs.sample_job diff --git a/acceptance/bundle/migrate/grants/output.txt b/acceptance/bundle/migrate/grants/output.txt index 146787d549a..86acb6398f5 100644 --- a/acceptance/bundle/migrate/grants/output.txt +++ b/acceptance/bundle/migrate/grants/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 6 unchanged Success! Migrated 6 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/permissions/output.txt b/acceptance/bundle/migrate/permissions/output.txt index f85c8d7bdbf..51caca51104 100644 --- a/acceptance/bundle/migrate/permissions/output.txt +++ b/acceptance/bundle/migrate/permissions/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged Success! Migrated 4 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/profile_arg/output.txt b/acceptance/bundle/migrate/profile_arg/output.txt index a6def38363a..082feee15b1 100644 --- a/acceptance/bundle/migrate/profile_arg/output.txt +++ b/acceptance/bundle/migrate/profile_arg/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -p non_existent321 -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json Validate the migration by running "databricks bundle plan -p non_existent321", there should be no actions planned. @@ -24,8 +22,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -p non_existent321 -t prod -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/prod/resources.json Validate the migration by running "databricks bundle plan -t prod -p non_existent321", there should be no actions planned. diff --git a/acceptance/bundle/migrate/runas/output.txt b/acceptance/bundle/migrate/runas/output.txt index 74b9a0217f3..f7034d74ec0 100644 --- a/acceptance/bundle/migrate/runas/output.txt +++ b/acceptance/bundle/migrate/runas/output.txt @@ -81,7 +81,6 @@ Consider using a adding a top-level permissions section such as the following: See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. in databricks.yml:5:3 -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. diff --git a/acceptance/bundle/migrate/var_arg/output.txt b/acceptance/bundle/migrate/var_arg/output.txt index a7f8c0e5b2e..3c20819211d 100644 --- a/acceptance/bundle/migrate/var_arg/output.txt +++ b/acceptance/bundle/migrate/var_arg/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate --var=job_name=Custom Job Name -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json Validate the migration by running "databricks bundle plan --var 'job_name=Custom Job Name'", there should be no actions planned. @@ -39,8 +37,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate --var job_name=Custom Job Name -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json Validate the migration by running "databricks bundle plan --var 'job_name=Custom Job Name'", there should be no actions planned. diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index f4733c152eb..52799dd79b8 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -4,14 +4,9 @@ import ( "encoding/json" "fmt" "os" - "reflect" - "strings" - "sync" "github.com/databricks/cli/bundle/deploy/terraform" - tfschema "github.com/databricks/cli/bundle/internal/tf/schema" "github.com/databricks/cli/bundle/terraform_dabs_map" - "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" tfjson "github.com/hashicorp/terraform-json" ) @@ -55,23 +50,9 @@ func ParseTFStateAttrs(path string) (TFStateAttrs, error) { return result, nil } -// tfSchemaTypeMap maps TF resource type name → schema struct type (via AllResources json tags). -var tfSchemaTypeMap = sync.OnceValue(func() map[string]reflect.Type { - t := reflect.TypeFor[tfschema.AllResources]() - m := make(map[string]reflect.Type, t.NumField()) - for f := range t.Fields() { - tag := strings.Split(f.Tag.Get("json"), ",")[0] - if tag != "" && tag != "-" { - m[tag] = f.Type - } - } - return m -}) - // LookupTFField looks up a field from TF state attributes for a bundle resource. // group is the DABs group (e.g. "pipelines"), name is the resource name. // fieldPath is the path to the field (may be in DABs or TF naming; both handled by DABsPathToTerraform). -// Returns (nil, nil) for empty/zero fields, error if the resource or field is not found. func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath.PathNode) (any, error) { tfType, ok := terraform.GroupToTerraformName[group] if !ok { @@ -91,16 +72,62 @@ func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath return nil, fmt.Errorf("%s.%s not found in TF state", tfType, name) } - schemaType, ok := tfSchemaTypeMap()[tfType] - if !ok { - return nil, fmt.Errorf("no schema type registered for %q", tfType) - } - - // Unmarshal attributes into a new instance of the schema struct. - ptr := reflect.New(schemaType) - if err := json.Unmarshal(attrsJSON, ptr.Interface()); err != nil { + // Unmarshal into map[string]any to handle TF list-blocks: in TF state, single-block + // fields are stored as single-element arrays [{"field": "value"}], not as plain objects. + // Navigating via map avoids the json.Unmarshal type mismatch between []T in JSON and + // struct-typed schema fields. + var attrs map[string]any + if err := json.Unmarshal(attrsJSON, &attrs); err != nil { return nil, fmt.Errorf("cannot parse TF state for %s.%s: %w", tfType, name, err) } - return structaccess.Get(ptr.Interface(), tfFieldPath) + return navigateTFState(attrs, tfFieldPath) +} + +// navigateTFState walks the TF state map using the given path. +// TF stores single-block fields as single-element arrays ([{…}]). When a string-key +// step encounters a []any, it auto-descends into element [0] so callers can use plain +// paths like "continuous.pause_status" even though TF stores them as [{"pause_status":…}]. +func navigateTFState(data map[string]any, path *structpath.PathNode) (any, error) { + var current any = data + for _, node := range path.AsSlice() { + if current == nil { + return nil, nil + } + + if key, ok := node.StringKey(); ok { + // Auto-unwrap TF list-blocks: if the current value is a single-element + // array and the next step wants a map key, descend into element 0. + if arr, isArr := current.([]any); isArr { + if len(arr) == 0 { + return nil, nil + } + current = arr[0] + } + m, ok := current.(map[string]any) + if !ok { + return nil, fmt.Errorf("expected map at %q, got %T", key, current) + } + val, ok := m[key] + if !ok { + return nil, fmt.Errorf("%q: key not found", key) + } + current = val + } else if idx, ok := node.Index(); ok { + switch v := current.(type) { + case []any: + if idx < 0 || idx >= len(v) { + return nil, fmt.Errorf("index %d out of range (len %d)", idx, len(v)) + } + current = v[idx] + default: + // TF [0] on a non-slice (already unwrapped) is a no-op. + if idx == 0 { + continue + } + return nil, fmt.Errorf("index %d: not a slice (%T)", idx, current) + } + } + } + return current, nil } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index fb2179145d3..9b217062fc8 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -68,6 +68,10 @@ to the workspace so that subsequent deploys of this bundle use direct deployment Args: root.NoArgs, } + // --noplancheck kept for backward compatibility; the plan check was removed + // because the command no longer invokes the Terraform engine. + cmd.Flags().Bool("noplancheck", false, "No-op (kept for compatibility).") + cmd.RunE = func(cmd *cobra.Command, args []string) error { extraArgsStr := getCommonArgs(cmd) From 35448aa098295c2984b701f1b11f2b57b5d42b64 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 12:55:02 +0200 Subject: [PATCH 08/16] migrate: compute depends_on from refs before resolving, restore --noplancheck The refs map is shared with sv.Refs and gets mutated (entries deleted) during reference resolution. depends_on must be computed before that loop runs. Also restore --noplancheck as a no-op flag kept for backward compatibility (used by the invariant test suite for job_with_depends_on config). Co-authored-by: Isaac --- bundle/migrate/build_state.go | 44 ++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index af2b3449086..785c325bfad 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -4,15 +4,18 @@ import ( "context" "fmt" "maps" + "slices" "strings" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" @@ -120,6 +123,45 @@ func BuildStateFromTF( sv := structvar.NewStructVar(newStateValue, refs) + // Compute depends_on from cross-resource references before resolving them + // (resolution deletes entries from the refs map). + // Same logic as makePlan in bundle/direct/bundle_plan.go. + var dependsOn []deployplan.DependsOnEntry //nolint:prealloc + for _, refTemplate := range refs { + ref, ok := dynvar.NewRef(dyn.V(refTemplate)) + if !ok { + continue + } + for _, targetPath := range ref.References() { + targetPathParsed, err := dyn.NewPathFromString(targetPath) + if err != nil { + continue + } + targetNodeDP, _ := config.GetNodeAndType(targetPathParsed) + targetNode := targetNodeDP.String() + fullRef := "${" + targetPath + "}" + found := false + for _, dep := range dependsOn { + if dep.Node == targetNode && dep.Label == fullRef { + found = true + break + } + } + if !found { + dependsOn = append(dependsOn, deployplan.DependsOnEntry{ + Node: targetNode, + Label: fullRef, + }) + } + } + } + slices.SortFunc(dependsOn, func(a, b deployplan.DependsOnEntry) int { + if a.Node != b.Node { + return strings.Compare(a.Node, b.Node) + } + return strings.Compare(a.Label, b.Label) + }) + // Resolve each reference using TF state. // node format: "resources.." or "resources...permissions" parts := strings.SplitN(node, ".", 4) @@ -170,7 +212,7 @@ func BuildStateFromTF( } } - if err := stateDB.SaveState(node, idEntry.ID, sv.Value, nil); err != nil { + if err := stateDB.SaveState(node, idEntry.ID, sv.Value, dependsOn); err != nil { return fmt.Errorf("%s: SaveState: %w", node, err) } } From aea8177adee78955aabbc460d3fb9ef7ab542937 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 5 Jun 2026 15:01:45 +0200 Subject: [PATCH 09/16] migrate: fix TF field lookup for postgres spec wrapper and model_id alias Two categories of field lookup failures in LookupTFField: 1. Postgres resources (postgres_projects, postgres_branches, etc.) use a 'spec' wrapper via DABsToTerraformWrappers, but some fields like 'name' are at the TF state root, not under spec. When the spec-prefixed path fails, retry with the original unwrapped path. 2. Model permissions reference 'model_id' (the numeric model ID) which TF stores as 'registered_model_id'. Add a tfStateFieldAliases map for such state-only field name mismatches. Also update acceptance output files: - default-python: remove musterr/--noplancheck pattern; the plan check that made the first migrate call fail no longer exists, so just call migrate once. - runas: remove plan-check output lines that appeared before 'Success!'. Co-authored-by: Isaac --- .../bundle/migrate/default-python/output.txt | 10 +---- .../bundle/migrate/default-python/script | 3 +- acceptance/bundle/migrate/runas/output.txt | 13 ------ bundle/migrate/tf_state.go | 44 ++++++++++++++++++- 4 files changed, 45 insertions(+), 25 deletions(-) diff --git a/acceptance/bundle/migrate/default-python/output.txt b/acceptance/bundle/migrate/default-python/output.txt index 92efb3066d6..fb554bc258d 100644 --- a/acceptance/bundle/migrate/default-python/output.txt +++ b/acceptance/bundle/migrate/default-python/output.txt @@ -22,15 +22,7 @@ Deployment complete! >>> print_state.py ->>> musterr [CLI] bundle deployment migrate -Building python_artifact... -Building python_artifact... -update jobs.sample_job - -Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged -Error: 'databricks bundle plan' shows actions planned, aborting migration. Please run 'databricks bundle deploy' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option - ->>> [CLI] bundle deployment migrate --noplancheck +>>> [CLI] bundle deployment migrate Building python_artifact... Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/my_default_python/.databricks/bundle/dev/resources.json diff --git a/acceptance/bundle/migrate/default-python/script b/acceptance/bundle/migrate/default-python/script index c9b585dbea7..e3a0b9b3e05 100755 --- a/acceptance/bundle/migrate/default-python/script +++ b/acceptance/bundle/migrate/default-python/script @@ -5,8 +5,7 @@ cd my_default_python trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy trace print_state.py > ../out.state_original.json -trace musterr $CLI bundle deployment migrate -trace $CLI bundle deployment migrate --noplancheck +trace $CLI bundle deployment migrate trace print_state.py > ../out.state_after_migration.json trace jq '.. | .libraries? | select(.)' ../out.state_after_migration.json diff --git a/acceptance/bundle/migrate/runas/output.txt b/acceptance/bundle/migrate/runas/output.txt index f7034d74ec0..f8a9b475bdc 100644 --- a/acceptance/bundle/migrate/runas/output.txt +++ b/acceptance/bundle/migrate/runas/output.txt @@ -81,19 +81,6 @@ Consider using a adding a top-level permissions section such as the following: See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. in databricks.yml:5:3 -Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups -If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. - -Consider using a adding a top-level permissions section such as the following: - - permissions: - - user_name: [USERNAME] - level: CAN_MANAGE - -See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. - in databricks.yml:5:3 - -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/production/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 52799dd79b8..90a5d06bb39 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -11,6 +11,15 @@ import ( tfjson "github.com/hashicorp/terraform-json" ) +// tfStateFieldAliases maps DABs group → DABs field name → TF state field name for +// cases where a DABs state-computed field has a different name in TF state. +// These fields are not captured by DABsToTerraformRenameMap because they are +// state-only (not part of the bundle config struct). +var tfStateFieldAliases = map[string]map[string]string{ + // models.model_id is the numeric model ID; TF stores it as registered_model_id. + "models": {"model_id": "registered_model_id"}, +} + // TFStateAttrs maps (tfResourceType → resourceName → raw JSON attributes). type TFStateAttrs map[string]map[string]json.RawMessage @@ -81,7 +90,40 @@ func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath return nil, fmt.Errorf("cannot parse TF state for %s.%s: %w", tfType, name, err) } - return navigateTFState(attrs, tfFieldPath) + value, err := navigateTFState(attrs, tfFieldPath) + if err == nil { + return value, nil + } + + // Some DABs fields are top-level in TF state but DABsPathToTerraform added a + // wrapper prefix (e.g. "spec" for postgres resources). When the wrapped path + // fails, retry with the original unwrapped path. + if _, hasWrapper := terraform_dabs_map.DABsToTerraformWrappers[group]; hasWrapper { + if v, e := navigateTFState(attrs, fieldPath); e == nil { + return v, nil + } + } + + // Apply state-only field aliases for fields whose DABs name differs from TF state name. + if aliases, ok := tfStateFieldAliases[group]; ok { + // Replace the first path segment if it matches a known alias. + if head, ok := fieldPath.StringKey(); ok { + if tfName, ok := aliases[head]; ok { + aliasPath := structpath.NewStringKey(nil, tfName) + if rest := fieldPath.SkipPrefix(1); rest != nil { + _ = rest // navigate through the alias root + } + // Translate aliased path with full DABsToTerraform for the renamed field. + if aliasFieldPath, e := terraform_dabs_map.DABsPathToTerraform(group, aliasPath); e == nil { + if v, e := navigateTFState(attrs, aliasFieldPath); e == nil { + return v, nil + } + } + } + } + } + + return nil, err } // navigateTFState walks the TF state map using the given path. From c8f326ffa2c04e881b7274b057651564abca3e04 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 5 Jun 2026 17:08:20 +0200 Subject: [PATCH 10/16] acceptance: remove plan check output from snapshot-comparison Co-authored-by: Isaac --- acceptance/bundle/deploy/snapshot-comparison/output.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/acceptance/bundle/deploy/snapshot-comparison/output.txt b/acceptance/bundle/deploy/snapshot-comparison/output.txt index 1db9fa5024d..b5bb597e264 100644 --- a/acceptance/bundle/deploy/snapshot-comparison/output.txt +++ b/acceptance/bundle/deploy/snapshot-comparison/output.txt @@ -8,8 +8,6 @@ Deployment complete! === Run migrate on bundle 1 >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/bundle1/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. From 7636ff935602426f5a9df16ab3f09a6a6376d856 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 5 Jun 2026 21:30:21 +0200 Subject: [PATCH 11/16] migrate: add unit tests for BuildStateFromTF Tests cover: - Basic job stored with correct ID - Resource absent from TF state is skipped - Cross-resource string ref: depends_on computed, field resolved from TF state - Cross-resource numeric ref: int value stored as number not string - Dashboard etag stored from etags map Co-authored-by: Isaac --- bundle/migrate/build_state_test.go | 223 +++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 bundle/migrate/build_state_test.go diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go new file mode 100644 index 00000000000..2a673f7fe75 --- /dev/null +++ b/bundle/migrate/build_state_test.go @@ -0,0 +1,223 @@ +package migrate_test + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/migrate" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/yamlloader" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// rootFromYAML builds a config.Root from a YAML snippet. +// Template strings like "${resources.jobs.src.name}" are preserved in the +// internal dyn.Value so BuildStateFromTF can find them via ExtractReferences. +func rootFromYAML(t *testing.T, yaml string) config.Root { + t.Helper() + v, err := yamlloader.LoadYAML("test", bytes.NewBufferString(yaml)) + require.NoError(t, err) + var root config.Root + require.NoError(t, convert.ToTyped(&root, v)) + require.NoError(t, root.Mutate(func(_ dyn.Value) (dyn.Value, error) { return v, nil })) + return root +} + +// runBuildStateFromTF is a helper that runs BuildStateFromTF, finalizes the +// state, then reloads it so callers can inspect ResourceEntry (State + DependsOn). +func runBuildStateFromTF( + t *testing.T, + yaml string, + tfAttrs migrate.TFStateAttrs, + tfIDs terraform.ExportedResourcesMap, + etags map[string]string, +) map[string]dstate.ResourceEntry { + t.Helper() + + root := rootFromYAML(t, yaml) + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + + statePath := filepath.Join(t.TempDir(), "resources.json") + + var db dstate.DeploymentState + db.OpenWithData(statePath, dstate.NewDatabase("lineage", 1)) + require.NoError(t, db.UpgradeToWrite()) + + err = migrate.BuildStateFromTF(t.Context(), &root, adapters, &db, tfAttrs, tfIDs, etags) + require.NoError(t, err) + + _, err = db.Finalize(t.Context()) + require.NoError(t, err) + + // Reload from disk to access the full ResourceEntry (State JSON + DependsOn). + raw, err := os.ReadFile(statePath) + require.NoError(t, err) + var data dstate.Database + require.NoError(t, json.Unmarshal(raw, &data)) + return data.State +} + +func TestBuildStateFromTF_BasicJob(t *testing.T) { + bundleYAML := ` +resources: + jobs: + my_job: + name: "hello" +` + tfAttrs := migrate.TFStateAttrs{ + "databricks_job": { + "my_job": json.RawMessage(`{"id": "123", "name": "hello"}`), + }, + } + tfIDs := terraform.ExportedResourcesMap{ + "resources.jobs.my_job": {ID: "123"}, + } + + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + entry, ok := state["resources.jobs.my_job"] + require.True(t, ok) + assert.Equal(t, "123", entry.ID) + assert.Empty(t, entry.DependsOn) +} + +func TestBuildStateFromTF_ResourceNotInTFState_Skipped(t *testing.T) { + bundleYAML := ` +resources: + jobs: + new_job: + name: "new" + existing_job: + name: "existing" +` + tfAttrs := migrate.TFStateAttrs{ + "databricks_job": { + "existing_job": json.RawMessage(`{"id": "456", "name": "existing"}`), + }, + } + tfIDs := terraform.ExportedResourcesMap{ + "resources.jobs.existing_job": {ID: "456"}, + } + + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + assert.Contains(t, state, "resources.jobs.existing_job") + assert.NotContains(t, state, "resources.jobs.new_job") +} + +func TestBuildStateFromTF_DependsOnComputedFromRefs(t *testing.T) { + bundleYAML := ` +resources: + pipelines: + src: + name: "source-pipeline" + jobs: + dst: + name: "${resources.pipelines.src.name}" +` + tfAttrs := migrate.TFStateAttrs{ + "databricks_pipeline": { + "src": json.RawMessage(`{"id": "p1", "name": "source-pipeline"}`), + }, + "databricks_job": { + "dst": json.RawMessage(`{"id": "j1", "name": "source-pipeline"}`), + }, + } + tfIDs := terraform.ExportedResourcesMap{ + "resources.pipelines.src": {ID: "p1"}, + "resources.jobs.dst": {ID: "j1"}, + } + + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + entry, ok := state["resources.jobs.dst"] + require.True(t, ok) + + // depends_on must point back to the referenced pipeline + require.Len(t, entry.DependsOn, 1) + assert.Equal(t, deployplan.DependsOnEntry{ + Node: "resources.pipelines.src", + Label: "${resources.pipelines.src.name}", + }, entry.DependsOn[0]) + + // resolved field value + var jobState map[string]any + require.NoError(t, json.Unmarshal(entry.State, &jobState)) + assert.Equal(t, "source-pipeline", jobState["name"]) +} + +func TestBuildStateFromTF_NumericFieldReference(t *testing.T) { + // dst_job.max_concurrent_runs references src_job's int field. + // Verifies that the resolved value is stored as a number (not a string) + // and that depends_on is recorded. + bundleYAML := ` +resources: + jobs: + src_job: + name: "source" + max_concurrent_runs: 4 + dst_job: + name: "dest" + max_concurrent_runs: "${resources.jobs.src_job.max_concurrent_runs}" +` + tfAttrs := migrate.TFStateAttrs{ + "databricks_job": { + "src_job": json.RawMessage(`{"id": "111", "name": "source", "max_concurrent_runs": 4}`), + "dst_job": json.RawMessage(`{"id": "222", "name": "dest", "max_concurrent_runs": 4}`), + }, + } + tfIDs := terraform.ExportedResourcesMap{ + "resources.jobs.src_job": {ID: "111"}, + "resources.jobs.dst_job": {ID: "222"}, + } + + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + + entry, ok := state["resources.jobs.dst_job"] + require.True(t, ok) + + // depends_on must point to src_job + require.Len(t, entry.DependsOn, 1) + assert.Equal(t, "resources.jobs.src_job", entry.DependsOn[0].Node) + + // max_concurrent_runs must be stored as a number, not a string + var jobState map[string]any + require.NoError(t, json.Unmarshal(entry.State, &jobState)) + assert.EqualValues(t, 4, jobState["max_concurrent_runs"]) +} + +func TestBuildStateFromTF_EtagStoredForDashboard(t *testing.T) { + bundleYAML := ` +resources: + dashboards: + my_dash: + display_name: "My Dashboard" +` + tfAttrs := migrate.TFStateAttrs{ + "databricks_dashboard": { + "my_dash": json.RawMessage(`{"id": "d1", "display_name": "My Dashboard"}`), + }, + } + tfIDs := terraform.ExportedResourcesMap{ + "resources.dashboards.my_dash": {ID: "d1"}, + } + etags := map[string]string{ + "resources.dashboards.my_dash": "etag-abc123", + } + + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, etags) + entry, ok := state["resources.dashboards.my_dash"] + require.True(t, ok) + + var dashState map[string]any + require.NoError(t, json.Unmarshal(entry.State, &dashState)) + assert.Equal(t, "etag-abc123", dashState["etag"]) +} From a2cd7293ad728cdf1f634533c502f55328a11bcb Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 7 Jun 2026 20:39:22 +0200 Subject: [PATCH 12/16] =?UTF-8?q?migrate:=20pass=20nil=20client=20to=20Ini?= =?UTF-8?q?tAll=20=E2=80=94=20PrepareState=20never=20uses=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Isaac --- bundle/statemgmt/upload_state_for_yaml_sync.go | 2 +- cmd/bundle/deployment/migrate.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 4a6f804789c..4f5a2645e62 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -172,7 +172,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("failed to create uninterpolated config: %w", err) } - adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) + adapters, err := dresources.InitAll(nil) if err != nil { return false, err } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 9b217062fc8..57ca89f6f6f 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -175,7 +175,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric return root.ErrAlreadyPrinted } - adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) + adapters, err := dresources.InitAll(nil) if err != nil { return err } From 3bc926ff46351322dee5dac086821844d5fd254b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 7 Jun 2026 21:22:18 +0200 Subject: [PATCH 13/16] migrate: eliminate duplicate TF state reads; etag from attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both callers were reading and parsing the same .tfstate file twice — once for resource IDs, once for full attributes. Now a single ParseTFStateFull reads and unmarshals the file once, returning attrs, IDs, and lineage/serial together. Also drop the separate `etags map[string]string` parameter from BuildStateFromTF. The dashboard etag is a regular attribute in the TF state JSON ("etag" field), so LookupTFField finds it directly without any special-case plumbing. Implementation: - terraform: expose ParseResourcesStateFromBytes so callers can pass already-read bytes - migrate: add ParseTFStateFull / parseTFStateAttrsFromBytes - migrate: remove etags param; look up "etag" via LookupTFField - tests: etag test now puts the etag in the TF attributes JSON Co-authored-by: Isaac --- bundle/deploy/terraform/util.go | 24 ++++++++--- bundle/migrate/build_state.go | 13 +++--- bundle/migrate/build_state_test.go | 19 ++++----- bundle/migrate/tf_state.go | 40 ++++++++++++++++++- .../statemgmt/upload_state_for_yaml_sync.go | 33 +++------------ cmd/bundle/deployment/migrate.go | 16 +------- 6 files changed, 79 insertions(+), 66 deletions(-) diff --git a/bundle/deploy/terraform/util.go b/bundle/deploy/terraform/util.go index 7ca5e9a1d14..51df7f49cd0 100644 --- a/bundle/deploy/terraform/util.go +++ b/bundle/deploy/terraform/util.go @@ -51,6 +51,15 @@ type stateInstanceAttributes struct { } // Returns a mapping resourceKey -> stateInstanceAttributes +// ParseResourcesStateFromBytes parses a terraform state file from already-read bytes. +func ParseResourcesStateFromBytes(ctx context.Context, raw []byte) (ExportedResourcesMap, error) { + var state resourcesState + if err := json.Unmarshal(raw, &state); err != nil { + return nil, err + } + return resourcesStateToMap(ctx, &state) +} + func parseResourcesState(ctx context.Context, path string) (ExportedResourcesMap, error) { rawState, err := os.ReadFile(path) if err != nil { @@ -59,12 +68,10 @@ func parseResourcesState(ctx context.Context, path string) (ExportedResourcesMap } return nil, err } - var state resourcesState - err = json.Unmarshal(rawState, &state) - if err != nil { - return nil, err - } + return ParseResourcesStateFromBytes(ctx, rawState) +} +func resourcesStateToMap(ctx context.Context, state *resourcesState) (ExportedResourcesMap, error) { if state.Version != SupportedStateVersion { return nil, fmt.Errorf("unsupported deployment state version: %d. Try re-deploying the bundle", state.Version) } @@ -131,5 +138,10 @@ func ParseResourcesState(ctx context.Context, b *bundle.Bundle) (ExportedResourc return nil, err } filename, _ := b.StateFilenameTerraform(ctx) - return parseResourcesState(ctx, filepath.Join(cacheDir, filename)) + return ParseResourcesStateFromPath(ctx, filepath.Join(cacheDir, filename)) +} + +// ParseResourcesStateFromPath parses a terraform state file at a known path. +func ParseResourcesStateFromPath(ctx context.Context, path string) (ExportedResourcesMap, error) { + return parseResourcesState(ctx, path) } diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 785c325bfad..1accf0e576e 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -32,7 +32,6 @@ func BuildStateFromTF( stateDB *dstate.DeploymentState, tfAttrs TFStateAttrs, tfIDs terraform.ExportedResourcesMap, - etags map[string]string, ) error { // Collect all resource nodes (same patterns as makePlan). var nodes []string @@ -205,10 +204,14 @@ func BuildStateFromTF( return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) } - // Handle etag for dashboards. - if etag := etags[node]; etag != "" { - if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { - return fmt.Errorf("%s: cannot set etag: %w", node, err) + // Handle etag for dashboards: look it up directly from TF state attributes. + // The "etag" field is a computed TF attribute not present in the bundle config, + // so it does not flow through PrepareState/ExtractReferences. + if etag, err := LookupTFField(tfAttrs, group, srcName, structpath.NewStringKey(nil, "etag")); err == nil && etag != nil { + if etagStr, ok := etag.(string); ok && etagStr != "" { + if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etagStr); err != nil { + return fmt.Errorf("%s: cannot set etag: %w", node, err) + } } } diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index 2a673f7fe75..a7db542e09e 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -40,7 +40,6 @@ func runBuildStateFromTF( yaml string, tfAttrs migrate.TFStateAttrs, tfIDs terraform.ExportedResourcesMap, - etags map[string]string, ) map[string]dstate.ResourceEntry { t.Helper() @@ -54,7 +53,7 @@ func runBuildStateFromTF( db.OpenWithData(statePath, dstate.NewDatabase("lineage", 1)) require.NoError(t, db.UpgradeToWrite()) - err = migrate.BuildStateFromTF(t.Context(), &root, adapters, &db, tfAttrs, tfIDs, etags) + err = migrate.BuildStateFromTF(t.Context(), &root, adapters, &db, tfAttrs, tfIDs) require.NoError(t, err) _, err = db.Finalize(t.Context()) @@ -84,7 +83,7 @@ resources: "resources.jobs.my_job": {ID: "123"}, } - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) entry, ok := state["resources.jobs.my_job"] require.True(t, ok) assert.Equal(t, "123", entry.ID) @@ -109,7 +108,7 @@ resources: "resources.jobs.existing_job": {ID: "456"}, } - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) assert.Contains(t, state, "resources.jobs.existing_job") assert.NotContains(t, state, "resources.jobs.new_job") } @@ -137,7 +136,7 @@ resources: "resources.jobs.dst": {ID: "j1"}, } - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) entry, ok := state["resources.jobs.dst"] require.True(t, ok) @@ -179,7 +178,7 @@ resources: "resources.jobs.dst_job": {ID: "222"}, } - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, nil) + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) entry, ok := state["resources.jobs.dst_job"] require.True(t, ok) @@ -195,6 +194,7 @@ resources: } func TestBuildStateFromTF_EtagStoredForDashboard(t *testing.T) { + // Etag is a top-level attribute in the TF dashboard state JSON; no separate map needed. bundleYAML := ` resources: dashboards: @@ -203,17 +203,14 @@ resources: ` tfAttrs := migrate.TFStateAttrs{ "databricks_dashboard": { - "my_dash": json.RawMessage(`{"id": "d1", "display_name": "My Dashboard"}`), + "my_dash": json.RawMessage(`{"id": "d1", "display_name": "My Dashboard", "etag": "etag-abc123"}`), }, } tfIDs := terraform.ExportedResourcesMap{ "resources.dashboards.my_dash": {ID: "d1"}, } - etags := map[string]string{ - "resources.dashboards.my_dash": "etag-abc123", - } - state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs, etags) + state := runBuildStateFromTF(t, bundleYAML, tfAttrs, tfIDs) entry, ok := state["resources.dashboards.my_dash"] require.True(t, ok) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 90a5d06bb39..217e021f637 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -1,6 +1,7 @@ package migrate import ( + "context" "encoding/json" "fmt" "os" @@ -23,13 +24,50 @@ var tfStateFieldAliases = map[string]map[string]string{ // TFStateAttrs maps (tfResourceType → resourceName → raw JSON attributes). type TFStateAttrs map[string]map[string]json.RawMessage +// TFStateMeta holds the top-level metadata from a terraform state file. +type TFStateMeta struct { + Lineage string + Serial int +} + +// ParseTFStateFull reads the terraform state file once and returns the full +// attribute map, the resource ID map, and the state metadata (lineage/serial). +// Avoids reading and unmarshaling the file multiple times. +func ParseTFStateFull(ctx context.Context, path string) (TFStateAttrs, terraform.ExportedResourcesMap, TFStateMeta, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, nil, TFStateMeta{}, err + } + + var meta struct { + Lineage string `json:"lineage"` + Serial int `json:"serial"` + } + if err := json.Unmarshal(raw, &meta); err != nil { + return nil, nil, TFStateMeta{}, err + } + + attrs, err := parseTFStateAttrsFromBytes(raw) + if err != nil { + return nil, nil, TFStateMeta{}, err + } + ids, err := terraform.ParseResourcesStateFromBytes(ctx, raw) + if err != nil { + return nil, nil, TFStateMeta{}, err + } + return attrs, ids, TFStateMeta{Lineage: meta.Lineage, Serial: meta.Serial}, nil +} + // ParseTFStateAttrs parses the terraform state file returning full attribute JSON per resource. func ParseTFStateAttrs(path string) (TFStateAttrs, error) { raw, err := os.ReadFile(path) if err != nil { return nil, err } + return parseTFStateAttrsFromBytes(raw) +} +func parseTFStateAttrsFromBytes(raw []byte) (TFStateAttrs, error) { var state struct { Version int `json:"version"` Resources []struct { @@ -41,11 +79,9 @@ func ParseTFStateAttrs(path string) (TFStateAttrs, error) { } `json:"instances"` } `json:"resources"` } - if err := json.Unmarshal(raw, &state); err != nil { return nil, err } - result := make(TFStateAttrs) for _, r := range state.Resources { if r.Mode != tfjson.ManagedResourceMode || len(r.Instances) == 0 { diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 4f5a2645e62..77a010488a0 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -98,50 +98,27 @@ func uploadState(ctx context.Context, b *bundle.Bundle) error { } func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bundle, snapshotPath string) (bool, error) { - terraformResources, err := terraform.ParseResourcesState(ctx, b) + _, localTerraformPath := b.StateFilenameTerraform(ctx) + tfAttrs, terraformResources, tfMeta, err := migrate.ParseTFStateFull(ctx, localTerraformPath) if err != nil { return false, fmt.Errorf("failed to parse terraform state: %w", err) } - // ParseResourcesState returns nil when the terraform state file doesn't exist + // ParseTFStateFull returns nil IDs when the terraform state file doesn't exist // (e.g. first deploy with no resources). if terraformResources == nil { return false, nil } - _, localTerraformPath := b.StateFilenameTerraform(ctx) - data, err := os.ReadFile(localTerraformPath) - if err != nil { - return false, fmt.Errorf("failed to read terraform state: %w", err) - } - - tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) - if err != nil { - return false, fmt.Errorf("failed to read terraform state attributes: %w", err) - } - state := make(map[string]dstate.ResourceEntry) - etags := map[string]string{} - for key, resourceEntry := range terraformResources { state[key] = dstate.ResourceEntry{ ID: resourceEntry.ID, State: json.RawMessage("{}"), } - if resourceEntry.ETag != "" { - etags[key] = resourceEntry.ETag - } - } - - var tfState struct { - Lineage string `json:"lineage"` - Serial int `json:"serial"` - } - if err := json.Unmarshal(data, &tfState); err != nil { - return false, err } - migratedDB := dstate.NewDatabase(tfState.Lineage, tfState.Serial+1) + migratedDB := dstate.NewDatabase(tfMeta.Lineage, tfMeta.Serial+1) migratedDB.State = state var stateDB dstate.DeploymentState @@ -181,7 +158,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("upgrading state for apply: %w", err) } - if err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfAttrs, terraformResources, etags); err != nil { + if err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfAttrs, terraformResources); err != nil { return false, err } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 57ca89f6f6f..3c17e5c4be5 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/migrate" @@ -114,7 +113,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("reading %s: %w", localTerraformPath, err) } - terraformResources, err := terraform.ParseResourcesState(ctx, b) + tfAttrs, terraformResources, _, err := migrate.ParseTFStateFull(ctx, localTerraformPath) if err != nil { return fmt.Errorf("failed to parse terraform state: %w", err) } @@ -134,23 +133,12 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("state file %s already exists", localPath) } - tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) - if err != nil { - return fmt.Errorf("failed to read terraform state attributes: %w", err) - } - - etags := map[string]string{} - state := make(map[string]dstate.ResourceEntry) for key, resourceEntry := range terraformResources { state[key] = dstate.ResourceEntry{ ID: resourceEntry.ID, State: json.RawMessage("{}"), } - if resourceEntry.ETag != "" { - // dashboard: - etags[key] = resourceEntry.ETag - } } migratedDB := dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1) @@ -184,7 +172,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("upgrading state for apply: %w", err) } - if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, terraformResources, etags); err != nil { + if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, terraformResources); err != nil { return err } From 77ed3c926b4b4b62d0995691c5bec50534da155a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 7 Jun 2026 21:27:02 +0200 Subject: [PATCH 14/16] migrate: mark ParseTFStateAttrs as deadcode:allow Still useful as a standalone API; suppress the dead-code checker. Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 217e021f637..cd7ba7d380d 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -58,6 +58,7 @@ func ParseTFStateFull(ctx context.Context, path string) (TFStateAttrs, terraform return attrs, ids, TFStateMeta{Lineage: meta.Lineage, Serial: meta.Serial}, nil } +//deadcode:allow retained as standalone API for callers that only need attributes. // ParseTFStateAttrs parses the terraform state file returning full attribute JSON per resource. func ParseTFStateAttrs(path string) (TFStateAttrs, error) { raw, err := os.ReadFile(path) From 0cd208a598e3796fc9f9252993911cba4c9aa60e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 7 Jun 2026 21:53:12 +0200 Subject: [PATCH 15/16] migrate: fix gofmt in tf_state.go Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index cd7ba7d380d..a512446fa47 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -58,8 +58,9 @@ func ParseTFStateFull(ctx context.Context, path string) (TFStateAttrs, terraform return attrs, ids, TFStateMeta{Lineage: meta.Lineage, Serial: meta.Serial}, nil } -//deadcode:allow retained as standalone API for callers that only need attributes. // ParseTFStateAttrs parses the terraform state file returning full attribute JSON per resource. +// +//deadcode:allow retained as standalone API for callers that only need attributes. func ParseTFStateAttrs(path string) (TFStateAttrs, error) { raw, err := os.ReadFile(path) if err != nil { From bbbd75664bddea118a1f233681d4189af38909c0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 7 Jun 2026 22:22:36 +0200 Subject: [PATCH 16/16] migrate: handle missing TF state file in ParseTFStateFull Return (nil, nil, ...) when the state file doesn't exist, matching the old parseResourcesState behaviour. Empty bundles with no resources don't create a terraform.tfstate file. Co-authored-by: Isaac --- bundle/migrate/tf_state.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index a512446fa47..72b64e77f19 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -3,6 +3,7 @@ package migrate import ( "context" "encoding/json" + "errors" "fmt" "os" @@ -36,6 +37,9 @@ type TFStateMeta struct { func ParseTFStateFull(ctx context.Context, path string) (TFStateAttrs, terraform.ExportedResourcesMap, TFStateMeta, error) { raw, err := os.ReadFile(path) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil, TFStateMeta{}, nil + } return nil, nil, TFStateMeta{}, err }