-
Notifications
You must be signed in to change notification settings - Fork 493
fix(cli): rewrite pooler to direct for initial pg-delta db pull #5833
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -25,6 +25,7 @@ import ( | |
| "github.com/supabase/cli/internal/migration/new" | ||
| "github.com/supabase/cli/internal/migration/repair" | ||
| "github.com/supabase/cli/internal/utils" | ||
| "github.com/supabase/cli/internal/utils/flags" | ||
| "github.com/supabase/cli/pkg/migration" | ||
| ) | ||
|
|
||
|
|
@@ -158,6 +159,11 @@ func dumpRemoteSchema(ctx context.Context, path string, config pgconn.Config, fs | |
| } | ||
|
|
||
| func diffRemoteSchema(ctx context.Context, schema []string, path string, config pgconn.Config, usePgDeltaDiff bool, differ diff.DiffFunc, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { | ||
| initialPull := len(schema) == 0 | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
A normal non-initial Useful? React with 👍 / 👎. |
||
| if usePgDeltaDiff && initialPull && len(flags.ProjectRef) > 0 { | ||
| // Pooler→direct rewrite for pg-delta TARGET only (see ResolveDirectDbConfigForPgDelta). | ||
| config = utils.ResolveDirectDbConfigForPgDelta(config, flags.ProjectRef, utils.CurrentProfile.ProjectHost) | ||
| } | ||
| // Diff remote db (source) & shadow db (target) and write it as a new migration. | ||
| result, err := diff.DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, usePgDeltaDiff, options...) | ||
| if err != nil { | ||
|
|
@@ -181,6 +187,12 @@ func diffRemoteSchema(ctx context.Context, schema []string, path string, config | |
| return errors.Errorf("%w (debug bundle: %s)", errInSync, debugDir) | ||
| } | ||
| } | ||
| if usePgDeltaDiff && initialPull { | ||
| // Hint + optional introspection-specific error (see handleInitialPgDeltaEmptyPull). | ||
| if err := handleInitialPgDeltaEmptyPull(ctx, config, options...); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return errors.New(errInSync) | ||
| } | ||
| if err := utils.MkdirIfNotExistFS(fsys, filepath.Dir(path)); err != nil { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -277,6 +277,40 @@ func ProjectRefFromDirectDbHost(host string) (string, bool) { | |
| return matches[2], true | ||
| } | ||
|
|
||
| // IsPoolerDbHost reports whether host is a Supabase connection pooler endpoint. | ||
| func IsPoolerDbHost(host string) bool { | ||
| host = strings.ToLower(host) | ||
| return host == "pooler.supabase.com" || strings.HasSuffix(host, ".pooler.supabase.com") | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
This only recognizes production Useful? React with 👍 / 👎. |
||
| } | ||
|
|
||
| // ResolveDirectDbConfigForPgDelta rewrites a linked pooler connection to the direct | ||
| // Supabase database host so pg-delta can introspect the full remote catalog on an | ||
| // initial migration-style db pull. | ||
| // | ||
| // Inverse of PoolerFallbackConfig (container tried direct, retry pooler): the CLI may | ||
| // have picked pooler in NewDbConfigWithPassword when the host OS cannot dial direct, | ||
| // but pg-delta runs inside Docker and is given this config as its remote TARGET only. | ||
| func ResolveDirectDbConfigForPgDelta(config pgconn.Config, projectRef, projectHost string) pgconn.Config { | ||
| if _, ok := ProjectRefFromDirectDbHost(config.Host); ok { | ||
| return config | ||
| } | ||
| if !IsPoolerDbHost(config.Host) { | ||
| return config | ||
| } | ||
| direct := config | ||
| direct.Host = GetSupabaseDbHost(projectRef) | ||
| direct.Port = 5432 | ||
| direct.User = "postgres" | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When Useful? React with 👍 / 👎. |
||
| // Drop Supavisor tenant routing; not used on direct connections. | ||
| if len(direct.RuntimeParams) > 0 { | ||
| delete(direct.RuntimeParams, "options") | ||
| if len(direct.RuntimeParams) == 0 { | ||
| direct.RuntimeParams = nil | ||
| } | ||
| } | ||
| return direct | ||
| } | ||
|
|
||
| // WarnIPv6PoolerFallback prints a user-visible warning explaining that the direct | ||
| // database connection could not be used because the current environment does not | ||
| // support IPv6, and that the CLI is retrying through the IPv4 connection pooler. | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -29,7 +29,10 @@ import { | |
| } from "../../../shared/legacy-db-config.toml-read.ts"; | ||
| import type { LegacyDbConnType } from "../../../shared/legacy-db-target-flags.ts"; | ||
| import { legacyMakeDir } from "../../../shared/legacy-make-dir.ts"; | ||
| import { legacyToPostgresURL } from "../../../shared/legacy-postgres-url.ts"; | ||
| import { | ||
| legacyHostFromPostgresURL, | ||
| legacyToPostgresURL, | ||
| } from "../../../shared/legacy-postgres-url.ts"; | ||
| import { legacySchemaToCsvField } from "../../../shared/legacy-schema-flags.ts"; | ||
| import { LegacyLinkedProjectCache } from "../../../telemetry/legacy-linked-project-cache.service.ts"; | ||
| import { LegacyTelemetryState } from "../../../telemetry/legacy-telemetry-state.service.ts"; | ||
|
|
@@ -49,6 +52,7 @@ import { legacyStreamPgDump } from "../shared/legacy-pg-dump.run.ts"; | |
| import { | ||
| legacyEmitPoolerFallbackWarning, | ||
| legacyIsDirectLinkedHost, | ||
| legacyResolveDirectDbConfigForPgDelta, | ||
| legacyRunWithPoolerFallback, | ||
| } from "../shared/legacy-pooler-fallback.ts"; | ||
| import { legacyDumpSchemaScript } from "../shared/legacy-pg-dump.scripts.ts"; | ||
|
|
@@ -64,7 +68,12 @@ import { | |
| legacyExportCatalogPgDelta, | ||
| legacyIsPgDeltaDebugEnabled, | ||
| } from "../shared/legacy-pgdelta.ts"; | ||
| import { legacySaveEmptyPgDeltaPullDebug } from "./pull.debug.ts"; | ||
| import { | ||
| LEGACY_PG_DELTA_INTROSPECTION_ERROR, | ||
| legacyFormatInitialPgDeltaEmptyPullHint, | ||
| legacySaveEmptyPgDeltaPullDebug, | ||
| legacySummarizeCatalogJson, | ||
| } from "./pull.debug.ts"; | ||
| import { LegacyDeclarativeSeam } from "../shared/legacy-pgdelta.seam.service.ts"; | ||
| import type { LegacyDbPullFlags } from "./pull.command.ts"; | ||
| import { | ||
|
|
@@ -267,14 +276,16 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy | |
| const withPoolerFallback = <A, E extends { readonly message: string }, R>( | ||
| directTarget: string, | ||
| attempt: (targetRef: string) => Effect.Effect<A, E, R>, | ||
| ) => | ||
| attempt(directTarget).pipe( | ||
| ) => { | ||
| // Pooler fallback gates on the container TARGET host, not `resolved.conn.host`. | ||
| const targetHost = legacyHostFromPostgresURL(directTarget); | ||
| return attempt(directTarget).pipe( | ||
| Effect.catch((error) => | ||
| Effect.gen(function* () { | ||
| if ( | ||
| legacyIsDirectLinkedHost({ | ||
| connType, | ||
| host: resolved.conn.host, | ||
| host: targetHost, | ||
| isLocal: resolved.isLocal, | ||
| projectHost: cliConfig.projectHost, | ||
| }) && | ||
|
|
@@ -292,14 +303,15 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy | |
| }) | ||
| .pipe(Effect.orElseSucceed(() => Option.none())); | ||
| if (Option.isSome(pooler)) { | ||
| yield* legacyEmitPoolerFallbackWarning(resolved.conn.host); | ||
| yield* legacyEmitPoolerFallbackWarning(targetHost); | ||
| return yield* attempt(connToUrl(pooler.value)); | ||
| } | ||
| } | ||
| return yield* Effect.fail(error); | ||
| }), | ||
| ), | ||
| ); | ||
| }; | ||
|
|
||
| const usePgDeltaDiff = legacyResolvePullDiffEngine({ | ||
| engineFlagChanged: Option.isSome(flags.diffEngine), | ||
|
|
@@ -447,14 +459,24 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy | |
| }), | ||
| ); | ||
| } | ||
| const initialPull = sync.kind === "missing"; | ||
| // Initial pg-delta pull only: if `--linked` resolved to a pooler host, rewrite | ||
| // the pg-delta TARGET to `db.<ref>.<host>` (see legacyResolveDirectDbConfigForPgDelta). | ||
| // The CLI postgres session (`resolved.conn`) is unchanged. | ||
| const pgDeltaTargetConn = | ||
| initialPull && usePgDeltaDiff && connType === "linked" && linkedRef !== undefined | ||
| ? legacyResolveDirectDbConfigForPgDelta(resolved.conn, linkedRef, cliConfig.projectHost) | ||
| : resolved.conn; | ||
| const pgDeltaTargetUrl = connToUrl(pgDeltaTargetConn); | ||
| const diffTargetUrl = initialPull && usePgDeltaDiff ? pgDeltaTargetUrl : targetUrl; | ||
| // Initial pull, migra engine (Go's `run` → `assertRemoteInSync` returns | ||
| // `errMissing`): seed the migration file with a pg_dump of the remote schema | ||
| // (`dumpRemoteSchema`, `pull.go:144-158`), then run the migra diff below as a | ||
| // second pass appended to the same file (`diffRemoteSchema(ctx, nil, …)`), | ||
| // which captures default privileges / managed schemas pg_dump can't emit. | ||
| // pg-delta initial pulls skip the dump (`pull.go:126` `if !usePgDeltaDiff`): | ||
| // they diff against an empty shadow, which already yields the full schema. | ||
| const seededFromDump = sync.kind === "missing" && !usePgDeltaDiff; | ||
| const seededFromDump = initialPull && !usePgDeltaDiff; | ||
| // Tracks whether the pg_dump seed wrote any bytes, for Go's | ||
| // `ensureMigrationWritten` (`pull.go:68,263-268`): an empty dump + empty diff | ||
| // is "in sync", a non-empty dump is a valid initial migration on its own. | ||
|
|
@@ -559,7 +581,7 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy | |
| // Native diff: shadow (baseline + local migrations) vs remote → migration SQL. | ||
| // For the initial pull (no local migrations) the schema filter is ignored, | ||
| // matching Go's `diffRemoteSchema(ctx, nil, …)`. | ||
| const diffSchema = sync.kind === "missing" ? [] : flags.schema; | ||
| const diffSchema = initialPull ? [] : flags.schema; | ||
| // Go's `DiffDatabase` emits these to stderr before provisioning + diffing | ||
| // (`internal/db/diff/diff.go:189,234-237`); the shadow seam doesn't, so the | ||
| // pull handler emits them itself to match the migration-style `db pull` output. | ||
|
|
@@ -581,7 +603,7 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy | |
| // Use the declarative target override when present (Go substitutes it | ||
| // for the diff target, `diff.go:196-197`); for remote pulls it's | ||
| // undefined, so this is the direct target URL as before. | ||
| const target = shadow.targetUrlOverride ?? targetUrl; | ||
| const target = shadow.targetUrlOverride ?? diffTargetUrl; | ||
| yield* output.raw( | ||
| diffSchema.length > 0 | ||
| ? `Diffing schemas: ${diffSchema.join(",")}\n` | ||
|
|
@@ -647,8 +669,8 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy | |
| if (diffOutcome.capture !== undefined) { | ||
| const debugDir = yield* legacySaveEmptyPgDeltaPullDebug({ | ||
| ctx, | ||
| conn: resolved.conn, | ||
| targetUrl, | ||
| conn: pgDeltaTargetConn, | ||
| targetUrl: pgDeltaTargetUrl, | ||
| sourceCatalog: diffOutcome.capture.sourceCatalog, | ||
| pgDeltaStderr: diffOutcome.capture.stderr, | ||
| id: legacyFormatDebugId(yield* Clock.currentTimeMillis), | ||
|
|
@@ -673,6 +695,23 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy | |
| ); | ||
| } | ||
| } | ||
| if (initialPull && usePgDeltaDiff && !legacyIsPgDeltaDebugEnabled()) { | ||
| // Shown on every empty initial pg-delta pull (unless PGDELTA_DEBUG took the | ||
| // debug-bundle path above). Helps when pooler introspection silently fails. | ||
| yield* output.raw(`${legacyFormatInitialPgDeltaEmptyPullHint()}\n`, "stderr"); | ||
| // Probe the remote catalog: if objects exist but pg-delta returned no SQL, | ||
| // fail with a specific introspection error instead of generic "in sync". | ||
| const targetCatalog = yield* legacyExportCatalogPgDelta(ctx, { | ||
| targetRef: pgDeltaTargetUrl, | ||
| role: "postgres", | ||
| }).pipe(Effect.catch(() => Effect.succeed(""))); | ||
| if (legacySummarizeCatalogJson(targetCatalog).totalObjects > 0) { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
For an initial pg-delta pull against a project with no user-defined schema, the catalog export can still contain default/managed schema entries, and Useful? React with 👍 / 👎. |
||
| return yield* Effect.fail( | ||
| new LegacyDbPullInSyncError({ message: LEGACY_PG_DELTA_INTROSPECTION_ERROR }), | ||
| ); | ||
| } | ||
| // Catalog also empty → truly nothing pullable; fall through to generic error. | ||
| } | ||
| return yield* Effect.fail( | ||
| new LegacyDbPullInSyncError({ message: "No schema changes found" }), | ||
| ); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For an initial pg-delta pull against a project with no user-defined schema,
exportTargetCatalogstill returns the default/managed catalog entries (for example schema nodes), andSummarizeCatalogJSONcounts any node with aschemafield. That meansTotalObjectscan be nonzero even though there is truly nothing to pull, so this path reports the new “could not introspect” error instead of the intended generic “No schema changes found”; base the probe on pullable user objects or the same Supabase filter used by the diff.Useful? React with 👍 / 👎.