feat(kernel): SEA-via-kernel backend (opt-in, behind build tag)

.github/workflows/go.yml

@@ -90,6 +90,12 @@ jobs:
           fi
           go get -v -t -d ./...
 
+      # This matrix builds the default pure-Go driver (CGO_ENABLED=0), which does
+      # NOT compile the SEA-via-kernel backend (//go:build cgo && databricks_kernel).
+      # A dedicated CGO_ENABLED=1 `-tags databricks_kernel` test job needs the
+      # kernel static library linked in CI, which arrives with the kernel
+      # distribution work (a pinned-revision source build or a published .a).
+      # Until then the kernel path's cgo files are not exercised here.
       - name: Test
         run: make test
         env:

connector.go

@@ -15,6 +15,7 @@ import (
 	"github.com/databricks/databricks-sql-go/auth/pat"
 	"github.com/databricks/databricks-sql-go/auth/tokenprovider"
 	"github.com/databricks/databricks-sql-go/driverctx"
+	"github.com/databricks/databricks-sql-go/internal/backend"
 	"github.com/databricks/databricks-sql-go/internal/backend/thrift"
 	"github.com/databricks/databricks-sql-go/internal/client"
 	"github.com/databricks/databricks-sql-go/internal/config"
@@ -32,9 +33,18 @@ type connector struct {
 func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 	defer debuglog.Track(ctx, "connector.Connect", "host=%s", c.cfg.Host)()
 
-	// Build the execution backend. Currently always the Thrift backend; a second
-	// SEA-via-kernel backend will be selected here by a connect-time flag.
-	be, err := thrift.New(ctx, c.cfg, c.client)
+	// Build the execution backend. Thrift is the default; the SEA-via-kernel
+	// backend is selected when UseKernel is set. newKernelBackend is build-tag
+	// gated: in the default pure-Go build it returns a clear "not linked in"
+	// error, so the kernel path compiles and links only under -tags
+	// databricks_kernel + CGO_ENABLED=1.
+	var be backend.Backend
+	var err error
+	if c.cfg.UseKernel {
+		be, err = newKernelBackend(ctx, c.cfg)
+	} else {
+		be, err = thrift.New(ctx, c.cfg, c.client)
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -77,7 +87,14 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 		conn.telemetry.RecordOperation(ctx, conn.id, "", telemetry.OperationTypeCreateSession, sessionLatencyMs, nil)
 	}
 
-	log.Info().Msgf("connect: host=%s port=%d httpPath=%s serverProtocolVersion=0x%X", c.cfg.Host, c.cfg.Port, c.cfg.HTTPPath, be.ServerProtocolVersion())
+	// ServerProtocolVersion is Thrift-specific (not on the neutral backend
+	// interface); the kernel backend has no negotiated Thrift protocol, so log it
+	// only when present.
+	if tb, ok := be.(*thrift.Backend); ok {
+		log.Info().Msgf("connect: host=%s port=%d httpPath=%s serverProtocolVersion=0x%X", c.cfg.Host, c.cfg.Port, c.cfg.HTTPPath, tb.ServerProtocolVersion())
+	} else {
+		log.Info().Msgf("connect: host=%s port=%d httpPath=%s backend=kernel", c.cfg.Host, c.cfg.Port, c.cfg.HTTPPath)
+	}
 
 	return conn, nil
 }
@@ -301,6 +318,26 @@ func WithHTTPPath(path string) ConnOption {
 	}
 }
 
+// WithUseKernel selects the SEA-via-kernel backend instead of the default
+// Thrift backend. It has effect only in a build compiled with
+// `-tags databricks_kernel` and CGO_ENABLED=1; in the default pure-Go build a
+// connection made with this option set returns a clear error at connect time
+// (the kernel backend is not linked in).
+func WithUseKernel(useKernel bool) ConnOption {
+	return func(c *config.Config) {
+		c.UseKernel = useKernel
+	}
+}
+
+// WithWarehouseID sets the bare SQL warehouse id. The kernel backend addresses a
+// warehouse by id; when set it is preferred over the http path. The Thrift
+// backend ignores it and continues to route by http path.
+func WithWarehouseID(id string) ConnOption {
+	return func(c *config.Config) {
+		c.WarehouseID = id
+	}
+}
+
 // WithMaxRows sets up the max rows fetched per request. Default is 10000
 func WithMaxRows(n int) ConnOption {
 	return func(c *config.Config) {

doc.go

@@ -155,6 +155,70 @@ The result log may look like this:
 
 	{"level":"debug","connId":"01ed6545-5669-1ec7-8c7e-6d8a1ea0ab16","corrId":"workflow-example","queryId":"01ed6545-57cc-188a-bfc5-d9c0eaf8e189","time":1668558402,"message":"Run Main elapsed time: 1.298712292s"}
 
+# SEA-via-kernel backend (experimental)
+
+By default the driver uses the Thrift/HiveServer2 backend and is pure Go
+(CGO_ENABLED=0, go-gettable, cross-compilable). An experimental second backend
+runs statements over the Statement Execution API via the Rust
+databricks-sql-kernel, reached through a cgo C ABI. It is opt-in and compiled in
+only under a build tag, so the default build is unchanged.
+
+To use it, build with the databricks_kernel tag and CGO enabled, and select it
+per connection with WithUseKernel:
+
+	CGO_ENABLED=1 go build -tags databricks_kernel ./...
+
+	connector, _ := dbsql.NewConnector(
+		dbsql.WithServerHostname(host),
+		dbsql.WithHTTPPath(httpPath),
+		dbsql.WithAccessToken(token),
+		dbsql.WithUseKernel(true),
+	)
+	db := sql.OpenDB(connector)
+
+In a build without the tag, WithUseKernel(true) returns a clear error at connect
+time rather than silently using Thrift.
+
+To see the kernel's own logs interleaved with the driver's, set DBSQL_KERNEL_DEBUG
+to any non-empty value. That single flag turns on the driver's binding step tracer
+AND installs the kernel's Rust log subscriber, so both write to the same stderr in
+execution order. It is off by default (and must stay off during benchmarks — debug
+logging perturbs latency). The kernel's verbosity is then controlled by RUST_LOG,
+which the kernel honors directly; filter on the target databricks::sql::kernel
+(note the colons — it is the kernel's explicit log target, NOT the crate module
+path databricks_sql_kernel, which would match nothing):
+
+	# kernel logs only:
+	DBSQL_KERNEL_DEBUG=1 RUST_LOG=databricks::sql::kernel=debug ./your_app 2>&1
+	# kernel logs plus its HTTP stack (hyper/reqwest):
+	DBSQL_KERNEL_DEBUG=1 RUST_LOG=debug ./your_app 2>&1
+
+The kernel backend currently supports PAT authentication; reading scalar, nested,
+and complex-typed results (CloudFetch is handled transparently); context
+cancellation during execute (a cancelled ctx fires a real server-side cancel; on
+the read path cancellation is honored at result-batch boundaries, not mid-fetch);
+and the TLS, proxy, and session-conf (query tags, statement timeout, time zone)
+connection options. OAuth (M2M/U2M), initial catalog/schema, and
+WithEnableMetricViewMetadata are not yet supported and return a clear error at
+connect time rather than being silently ignored; likewise WithTimeout (a server
+query timeout the kernel C ABI can't set) and WithRetries used to disable retries
+(the kernel retries internally). Bound query parameters are likewise not yet
+supported and return a clear error at execute time (they arrive per-query, not at
+connect). None of these is silently ignored. (Metadata is issued as ordinary SQL
+— SHOW/DESCRIBE/information_schema — and runs on this backend like any other
+query.)
+
+WithMaxRows and retry tuning (WithRetries with a positive limit) are accepted but
+not applied on the kernel path: the kernel manages result fetching and retries
+internally, below the C ABI, with no user-facing knob.
+
+Two further kernel-backend caveats. The INTERVAL types (year-month / day-time)
+listed in the type table below are not yet handled by the kernel scanner and
+return a scan error; use the default (Thrift) backend for interval columns. And
+the kernel backend does not yet surface a per-statement server query id, so a
+QueryIdCallback (see below) fires with "" and no EXECUTE_STATEMENT telemetry is
+emitted for kernel queries.
+
 # Programmatically Retrieving Connection and Query Id
 
 Use the driverctx package under driverctx/ctx.go to add callbacks to the query context to receive the connection id and query id.