diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 9164ced8..df12b05f 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,7 +2,11 @@ name: Go on: push: - branches: [main] + # TEMPORARY: mani/sea-kernel-build-recipe is here only to run CI on the + # stacked build-recipe branch (whose PR base is the feature branch, not + # main, so the pull_request trigger below won't fire for it). This entry + # MUST be removed before merge — do not ship it to main. + branches: [main, mani/sea-kernel-build-recipe] pull_request: branches: [main] @@ -92,10 +96,8 @@ jobs: # This matrix builds the default pure-Go driver (CGO_ENABLED=0), which does # NOT compile the SEA-via-kernel backend (//go:build cgo && databricks_kernel). - # A dedicated CGO_ENABLED=1 `-tags databricks_kernel` test job needs the - # kernel static library linked in CI, which arrives with the kernel - # distribution work (a pinned-revision source build or a published .a). - # Until then the kernel path's cgo files are not exercised here. + # That opt-in path is exercised by the separate build-and-test-kernel job + # below, which builds the kernel static library and links it with CGO. - name: Test run: make test env: @@ -106,3 +108,142 @@ jobs: - name: Build run: make linux + + build-and-test-kernel: + name: Test (kernel backend) + # Exercises the opt-in SEA-via-kernel backend: builds the Rust kernel static + # lib from the pinned KERNEL_REV and runs the `databricks_kernel`-tagged unit + # tests with CGO. Separate from build-and-test so that job's CGO_ENABLED=0 + # pure-Go invariant is untouched. + # + # Bound the blast radius of the source build: it clones an external repo and + # cold-compiles ~200 Rust crates, so a network stall or hung cargo would + # otherwise hold a protected-runner slot up to the 360-min default. A tight + # per-ref concurrency group also collapses redundant heavy builds when the + # branch is pushed repeatedly (each push cancels the prior in-flight run). + timeout-minutes: 30 + concurrency: + group: kernel-build-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest + + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + # Go module proxy (GOPROXY + ~/.netrc) via JFrog OIDC. Also exports + # JFROG_ACCESS_TOKEN, which the cargo step below reuses. + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + # The protected runner blocks direct crates.io access (go/hardened-gha), + # so point cargo at the JFrog crates proxy. This repo's setup-jfrog is + # Go-only, so configure cargo inline here reusing its JFROG_ACCESS_TOKEN. + # The token stays in ~/.cargo/credentials.toml (not a CARGO*-prefixed env + # var) so rust-cache keys stay stable across runs. + - name: Configure cargo to use JFrog + shell: bash + run: | + set -euo pipefail + mkdir -p ~/.cargo + cat > ~/.cargo/config.toml << 'EOF' + [source.crates-io] + replace-with = "jfrog" + + [source.jfrog] + registry = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + + [registries.jfrog] + index = "sparse+https://databricks.jfrog.io/artifactory/api/cargo/db-cargo-remote/index/" + credential-provider = ["cargo:token"] + EOF + cat > ~/.cargo/credentials.toml << EOF + [registries.jfrog] + token = "Bearer ${JFROG_ACCESS_TOKEN}" + EOF + chmod 600 ~/.cargo/credentials.toml + + - name: Set up Go Toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: '1.25.x' + cache: false + + # Install the exact toolchain the kernel .a is built with. rust-toolchain.toml + # at the repo root is what actually governs the cargo build (it's a parent of + # build/kernel-src/, and the kernel repo pins no toolchain of its own), so a + # floating `stable` here would drift the archive under a fixed KERNEL_REV. + # Keep this in lockstep with rust-toolchain.toml's channel. + - name: Set up Rust Toolchain + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 + with: + toolchain: 1.96.1 + + # The kernel repo (databricks/databricks-sql-kernel) is private, and the + # hardened runner has no ambient git credentials, so kernel-lib.sh's fetch + # fails with "could not read Username". Mint a repo-scoped token from the + # INTEGRATION_TEST_APP GitHub App (installed on the org with kernel read + # access — the same mechanism the ODBC driver uses) and rewrite the kernel + # HTTPS URL to carry it. This is transparent to kernel-lib.sh. Only needed + # while the kernel repo is private and this job builds it from source; drop + # it once the repo is public (the download path never clones the source). + - name: Generate GitHub App token for databricks-sql-kernel + id: kernel-token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 + with: + app-id: ${{ secrets.INTEGRATION_TEST_APP_ID }} + private-key: ${{ secrets.INTEGRATION_TEST_PRIVATE_KEY }} + owner: databricks + repositories: databricks-sql-kernel + + - name: Rewrite kernel repo URL to authenticated HTTPS + env: + TOKEN: ${{ steps.kernel-token.outputs.token }} + run: | + git config --global \ + url."https://x-access-token:${TOKEN}@github.com/databricks/".insteadOf \ + "https://github.com/databricks/" + + # Cache the built kernel .a keyed on KERNEL_REV: rebuild only when the pin + # moves. The ~85MB archive dwarfs a rebuild trigger, so keep the key tight. + - name: Cache kernel static lib + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + internal/backend/kernel/lib + internal/backend/kernel/include + key: ${{ runner.os }}-kernellib-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + + # Cache the cargo registry + kernel build tree so a cache miss on the .a + # is still an incremental Rust build, not a cold ~200-crate compile. + - name: Cache cargo + kernel build tree + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + build/kernel-src/target + key: ${{ runner.os }}-kernel-cargo-${{ hashFiles('KERNEL_REV', 'rust-toolchain.toml') }} + restore-keys: | + ${{ runner.os }}-kernel-cargo- + + # make (for the recipe) and a C compiler (cgo compiles the kernel binding + # stubs and links libdatabricks_sql_kernel.a). cc is preinstalled on the + # protected runner — the kernel repo's own c-abi job relies on the same — + # so these conditional installs are a no-op guard that keeps the job robust + # to a future image change rather than a known gap. + - name: Install build prerequisites (make, C compiler) + run: | + if ! command -v make &> /dev/null ; then + apt-get update && apt-get install -y make + fi + if ! command -v cc &> /dev/null ; then + apt-get update && apt-get install -y build-essential + fi + + # make test-kernel builds the kernel lib (make kernel-lib) if the cache + # missed, then runs `CGO_ENABLED=1 go test -tags databricks_kernel ./...`. + - name: Build kernel lib + run tagged tests + run: make test-kernel diff --git a/.gitignore b/.gitignore index 063361a8..970720b6 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,8 @@ _tmp* .vscode/ __debug_bin .DS_Store + +# Kernel (SEA) backend build artifacts — produced by `make kernel-lib`, never committed. +/build/kernel-src/ +/internal/backend/kernel/lib/ +/internal/backend/kernel/include/ diff --git a/KERNEL_REV b/KERNEL_REV new file mode 100644 index 00000000..cfbf3024 --- /dev/null +++ b/KERNEL_REV @@ -0,0 +1 @@ +9b90406 diff --git a/Makefile b/Makefile index ebe22c05..6744c9f9 100644 --- a/Makefile +++ b/Makefile @@ -81,3 +81,46 @@ build: linux darwin ## Build the multi-arch binaries $(PLATFORMS): mkdir -p bin GOOS=$(os) GOARCH=amd64 $(GO) build $(GOBUILD_ARGS) -ldflags '$(LDFLAGS)' -o bin/$(BINARY)-$(os)-amd64 . + +# ── Kernel (SEA) backend ────────────────────────────────────────────────────── +# Opt-in cgo build that links the Rust SQL kernel's C ABI, gated behind the +# `databricks_kernel` build tag. These targets are wholly separate from the +# pure-Go defaults above (which stay CGO_ENABLED=0 and never touch the kernel). +KERNEL_REV = $(shell cat KERNEL_REV) +KERNEL_REPO ?= https://github.com/databricks/databricks-sql-kernel.git +KERNEL_SRC ?= build/kernel-src +# Target OS/arch (honors GOOS/GOARCH overrides) vs the actual host, so the build +# step can reject a source cross-build that would drop a host .a into a foreign +# target dir. Multi-OS support is via native per-OS runners (host == target) or +# staging a prebuilt .a with KERNEL_LOCAL_A — not cross-compiling from source. +KERNEL_GOOS = $(shell go env GOOS) +KERNEL_GOARCH = $(shell go env GOARCH) +KERNEL_GOHOSTOS = $(shell go env GOHOSTOS) +KERNEL_GOHOSTARCH = $(shell go env GOHOSTARCH) +KERNEL_LIB_DIR = internal/backend/kernel/lib/$(KERNEL_GOOS)_$(KERNEL_GOARCH) +KERNEL_INC_DIR = internal/backend/kernel/include +KERNEL_GO = CGO_ENABLED=1 go +KERNEL_TAGS = -tags databricks_kernel + +.PHONY: kernel-lib +kernel-lib: ## Build the pinned kernel static lib + header into the cgo link dir (source build). + KERNEL_REPO="$(KERNEL_REPO)" KERNEL_REV="$(KERNEL_REV)" KERNEL_SRC="$(KERNEL_SRC)" \ + KERNEL_LIB_DIR="$(KERNEL_LIB_DIR)" KERNEL_INC_DIR="$(KERNEL_INC_DIR)" \ + KERNEL_GOOS="$(KERNEL_GOOS)" KERNEL_GOARCH="$(KERNEL_GOARCH)" \ + KERNEL_GOHOSTOS="$(KERNEL_GOHOSTOS)" KERNEL_GOHOSTARCH="$(KERNEL_GOHOSTARCH)" \ + KERNEL_LOCAL_A="$(KERNEL_LOCAL_A)" KERNEL_LOCAL_HEADER="$(KERNEL_LOCAL_HEADER)" \ + ./build/kernel-lib.sh + +.PHONY: build-kernel +build-kernel: kernel-lib ## Build the driver with the kernel backend linked. + $(KERNEL_GO) build $(KERNEL_TAGS) ./... + +.PHONY: test-kernel +test-kernel: kernel-lib ## Run the kernel-tagged unit tests (no warehouse needed). + $(KERNEL_GO) test $(KERNEL_TAGS) ./... + +.PHONY: kernel-lib-download +kernel-lib-download: ## Prod mode (download prebuilt .a): blocked until the kernel publishes release artifacts. + @echo "kernel-lib-download: blocked — the kernel does not yet publish per-platform .a release artifacts." + @echo "Use 'make kernel-lib' (source build) meanwhile. See the distribution design doc." + @false diff --git a/build/kernel-lib.sh b/build/kernel-lib.sh new file mode 100755 index 00000000..ffc17e6f --- /dev/null +++ b/build/kernel-lib.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# +# kernel-lib.sh — build (or copy in) the Databricks SQL kernel static library +# for the cgo `databricks_kernel` backend. +# +# Mode 1 (source build, the default and the only thing that works today): clone +# the kernel at the pinned KERNEL_REV, `cargo build` a self-contained static +# archive with pure-Rust TLS, and drop the .a + C header where the per-platform +# cgo_.go files link them (${SRCDIR}/lib/_ and ${SRCDIR}/include, +# both .gitignore'd). Kernel releases publish no binary assets yet, so building +# from a pinned commit is the current distribution model (mirrors the ODBC +# driver's Corrosion-from-source build). +# +# Local override: set KERNEL_LOCAL_A (and optionally KERNEL_LOCAL_HEADER) to +# copy an already-built archive instead of cloning + building — the analogue of +# the ODBC driver's KERNEL_LOCAL_PATH. +# +# Invoked by `make kernel-lib`, which supplies the environment below. It can +# also be run directly with the same variables set. +# +# Required env (Makefile provides these): +# KERNEL_REV kernel commit SHA to build (from the repo-root KERNEL_REV) +# KERNEL_REPO git URL of the kernel repo +# KERNEL_SRC working dir for the kernel checkout (gitignored) +# KERNEL_LIB_DIR dest dir for the .a (…/lib/_) +# KERNEL_INC_DIR dest dir for the header (…/include) +# Optional env: +# KERNEL_LOCAL_A path to a prebuilt libdatabricks_sql_kernel.a to copy +# KERNEL_LOCAL_HEADER path to a databricks_kernel.h to copy (defaults to a +# header next to KERNEL_LOCAL_A, else the checkout's) + +set -euo pipefail + +: "${KERNEL_LIB_DIR:?KERNEL_LIB_DIR must be set (run via 'make kernel-lib')}" +: "${KERNEL_INC_DIR:?KERNEL_INC_DIR must be set (run via 'make kernel-lib')}" + +LIB_NAME="libdatabricks_sql_kernel.a" +HEADER_NAME="databricks_kernel.h" + +log() { printf '[kernel-lib] %s\n' "$*" >&2; } + +emit_checksum() { + # A reproducibility breadcrumb: the archive path + its content hash. + local a="$KERNEL_LIB_DIR/$LIB_NAME" + if command -v sha256sum >/dev/null 2>&1; then + log "artifact: $a" + log "sha256: $(sha256sum "$a" | cut -d' ' -f1)" + elif command -v shasum >/dev/null 2>&1; then + log "artifact: $a" + log "sha256: $(shasum -a 256 "$a" | cut -d' ' -f1)" + fi +} + +mkdir -p "$KERNEL_LIB_DIR" "$KERNEL_INC_DIR" + +# ── Local override: copy a prebuilt archive, skip clone + cargo entirely. ────── +if [ -n "${KERNEL_LOCAL_A:-}" ]; then + [ -f "$KERNEL_LOCAL_A" ] || { log "KERNEL_LOCAL_A not found: $KERNEL_LOCAL_A"; exit 1; } + local_header="${KERNEL_LOCAL_HEADER:-}" + if [ -z "$local_header" ]; then + # Prefer a header next to the archive; fall back to the checkout's include/. + if [ -f "$(dirname "$KERNEL_LOCAL_A")/$HEADER_NAME" ]; then + local_header="$(dirname "$KERNEL_LOCAL_A")/$HEADER_NAME" + elif [ -n "${KERNEL_SRC:-}" ] && [ -f "$KERNEL_SRC/include/$HEADER_NAME" ]; then + local_header="$KERNEL_SRC/include/$HEADER_NAME" + fi + fi + [ -n "$local_header" ] && [ -f "$local_header" ] || { + log "header not found; set KERNEL_LOCAL_HEADER to a $HEADER_NAME"; exit 1; } + log "copying prebuilt archive from $KERNEL_LOCAL_A" + cp "$KERNEL_LOCAL_A" "$KERNEL_LIB_DIR/$LIB_NAME" + cp "$local_header" "$KERNEL_INC_DIR/$HEADER_NAME" + emit_checksum + exit 0 +fi + +# ── Mode 1: build from source at the pinned rev. ────────────────────────────── +: "${KERNEL_REV:?KERNEL_REV must be set (run via 'make kernel-lib')}" +: "${KERNEL_REPO:?KERNEL_REPO must be set (run via 'make kernel-lib')}" +: "${KERNEL_SRC:?KERNEL_SRC must be set (run via 'make kernel-lib')}" + +# Cache short-circuit: if a previously built archive + header are present AND the +# rev-stamp beside the archive matches KERNEL_REV, the artifact is already what +# this pin would produce — skip the git fetch + cargo build entirely (and don't +# even require a Rust toolchain). `kernel-lib` is .PHONY and CI re-invokes it on +# every run after restoring the .a cache (keyed on KERNEL_REV), so without this +# guard the build re-ran and overwrote the just-restored archive every time. +# The stamp (not just file existence) is what makes this safe against a local +# KERNEL_REV bump with a stale on-disk .a: a changed pin fails the match and +# forces a rebuild. In CI the .a-cache key already includes KERNEL_REV, so a pin +# change misses the cache and the artifact is simply absent — either way a bump +# rebuilds. +REV_STAMP="$KERNEL_LIB_DIR/.kernel-rev" +if [ -f "$KERNEL_LIB_DIR/$LIB_NAME" ] && [ -f "$KERNEL_INC_DIR/$HEADER_NAME" ] && + [ -f "$REV_STAMP" ] && [ "$(cat "$REV_STAMP")" = "$KERNEL_REV" ]; then + log "cache hit: $LIB_NAME already built for $KERNEL_REV — skipping source build" + emit_checksum + exit 0 +fi + +# Reject a source cross-build. `cargo build` below has no --target, so it emits +# the HOST triple's archive — but KERNEL_LIB_DIR is named for the TARGET +# (GOOS/GOARCH). If they differ, copying the host .a into the target dir would +# silently produce a wrong-arch artifact. Multi-OS is served by native per-OS +# runners (host == target, so this passes) or by staging a prebuilt cross-target +# .a via KERNEL_LOCAL_A (handled above, before this check) — not by cross- +# building the kernel from source, which the distribution design defers to the +# download path. Fail loud rather than mislink. Only enforced when the Makefile +# passes the host vars; a direct script call without them skips the check. +if [ -n "${KERNEL_GOOS:-}" ] && [ -n "${KERNEL_GOHOSTOS:-}" ] && + { [ "$KERNEL_GOOS" != "$KERNEL_GOHOSTOS" ] || [ "${KERNEL_GOARCH:-}" != "${KERNEL_GOHOSTARCH:-}" ]; }; then + log "refusing source cross-build: target ${KERNEL_GOOS}/${KERNEL_GOARCH} != host ${KERNEL_GOHOSTOS}/${KERNEL_GOHOSTARCH}." + log "build on a native ${KERNEL_GOOS} runner, or stage a prebuilt archive with KERNEL_LOCAL_A=." + exit 1 +fi + +command -v cargo >/dev/null 2>&1 || { + log "cargo not found — the source build needs a Rust toolchain." + log "install rustup, or use 'make kernel-lib KERNEL_LOCAL_A=' with a prebuilt .a." + exit 1 +} + +# Make $KERNEL_SRC a git repo pointed at the kernel remote, WITHOUT assuming an +# empty destination. CI caches build/kernel-src/target/ (for incremental Rust +# builds) but not .git, so on a cache hit the dir exists with a target/ subtree +# and no repo — a plain `git clone` would abort ("destination path already +# exists and is not an empty directory"). `git init` is idempotent and works +# whether the dir is absent, empty, or holds a restored target/; the kernel's +# own .gitignore excludes /target, so the later checkout leaves it untouched. +mkdir -p "$KERNEL_SRC" +if [ ! -d "$KERNEL_SRC/.git" ]; then + log "initializing git repo in $KERNEL_SRC (-> $KERNEL_REPO)" + git -C "$KERNEL_SRC" init --quiet +fi +if git -C "$KERNEL_SRC" remote get-url origin >/dev/null 2>&1; then + git -C "$KERNEL_SRC" remote set-url origin "$KERNEL_REPO" +else + git -C "$KERNEL_SRC" remote add origin "$KERNEL_REPO" +fi + +log "fetching + checking out $KERNEL_REV" +git -C "$KERNEL_SRC" fetch --all --tags --quiet || true +# checkout -f: this is a tool-managed, gitignored scratch checkout of a pinned +# third-party repo, so force past any leftover files (e.g. a source tree left +# behind if .git was lost). -f overwrites tracked-path collisions but leaves the +# gitignored target/ alone, so a cache-restored build tree survives. Without -f, +# an untracked-file conflict would masquerade as "commit not reachable" and send +# us down the PR-head fallback for the wrong reason. +# +# KERNEL_REV may be a bare commit that only lives on a PR ref (e.g. #163's head +# before it merges), so try the commit directly, then the PR head ref. +if ! git -C "$KERNEL_SRC" checkout -f --quiet "$KERNEL_REV" 2>/dev/null; then + log "commit not directly reachable; trying PR head refs" + git -C "$KERNEL_SRC" fetch --quiet origin '+refs/pull/*/head:refs/remotes/origin/pr/*' || true + git -C "$KERNEL_SRC" checkout -f --quiet "$KERNEL_REV" || { + log "could not check out $KERNEL_REV — is it pushed/fetchable?"; exit 1; } +fi +log "kernel at $(git -C "$KERNEL_SRC" rev-parse --short HEAD)" + +# --no-default-features --features tls-rustls: pure-Rust TLS (no system OpenSSL) +# keeps the archive self-contained and cross-compile-tractable. The kernel's +# default is tls-native, so the override is required. +# --locked: build against the kernel's committed Cargo.lock rather than +# re-resolving, so a fixed KERNEL_REV yields a fixed dependency graph (paired +# with the pinned rustc in rust-toolchain.toml, the .a is reproducible). Fails +# loud if the lock is stale instead of silently pulling newer deps. +log "cargo build --release --locked --no-default-features --features tls-rustls" +( cd "$KERNEL_SRC" && cargo build --release --locked --no-default-features --features tls-rustls ) + +src_a="$KERNEL_SRC/target/release/$LIB_NAME" +src_h="$KERNEL_SRC/include/$HEADER_NAME" +[ -f "$src_a" ] || { log "expected archive not produced: $src_a"; exit 1; } +[ -f "$src_h" ] || { log "expected header not found: $src_h"; exit 1; } + +cp "$src_a" "$KERNEL_LIB_DIR/$LIB_NAME" +cp "$src_h" "$KERNEL_INC_DIR/$HEADER_NAME" +# Record the rev this archive was built for so a later run can short-circuit +# (see the cache short-circuit above). Written last, only after a successful +# build, so a stamp never claims an artifact that isn't there. +printf '%s\n' "$KERNEL_REV" > "$REV_STAMP" +emit_checksum diff --git a/internal/backend/kernel/cgo.go b/internal/backend/kernel/cgo.go index 43a7d5af..852a467d 100644 --- a/internal/backend/kernel/cgo.go +++ b/internal/backend/kernel/cgo.go @@ -11,21 +11,23 @@ // the error mapping to the driver's error surface, and the gated step logger. // The backend, operation, and rows layers live in sibling files. // -// The directives below name the kernel library to link but carry no search -// paths, so the header and static lib locations are supplied at build time via -// the standard CGO_CFLAGS / CGO_LDFLAGS environment variables, e.g.: -// -// CGO_CFLAGS="-I/include" \ -// CGO_LDFLAGS="-L/target/release -Wl,-rpath,/target/release" \ -// go build -tags databricks_kernel ./... -// -// A shippable build instead links a committed per-platform prebuilt static lib -// via a ${SRCDIR}-relative path (the go-duckdb duckdb-go-bindings model); wiring -// that + a tagged CI job is a distribution follow-up. +// Link contract (${SRCDIR}-relative, machine-independent). The header is +// included from ${SRCDIR}/include and the static lib is linked from +// ${SRCDIR}/lib/_; the per-platform link flags live in the +// cgo_.go files beside this one. Both directories are produced by the +// build step (`make kernel-lib`), which checks out the kernel at the commit +// pinned in the repo-root KERNEL_REV file and `cargo build`s a static lib — +// so the kernel revision is a reviewable pin, never baked into a #cgo line +// (those expand only ${SRCDIR} and cannot run git or read env). The dirs are +// .gitignore'd; nothing kernel-built is committed. For local development +// against an existing checkout, `make kernel-lib KERNEL_LOCAL_A=.a +// KERNEL_LOCAL_HEADER=databricks_kernel.h` copies those in instead of +// building. The eventual release path downloads a published .a at the pinned +// rev rather than building it (see the driver's distribution design). package kernel /* -#cgo LDFLAGS: -ldatabricks_sql_kernel -ldl -lm +#cgo CFLAGS: -I${SRCDIR}/include #include #include "databricks_kernel.h" */ diff --git a/internal/backend/kernel/cgo_darwin.go b/internal/backend/kernel/cgo_darwin.go new file mode 100644 index 00000000..bc59d19e --- /dev/null +++ b/internal/backend/kernel/cgo_darwin.go @@ -0,0 +1,20 @@ +//go:build cgo && databricks_kernel && darwin && arm64 + +package kernel + +// Link flags for darwin/arm64. NOTE: this platform is not yet exercised in CI +// (M0 is linux/amd64); the flags below are the intended shape but must be +// validated on a mac before darwin is enabled. +// +// Two darwin-specific differences from linux: +// - Apple's ld64 does NOT accept the GNU `-l:.a` extension, so the +// archive is passed as a positional input by absolute ${SRCDIR} path +// instead. Since only the .a is placed under lib/darwin_arm64 (see +// kernel-lib.sh), there is no .so to accidentally prefer. +// - -lc++ (not -lstdc++) is the macOS C++ runtime; @loader_path keeps any +// dynamic reference resolvable relative to the built binary. + +/* +#cgo LDFLAGS: ${SRCDIR}/lib/darwin_arm64/libdatabricks_sql_kernel.a -lc++ -lm -Wl,-rpath,@loader_path +*/ +import "C" diff --git a/internal/backend/kernel/cgo_linux.go b/internal/backend/kernel/cgo_linux.go new file mode 100644 index 00000000..62274a34 --- /dev/null +++ b/internal/backend/kernel/cgo_linux.go @@ -0,0 +1,17 @@ +//go:build cgo && databricks_kernel && linux && amd64 + +package kernel + +// Link flags for linux/amd64. The static archive is forced with the +// -l:.a form (a GNU-ld extension) so the linker never prefers a +// same-named .so — the kernel's cargo build emits both a .a and a .so into the +// same dir, and a bare -ldatabricks_sql_kernel would pick the .so and bake in +// an rpath. -lstdc++/-lm/-ldl are the kernel's transitive system deps. +// +// The path is ${SRCDIR}-relative; `make kernel-lib` drops the archive at +// ${SRCDIR}/lib/linux_amd64/libdatabricks_sql_kernel.a. + +/* +#cgo LDFLAGS: -L${SRCDIR}/lib/linux_amd64 -l:libdatabricks_sql_kernel.a -lstdc++ -lm -ldl +*/ +import "C" diff --git a/internal/backend/kernel/cgo_unsupported.go b/internal/backend/kernel/cgo_unsupported.go new file mode 100644 index 00000000..bcc96910 --- /dev/null +++ b/internal/backend/kernel/cgo_unsupported.go @@ -0,0 +1,19 @@ +//go:build cgo && databricks_kernel && !(linux && amd64) && !(darwin && arm64) && !(windows && amd64) + +package kernel + +// This file is compiled only on GOOS/GOARCH combinations the kernel backend does +// not support. Per-platform link flags (cgo_.go) exist only for linux/amd64, +// darwin/arm64, and windows/amd64; on any other target there is no static archive +// to link, so cgo.go's C ABI calls would otherwise fail at the LINK step with an +// opaque "undefined reference to kernel_*". The Makefile's host==target guard +// does not catch this — it happily source-builds a host .a on e.g. linux/arm64 +// (Graviton) or an Intel Mac, only for the link to fall over with no matching +// LDFLAGS file. Referencing an undefined identifier here fails earlier, at +// COMPILE time, with a message that names the supported targets — a legible build +// error instead of a linker dump. +// +// Broader OS/arch coverage is tracked in the distribution design (native per-OS +// runners or a staged prebuilt .a); until then this guard makes the supported +// boundary explicit rather than latent. +const _ = kernel_backend_supports_only_linux_amd64_darwin_arm64_and_windows_amd64 diff --git a/internal/backend/kernel/cgo_windows.go b/internal/backend/kernel/cgo_windows.go new file mode 100644 index 00000000..ae650b9b --- /dev/null +++ b/internal/backend/kernel/cgo_windows.go @@ -0,0 +1,20 @@ +//go:build cgo && databricks_kernel && windows && amd64 + +package kernel + +// Link flags for windows/amd64. NOTE: this platform is not yet exercised in CI +// (M0 is linux/amd64); the flags below are the intended shape but must be +// validated on windows before it is enabled. +// +// cgo on windows uses the mingw/gcc toolchain, which links GNU archives (.a) — +// NOT the MSVC .lib that `cargo build --target x86_64-pc-windows-msvc` emits. +// So the kernel must be built for the windows-gnu target +// (`--target x86_64-pc-windows-gnu`) to produce a mingw-compatible .a; that +// target selection is the build step's responsibility. -lws2_32/-lwsock32 are +// the kernel's Winsock deps and -lrstrtmgr is Restart Manager (pulled in by the +// Rust std/dep graph on windows). + +/* +#cgo LDFLAGS: -L${SRCDIR}/lib/windows_amd64 -l:libdatabricks_sql_kernel.a -lws2_32 -lwsock32 -lrstrtmgr +*/ +import "C" diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000..f3f40f0c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,10 @@ +# Pins the Rust toolchain used to build the SEA kernel static lib, so the archive +# is reproducible: a floating `stable` drifts the .a (and its sha256) under a fixed +# KERNEL_REV as rustc/std advance. rustup honors this file for any cargo invocation +# in this dir or below — including the kernel checkout under build/kernel-src/ — so +# CI and local `make kernel-lib` build with the same compiler. Bump deliberately, +# in lockstep with the kernel's MSRV (kernel Cargo.toml edition 2021; no pin of its +# own). Only the opt-in `databricks_kernel` build reads this; the pure-Go default +# build has no Rust dependency. +[toolchain] +channel = "1.96.1"