Move protobuf and flatbuffer definitions into their owning crates

.cargo/config.toml

@@ -4,7 +4,6 @@ rustflags = [
 ]
 
 [alias]
-xtask = "run -p xtask --"
 vx = "run -p vortex-tui --"
 
 [publish]

.github/actions/setup-flatc/action.yml

@@ -1,5 +1,5 @@
 name: "Setup flatc"
-description: "Download and install flatc binary"
+description: "Download and install the flatc binary directly from GitHub releases CDN (no API calls)"
 inputs:
   flatc_version:
     description: "Version of the flatc binary"
@@ -8,9 +8,46 @@ runs:
   using: "composite"
   steps:
     - name: Download flatc
-      id: download-flatc
       shell: bash
       run: |
-        wget -O /tmp/flatc.zip "https://github.com/google/flatbuffers/releases/download/v${{ inputs.flatc_version }}/Linux.flatc.binary.clang++-18.zip"
-        unzip /tmp/flatc.zip flatc
-        mv flatc /usr/local/bin/
+        if [ "$(flatc --version 2>/dev/null)" = "flatc version ${{ inputs.flatc_version }}" ]; then
+          echo "flatc ${{ inputs.flatc_version }} is already installed"
+          exit 0
+        fi
+
+        ARCH=$(uname -m)
+        OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+
+        case "$OS" in
+          linux)  FLATC_ASSET="Linux.flatc.binary.clang++-18.zip" ;;
+          darwin)
+            case "$ARCH" in
+              arm64)  FLATC_ASSET="Mac.flatc.binary.zip" ;;
+              x86_64) FLATC_ASSET="MacIntel.flatc.binary.zip" ;;
+            esac
+            ;;
+          *)      FLATC_ASSET="Windows.flatc.binary.zip" ;;
+        esac
+
+        FLATC_URL="https://github.com/google/flatbuffers/releases/download/v${{ inputs.flatc_version }}/${FLATC_ASSET}"
+        curl -fsSL -o /tmp/flatc.zip "$FLATC_URL"
+        unzip -o /tmp/flatc.zip -d /tmp/flatc
+
+        # This sometimes runs in a container where user is root and there's no sudo.
+        if command -v sudo &>/dev/null
+        then
+          CMD=sudo
+        else
+          CMD=
+        fi
+
+        if [ "$OS" = "linux" ] || [ "$OS" = "darwin" ]; then
+          $CMD mv /tmp/flatc/flatc /usr/local/bin/
+        else
+          # Windows runners have no /usr/local/bin on PATH for non-bash steps.
+          mkdir -p "$HOME/.local/bin"
+          mv /tmp/flatc/flatc.exe "$HOME/.local/bin/"
+          cygpath -w "$HOME/.local/bin" >> "$GITHUB_PATH"
+        fi
+
+        rm -rf /tmp/flatc.zip /tmp/flatc

.github/actions/setup-prebuild/action.yml

@@ -47,3 +47,7 @@ runs:
         components: ${{ inputs.components }}
         targets: ${{ inputs.targets }}
         enable-sccache: "false"
+
+    # Pins the version; the prebuild AMIs bake in a flatc that may lag it.
+    - name: Install flatc (for FlatBuffers code generation)
+      uses: ./.github/actions/setup-flatc

.github/actions/setup-rust/action.yml

@@ -83,3 +83,6 @@ runs:
     - name: Install Protoc (for lance-encoding build step)
       if: runner.os != 'Windows'
       uses: ./.github/actions/setup-protoc
+
+    - name: Install flatc (for FlatBuffers code generation)
+      uses: ./.github/actions/setup-flatc

.github/workflows/ci.yml

@@ -220,7 +220,7 @@ jobs:
           # including all private docs.
           cargo doc --profile ci --no-deps --document-private-items --workspace --exclude vortex-python --exclude vortex-python-cuda
           # nextest doesn't support doc tests, so we run it here
-          cargo test --profile ci --doc --workspace --all-features --exclude vortex-jni --exclude vortex-ffi --exclude xtask --exclude vortex-python-cuda --no-fail-fast
+          cargo test --profile ci --doc --workspace --all-features --exclude vortex-jni --exclude vortex-ffi --exclude vortex-python-cuda --no-fail-fast
 
   build-rust:
     name: "Rust build (${{matrix.config.name}})"
@@ -442,7 +442,7 @@ jobs:
             --exclude vortex-nvcomp --exclude vortex-cub --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda `
             --exclude duckdb-bench `
             --exclude lance-bench --exclude datafusion-bench --exclude random-access-bench `
-            --exclude compress-bench --exclude xtask --exclude vortex-datafusion `
+            --exclude compress-bench --exclude vortex-datafusion `
             --exclude gpu-scan-cli --exclude vortex-sqllogictest
 
       - name: sccache stats
@@ -473,10 +473,10 @@ jobs:
           enable-sccache: "true"
       - name: Rust Tests
         run: |
-          cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest
+          cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude vortex-sqllogictest
       - uses: ./.github/actions/check-rebuild
         with:
-          command: "cargo test --profile ci --locked --workspace --all-features --no-run --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest"
+          command: "cargo test --profile ci --locked --workspace --all-features --no-run --exclude vortex-bench --exclude vortex-sqllogictest"
 
       - name: Alert incident.io
         if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'
@@ -675,12 +675,9 @@ jobs:
       - uses: ./.github/actions/setup-prebuild
         with:
           enable-sccache: "true"
+      - uses: ./.github/actions/setup-flatc
       - name: Install nightly for cbindgen macro expansion
         run: rustup toolchain install $NIGHTLY_TOOLCHAIN
-      - name: "regenerate all .fbs/.proto Rust code"
-        run: |
-          cargo run --profile ci -p xtask -- generate-fbs
-          cargo run --profile ci -p xtask -- generate-proto
       - name: "regenerate FFI header file"
         run: |
           cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi
@@ -689,16 +686,27 @@ jobs:
           git status --porcelain
           test -z "$(git status --porcelain)"
 
-      - name: "Checkout develop flatbuffers"
-        working-directory: vortex-flatbuffers/
+      # Schemas live under `<crate>/flatbuffers/` but resolve includes against those dirs
+      # collectively, so flatten each revision into one tree for flatc to compare.
+      - name: "Collect flatbuffer schemas from this revision and from develop"
         run: |
-          cp -R flatbuffers flatbuffers.HEAD
           git fetch origin develop --depth 1
-          git checkout origin/develop -- flatbuffers
+          collect() {
+            git ls-tree -r --name-only "$1" \
+              | grep -E '(^|/)flatbuffers/.*\.fbs$' \
+              | while read -r path; do
+                  rel="${path#*/flatbuffers/}"
+                  mkdir -p "$2/$(dirname "$rel")"
+                  git show "$1:$path" > "$2/$rel"
+                done
+          }
+          collect HEAD "$RUNNER_TEMP/fbs.head"
+          collect origin/develop "$RUNNER_TEMP/fbs.develop"
       - name: "Verify flatbuffer back-compat"
-        working-directory: vortex-flatbuffers/
+        working-directory: ${{ runner.temp }}
         run: |
-          find flatbuffers/ -type f -name "*.fbs" | sed 's/^flatbuffers\///' | xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{}
+          find fbs.develop/ -type f -name "*.fbs" | sed 's|^fbs.develop/||' \
+            | xargs -I{} -n1 flatc -I fbs.head --conform-includes fbs.develop --conform fbs.develop/{} fbs.head/{}
 
   ffi-c-test:
     name: "C API test build"

.github/workflows/codspeed.yml

@@ -56,7 +56,7 @@ jobs:
           - { shard: 5, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" }
           - { shard: 6, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" }
           - { shard: 7, name: "Encodings 4 & layout", packages: "vortex-sparse vortex-zigzag vortex-zstd vortex-layout" }
-          - { shard: 8, name: "Storage formats & row encoding", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks vortex-row" }
+          - { shard: 8, name: "Storage formats & row encoding", packages: "vortex-btrblocks vortex-row" }
           - { shard: 9, name: "Tensor & spatial", packages: "vortex-tensor vortex-spatial" }
     name: "Benchmark with Codspeed (Shard #${{ matrix.shard }})"
     timeout-minutes: 30

.github/workflows/musl.yml

@@ -50,7 +50,7 @@ jobs:
         run: |
           apk add --no-cache build-base clang clang-dev llvm-dev cmake make perl \
             pkgconf protobuf protobuf-dev openssl-dev zstd-dev git curl ca-certificates \
-            python3 python3-dev tar tzdata bash
+            python3 python3-dev tar tzdata bash flatbuffers
 
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
 

.github/workflows/rust-instrumented.yml

@@ -100,9 +100,9 @@ jobs:
             --llvm-path "${LLVM_TOOLS_BIN}" \
             --threads $(nproc) \
             --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \
-            --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \
-            --ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \
-            --ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \
+            --ignore 'home/*' --ignore 'vortex-build/*' --ignore 'target/*' --ignore 'vortex-error/*' \
+            --ignore 'vortex-python/*' --ignore 'vortex-jni/*' \
+            --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \
             --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' \
             --ignore benchmarks/* --ignore 'vortex-test/*' \
             -o ${{ env.GRCOV_OUTPUT_FILE }}

AGENTS.md

@@ -30,6 +30,11 @@ documentation in `docs/`, and benchmark tooling in `vortex-bench/` and `benchmar
   and the OpenDAL-backed services (`cos://`, `oss://`). Every binding resolves URLs through it.
 - `vortex-scan`, `vortex-session`, `vortex-datafusion`, and `vortex-duckdb` contain scan
   and execution integrations.
+- FlatBuffers (`.fbs`) and Protocol Buffers (`.proto`) schemas live in the crate that owns the
+  types they describe (`vortex-array`, `vortex-layout`, `vortex-file`, `vortex-ipc`), and are
+  compiled into `OUT_DIR` by that crate's `build.rs` via `vortex-build`. Generated code is never
+  checked in, and a schema that includes another crate's declares that crate with `depends_on`.
+  Building therefore requires `flatc` on `PATH` (or `FLATC` set); `protoc` is not needed.
 - `vortex-python` contains Python bindings. RST-flavored project docs live in `docs/`.
 
 ## Scoped Guidance

CONTRIBUTING.md

@@ -36,6 +36,18 @@ The contribution process is outlined below:
 
 ## Development Workflows
 
+### Build prerequisites
+
+Bindings for the `.fbs` and `.proto` schemas are generated at build time into `OUT_DIR` by the
+`build.rs` of the crate that owns each schema, and are never checked in.
+
+FlatBuffers generation shells out to the [`flatc`](https://github.com/google/flatbuffers/releases)
+compiler, so building any Vortex crate requires it on `PATH`, or its location in the `FLATC`
+environment variable. CI pins version `25.12.19`; other recent versions work, but may produce
+cosmetically different generated code.
+
+Protocol Buffers generation parses schemas in pure Rust, so `protoc` is not required.
+
 The repository uses [`uv`](https://docs.astral.sh/uv/) to manage its Python workspace. From the
 repository root, create or update the development environment with: