build: replace ccache + GHA cache with sccache backed by Azure Blob Storage

.github/workflows/build.yml

@@ -24,6 +24,7 @@ permissions:
 jobs:
   build:
     uses: ./.github/workflows/matrix.yml
+    secrets: inherit
     with:
       spec: ${{ inputs.spec || 'lts' }}
       publish: ${{ inputs.publish == '' || inputs.publish }}

.github/workflows/matrix.yml

@@ -92,16 +92,39 @@ jobs:
             registry: ghcr.io
             username: "${{github.actor}}"
             password: "${{secrets.GITHUB_TOKEN}}"
-      - name: restore ccache
-        uses: actions/cache/restore@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v5.0.2
-        with:
-          path: ~/.cache/kernel-ccache
-          # restore-keys is important here - it lets us restore the most recent cache key,
-          # *ignoring* the specific run ID, as a fuzzy match. So we can use previous build's
-          # caches for this flavor/arch even if the runid is not the same
-          key: "ccache-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}-${{ github.run_id }}"
-          restore-keys: |
-            ccache-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}-
+      - name: configure sccache
+        env:
+          SCCACHE_CONN_RW: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }}
+          SCCACHE_CONN_RO: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING_RO }}
+        run: |
+          # Only explicitly trusted, maintainer-gated triggers get the
+          # read-write token. Everything else - pull requests today, and any
+          # trigger added later (e.g. merge_group) - fails closed to the
+          # read-only token, so code that hasn't been reviewed and merged can
+          # never poison cache objects later consumed by release builds. Fork
+          # PRs have no secrets at all and fall back to a runner-local disk
+          # cache.
+          case "${{ github.event_name }}" in
+            push|schedule|workflow_dispatch)
+              SCCACHE_CONN="${SCCACHE_CONN_RW}"
+              SCCACHE_MODE="READ_WRITE"
+              ;;
+            *)
+              SCCACHE_CONN="${SCCACHE_CONN_RO}"
+              SCCACHE_MODE="READ_ONLY"
+              ;;
+          esac
+          if [ -n "${SCCACHE_CONN}" ]; then
+            # These names must match the -e passthrough list in
+            # hack/build/generate-docker-script.py (docker_compile); a name
+            # missing there silently never reaches the build container.
+            {
+              echo "SCCACHE_AZURE_CONNECTION_STRING=${SCCACHE_CONN}"
+              echo "SCCACHE_AZURE_RW_MODE=${SCCACHE_MODE}"
+              echo "SCCACHE_AZURE_BLOB_CONTAINER=sccache"
+              echo "SCCACHE_AZURE_KEY_PREFIX=kernel"
+            } >> "${GITHUB_ENV}"
+          fi
       - name: generate docker script
         run: "./hack/build/generate-docker-script.sh"
       - name: upload docker script
@@ -122,19 +145,6 @@ jobs:
           if-no-files-found: error
           compression-level: 0
           retention-days: 1
-      - name: save ccache
-        # Only save from main. GH's per-repo 10GB cache pool is shared across all refs,
-        # so PR-scoped saves evict main-scoped entries via LRU - and main is the only ref
-        # whose entries the next build can actually restore. PRs still benefit because
-        # they will fallthrough to/inherit parent branch caches
-        if: github.ref == 'refs/heads/main'
-        uses: actions/cache/save@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v5.0.2
-        with:
-          path: ~/.cache/kernel-ccache
-          # The run_id here is just for write-key uniqueness, as GH doesn't allow overwriting
-          # existing cache keys - the `restore` action will fuzzy-match and ignore the run_id
-          # for subsequent runs.
-          key: "ccache-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}-${{ github.run_id }}"
   merge:
     # Stitch the per-arch single-platform pushes from `build` into multi-arch
     # manifest lists. Only runs when publishing; no-op when nothing was pushed.

.github/workflows/test.yml

@@ -13,6 +13,7 @@ concurrency:
 jobs:
   test:
     uses: ./.github/workflows/matrix.yml
+    secrets: inherit
     with:
       spec: "only-latest-lts:flavor=host,zone,zone-nvidiagpu"
       publish: false

Dockerfile

@@ -12,20 +12,12 @@ FROM --platform=$BUILDPLATFORM scratch AS nvidia-modules
 ARG NV_MODULES_TARBALL_URL=
 ADD ${NV_MODULES_TARBALL_URL} /nvidia-modules.tar.gz
 
-FROM --platform=$BUILDPLATFORM debian:bookworm@sha256:ed4fcc40bb1162b6d2d32e7bec15044d13963779abbe63f67f1cd62b06220519 AS buildenv
-RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y \
-      build-essential squashfs-tools python3-yaml \
-      patch diffutils sed mawk findutils zstd \
-      python3 python3-packaging curl rsync cpio gpg grep \
-      flex bison pahole libssl-dev libelf-dev bc kmod ccache && \
-      rm -rf /var/lib/apt/lists/*
-ARG BUILDPLATFORM
-RUN if [ "${BUILDPLATFORM}" = "linux/amd64" ]; then \
-      apt-get update && apt-get install -y linux-headers-amd64 g++-aarch64-linux-gnu gcc-aarch64-linux-gnu && rm -rf /var/lib/apt/lists/*; fi
-RUN if [ "${BUILDPLATFORM}" = "linux/arm64" ] || [ "${BUILDPLATFORM}" = "linux/aarch64" ]; then \
-      apt-get update && apt-get install -y linux-headers-arm64 g++-x86-64-linux-gnu gcc-x86-64-linux-gnu && rm -rf /var/lib/apt/lists/*; fi
-ENV PATH="/usr/lib/ccache:${PATH}"
-RUN useradd -ms /bin/sh build
+# The toolchain (compilers, kbuild deps, sccache + wrappers) comes from the
+# published build environment image so it only changes via deliberate,
+# reviewed digest bumps - see Dockerfile.buildenv and buildenv.yml. Dependabot
+# keeps the pin current, with buildenv-diff.yml summarizing the package
+# changes in each bump PR.
+FROM --platform=$BUILDPLATFORM ghcr.io/edera-dev/kernel-buildenv:latest@sha256:c0bf191fcec390ef0b97522bcd35e6b776b8ec112676187e3208c933c28c8baa AS buildenv
 COPY --chown=build:build . /build
 USER build
 WORKDIR /build
@@ -41,22 +33,22 @@ COPY --from=firmware --chown=build:build /firmware.tar.sign /build/override-firm
 FROM build-staged AS build-staged-nvidiagpu
 COPY --from=nvidia-modules --chown=build:build /nvidia-modules.tar.gz /build/override-nvidia-modules.tar.gz
 
-FROM scratch AS kernel-ccachebuild
-COPY --from=ccachebuild kernel /kernel/image
-COPY --from=ccachebuild config.gz /kernel/config.gz
-COPY --from=ccachebuild addons.squashfs /kernel/addons.squashfs
-COPY --from=ccachebuild metadata /kernel/metadata
+FROM scratch AS kernel-prebuilt
+COPY --from=prebuilt kernel /kernel/image
+COPY --from=prebuilt config.gz /kernel/config.gz
+COPY --from=prebuilt addons.squashfs /kernel/addons.squashfs
+COPY --from=prebuilt metadata /kernel/metadata
 
-FROM alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS sdkbuild-ccachebuild
+FROM alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS sdkbuild-prebuilt
 ARG KERNEL_FLAVOR=zone
-COPY --from=ccachebuild sdk.tar.gz /sdk.tar.gz
-COPY --from=ccachebuild metadata /metadata
+COPY --from=prebuilt sdk.tar.gz /sdk.tar.gz
+COPY --from=prebuilt metadata /metadata
 RUN KERNEL_UNAME_R=$(grep '^KERNEL_UNAME_R=' /metadata | cut -d= -f2) && \
     mkdir -p /usr/src/kernel-sdk-${KERNEL_UNAME_R}-${KERNEL_FLAVOR} && \
     tar -zx -C /usr/src/kernel-sdk-${KERNEL_UNAME_R}-${KERNEL_FLAVOR} -f /sdk.tar.gz && \
     mkdir -p /lib/modules/${KERNEL_UNAME_R} && \
     ln -sf /usr/src/kernel-sdk-${KERNEL_UNAME_R}-${KERNEL_FLAVOR} /lib/modules/${KERNEL_UNAME_R}/build && \
     rm -rf /sdk.tar.gz /metadata
 
-FROM scratch AS sdk-ccachebuild
-COPY --from=sdkbuild-ccachebuild /usr/src /usr/src
+FROM scratch AS sdk-prebuilt
+COPY --from=sdkbuild-prebuilt /usr/src /usr/src

hack/build/docker-build-internal.sh

@@ -10,3 +10,8 @@ if [ -z "${KERNEL_VERSION}" ] || [ -z "${KERNEL_FLAVOR}" ]; then
 fi
 
 ./hack/build/build.sh
+
+# Surface cache effectiveness (hits vs misses, backend errors) in build logs.
+# || true: this may respawn an idle-timed-out server, and a transient backend
+# error there must not fail an otherwise-successful build (set -e above).
+sccache --show-stats || true

hack/build/generate-docker-script.py

@@ -9,10 +9,11 @@
 from matrix import CONFIG
 from util import format_image_name, maybe, smart_script_split, parse_text_bool, get_branch_tag_suffix
 
-# Targets that are handled via docker run + host CCACHE packaging stages.
-CCACHE_TARGET_MAP = {
-    "kernel": "kernel-ccachebuild",
-    "sdk": "sdk-ccachebuild",
+# Targets packaged from artifacts compiled out-of-band (the docker run compile
+# phase) and imported into the image build through the `prebuilt` context.
+PREBUILT_TARGET_MAP = {
+    "kernel": "kernel-prebuilt",
+    "sdk": "sdk-prebuilt",
 }
 
 # Targets skipped during the packaging phase (handled separately or not needed).
@@ -110,7 +111,7 @@ def docker_compile(
     firmware_url: str,
     firmware_sig_url: str,
 ) -> list[str]:
-    """Generate docker run commands to compile the kernel with ccache."""
+    """Generate docker run commands to compile the kernel with sccache."""
     lines = []
     has_firmware = flavor == "zone-amdgpu"
     has_nvidia = flavor == "zone-nvidiagpu"
@@ -124,7 +125,7 @@ def docker_compile(
     staged_iidfile = "image-id-%s-%s-%s" % (version, flavor, stage_target)
 
     lines += ["", "rm -rf target && mkdir -p target && chmod a+rwX target"]
-    lines += ['mkdir -p "${HOME}/.cache/kernel-ccache" && chmod -R a+rwX "${HOME}/.cache/kernel-ccache"']
+    lines += ['mkdir -p "${HOME}/.cache/kernel-sccache" && chmod -R a+rwX "${HOME}/.cache/kernel-sccache"']
 
     for arch in archs:
         platform = arch_to_platform(arch)
@@ -136,9 +137,19 @@ def docker_compile(
             "-e", quoted("KERNEL_VERSION=%s" % version),
             "-e", quoted("KERNEL_FLAVOR=%s" % flavor),
             "-e", quoted("KERNEL_SRC_URL=/build/override-kernel-src.tar.xz"),
-            "-e", quoted("CCACHE_DIR=/home/build/.cache/ccache"),
-            "-e", quoted("CCACHE_COMPRESS=1"),
-            "-v", quoted("${HOME}/.cache/kernel-ccache:/home/build/.cache/ccache"),
+            # The Azure sccache env is passed through by name only (no values),
+            # so the generated script stays free of secrets; docker omits any
+            # that are unset on the host. Without Azure config sccache falls
+            # back to the bind-mounted local disk cache below. This name list
+            # must match what the "configure sccache" step in
+            # .github/workflows/matrix.yml exports; a name missing here
+            # silently never reaches the build container.
+            "-e", quoted("SCCACHE_AZURE_CONNECTION_STRING"),
+            "-e", quoted("SCCACHE_AZURE_BLOB_CONTAINER"),
+            "-e", quoted("SCCACHE_AZURE_KEY_PREFIX"),
+            "-e", quoted("SCCACHE_AZURE_RW_MODE"),
+            "-e", quoted("SCCACHE_DIR=/home/build/.cache/sccache"),
+            "-v", quoted("${HOME}/.cache/kernel-sccache:/home/build/.cache/sccache"),
             "-v", quoted("${PWD}/target:/build/target"),
         ]
         if has_firmware:
@@ -181,7 +192,7 @@ def docker_build(
     lines = []
 
     version = dockerify_version(version)
-    actual_target = CCACHE_TARGET_MAP.get(target, target)
+    actual_target = PREBUILT_TARGET_MAP.get(target, target)
 
     # tag_version is the version string used for OCI tags; version is the
     # actual kernel version passed as a build arg and recorded in annotations.
@@ -219,7 +230,7 @@ def docker_build(
         image_build_command += ["--platform", quoted(platform)]
 
     if actual_target != target:
-        image_build_command += ["--build-context", quoted("ccachebuild=target")]
+        image_build_command += ["--build-context", quoted("prebuilt=target")]
 
     if mark_format is not None:
         image_build_command += [
@@ -353,7 +364,8 @@ def generate_builds(
         firmware_sig_url=firmware_sig_url,
     )
 
-    # Phase 2: Compile kernel via docker run with ccache bind-mounted from host.
+    # Phase 2: Compile kernel via docker run; sccache provides the compile
+    # cache (Azure Blob in CI, bind-mounted local disk otherwise).
     lines += docker_compile(
         version=kernel_version,
         flavor=kernel_flavor,
@@ -362,7 +374,7 @@ def generate_builds(
         firmware_sig_url=firmware_sig_url,
     )
 
-    # Phase 3: Package kernel and SDK images from the ccache-built artifacts.
+    # Phase 3: Package kernel and SDK images from the compiled artifacts.
     for image_config in image_configs:
         target = image_config["target"]
         if target in SKIP_PACKAGING_TARGETS:

hack/build/matrix.py

@@ -1,5 +1,6 @@
 import json
 import os
+import re
 
 import yaml
 import subprocess
@@ -9,7 +10,7 @@
 
 from packaging.version import Version, parse
 
-from util import matches_constraints, list_rsync_dir, format_image_name
+from util import matches_constraints, list_remote_git_tags, format_image_name
 
 try:
     from yaml import CLoader as Loader
@@ -48,44 +49,33 @@ def get_current_kernel_releases() -> dict[str, any]:
 
 @cache
 def get_all_kernel_releases() -> list[str]:
+    # Release tags (vX.Y[.Z]) map 1:1 to the published linux-X.Y[.Z].tar.xz
+    # artifacts; "-" excludes -rc and other pre-release tags. Tags only reach
+    # back to v2.6.11, but nothing anywhere near that old is buildable here.
     releases = []
-    for maybe_release_major in list_rsync_dir(
-        "rsync://rsync.kernel.org/pub/linux/kernel/"
+    for tag in list_remote_git_tags(
+        "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git"
     ):
-        if not (
-            maybe_release_major.startswith("v") and maybe_release_major.endswith("x")
-        ):
+        if not tag.startswith("v"):
             continue
-        for maybe_release_file in list_rsync_dir(
-            "rsync://rsync.kernel.org/pub/linux/kernel/%s/" % maybe_release_major
-        ):
-            if not (
-                maybe_release_file.startswith("linux-")
-                and maybe_release_file.endswith(".tar.xz")
-            ):
-                continue
-            kernel_version = maybe_release_file.replace("linux-", "").replace(
-                ".tar.xz", ""
-            )
-            if "-" in kernel_version:
-                continue
-            releases.append(kernel_version)
+        kernel_version = tag[1:]
+        if "-" in kernel_version:
+            continue
+        releases.append(kernel_version)
     return releases
 
 @cache
 def get_all_firmware_releases() -> list[str]:
+    # Snapshot tags are pure YYYYMMDD and map 1:1 to the published
+    # linux-firmware-YYYYMMDD.tar.xz artifacts, so lexicographic sort is
+    # chronological.
     snapshots = []
-    for maybe_release_snapshot in list_rsync_dir(
-        "rsync://rsync.kernel.org/pub/linux/kernel/firmware/"
+    for tag in list_remote_git_tags(
+        "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git"
     ):
-        if not (
-            maybe_release_snapshot.startswith("linux-firmware-") and maybe_release_snapshot.endswith(".xz")
-        ):
+        if not re.fullmatch(r"[0-9]{8}", tag):
             continue
-        firmware_snapshot_version = maybe_release_snapshot.replace("linux-firmware-", "").replace(
-            ".tar.xz", ""
-        )
-        snapshots.append(firmware_snapshot_version)
+        snapshots.append(tag)
     snapshots.sort()
     snapshots.reverse()
     return snapshots