diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c826f3d..2544671 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,6 +24,7 @@ permissions: jobs: build: uses: ./.github/workflows/matrix.yml + secrets: inherit with: spec: ${{ inputs.spec || 'lts' }} publish: ${{ inputs.publish == '' || inputs.publish }} diff --git a/.github/workflows/cache-refresh.yml b/.github/workflows/cache-refresh.yml deleted file mode 100644 index 40585f6..0000000 --- a/.github/workflows/cache-refresh.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Refresh ccache TTL -on: - # We use ccache with github save/restore to dramatically cut kernel build times. - # This works well, but GH has a 10GB limit for all cache entries, - # and a 7-day TTL for *each* cache entry. Which means that if we don't build a kernel for a week, - # we lose our cache benefit entirely, which stinks. The *correct* way to work around this is - # to replace GH's cache action with one that saves/restores directly from a dedicated S3 bucket - # we set up and manage. - # - # What *this* does is save/restore the cache every 4 days, well within the 7-day TTL, - # to keep GH from expiring them. Which is disgusting, but cheap. - schedule: - - cron: "0 0 */4 * *" - workflow_dispatch: -jobs: - discover: - name: discover cache keys - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.list.outputs.matrix }} - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 - with: - egress-policy: audit - - name: list ccache entries - id: list - env: - GH_TOKEN: ${{ github.token }} - run: | - # List main-scoped ccache-* cache keys, strip the run_id suffix to deduplicate - # by flavor/arch. We specifically skip non-main branches, becuse we only want to save/restore - # from main - in GH, PR branches can use `main` caches, but main cannot see PR branch caches, - # and saving duplicated PR branch caches counts against our 10GB github limit - matrix=$(gh api "/repos/${{ github.repository }}/actions/caches" --paginate \ - --jq '[.actions_caches[] - | select(.ref == "refs/heads/main" and (.key | startswith("ccache-"))) - | {prefix: (.key | gsub("-[0-9]+$"; ""))}] - | unique_by(.prefix) - | {entry: .}') - echo "matrix=$matrix" >> "$GITHUB_OUTPUT" - refresh: - name: "refresh ${{ matrix.entry.prefix }}" - needs: discover - if: needs.discover.outputs.matrix != '{"entry":[]}' - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.discover.outputs.matrix) }} - runs-on: ubuntu-latest - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 - with: - egress-policy: audit - - name: restore ccache - id: restore - uses: actions/cache/restore@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v5.0.2 - with: - path: ~/.cache/kernel-ccache - key: "${{ matrix.entry.prefix }}-${{ github.run_id }}" - restore-keys: | - ${{ matrix.entry.prefix }}- - - name: save ccache - if: steps.restore.outputs.cache-matched-key != '' - uses: actions/cache/save@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v5.0.2 - with: - path: ~/.cache/kernel-ccache - key: "${{ matrix.entry.prefix }}-${{ github.run_id }}" diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml index 34c18f5..4a4cce3 100644 --- a/.github/workflows/matrix.yml +++ b/.github/workflows/matrix.yml @@ -92,16 +92,39 @@ jobs: registry: ghcr.io username: "${{github.actor}}" password: "${{secrets.GITHUB_TOKEN}}" - - name: restore ccache - uses: actions/cache/restore@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v5.0.2 - with: - path: ~/.cache/kernel-ccache - # restore-keys is important here - it lets us restore the most recent cache key, - # *ignoring* the specific run ID, as a fuzzy match. So we can use previous build's - # caches for this flavor/arch even if the runid is not the same - key: "ccache-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}-${{ github.run_id }}" - restore-keys: | - ccache-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}- + - name: configure sccache + env: + SCCACHE_CONN_RW: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} + SCCACHE_CONN_RO: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING_RO }} + run: | + # Only explicitly trusted, maintainer-gated triggers get the + # read-write token. Everything else - pull requests today, and any + # trigger added later (e.g. merge_group) - fails closed to the + # read-only token, so code that hasn't been reviewed and merged can + # never poison cache objects later consumed by release builds. Fork + # PRs have no secrets at all and fall back to a runner-local disk + # cache. + case "${{ github.event_name }}" in + push|schedule|workflow_dispatch) + SCCACHE_CONN="${SCCACHE_CONN_RW}" + SCCACHE_MODE="READ_WRITE" + ;; + *) + SCCACHE_CONN="${SCCACHE_CONN_RO}" + SCCACHE_MODE="READ_ONLY" + ;; + esac + if [ -n "${SCCACHE_CONN}" ]; then + # These names must match the -e passthrough list in + # hack/build/generate-docker-script.py (docker_compile); a name + # missing there silently never reaches the build container. + { + echo "SCCACHE_AZURE_CONNECTION_STRING=${SCCACHE_CONN}" + echo "SCCACHE_AZURE_RW_MODE=${SCCACHE_MODE}" + echo "SCCACHE_AZURE_BLOB_CONTAINER=sccache" + echo "SCCACHE_AZURE_KEY_PREFIX=kernel" + } >> "${GITHUB_ENV}" + fi - name: generate docker script run: "./hack/build/generate-docker-script.sh" - name: upload docker script @@ -122,19 +145,6 @@ jobs: if-no-files-found: error compression-level: 0 retention-days: 1 - - name: save ccache - # Only save from main. GH's per-repo 10GB cache pool is shared across all refs, - # so PR-scoped saves evict main-scoped entries via LRU - and main is the only ref - # whose entries the next build can actually restore. PRs still benefit because - # they will fallthrough to/inherit parent branch caches - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v5.0.2 - with: - path: ~/.cache/kernel-ccache - # The run_id here is just for write-key uniqueness, as GH doesn't allow overwriting - # existing cache keys - the `restore` action will fuzzy-match and ignore the run_id - # for subsequent runs. - key: "ccache-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}-${{ github.run_id }}" merge: # Stitch the per-arch single-platform pushes from `build` into multi-arch # manifest lists. Only runs when publishing; no-op when nothing was pushed. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed17fa2..5dc1ba0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,6 +13,7 @@ concurrency: jobs: test: uses: ./.github/workflows/matrix.yml + secrets: inherit with: spec: "only-latest-lts:flavor=host,zone,zone-nvidiagpu" publish: false diff --git a/Dockerfile b/Dockerfile index aa7e01e..b104ed1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,20 +12,12 @@ FROM --platform=$BUILDPLATFORM scratch AS nvidia-modules ARG NV_MODULES_TARBALL_URL= ADD ${NV_MODULES_TARBALL_URL} /nvidia-modules.tar.gz -FROM --platform=$BUILDPLATFORM debian:bookworm@sha256:ed4fcc40bb1162b6d2d32e7bec15044d13963779abbe63f67f1cd62b06220519 AS buildenv -RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y \ - build-essential squashfs-tools python3-yaml \ - patch diffutils sed mawk findutils zstd \ - python3 python3-packaging curl rsync cpio gpg grep \ - flex bison pahole libssl-dev libelf-dev bc kmod ccache && \ - rm -rf /var/lib/apt/lists/* -ARG BUILDPLATFORM -RUN if [ "${BUILDPLATFORM}" = "linux/amd64" ]; then \ - apt-get update && apt-get install -y linux-headers-amd64 g++-aarch64-linux-gnu gcc-aarch64-linux-gnu && rm -rf /var/lib/apt/lists/*; fi -RUN if [ "${BUILDPLATFORM}" = "linux/arm64" ] || [ "${BUILDPLATFORM}" = "linux/aarch64" ]; then \ - apt-get update && apt-get install -y linux-headers-arm64 g++-x86-64-linux-gnu gcc-x86-64-linux-gnu && rm -rf /var/lib/apt/lists/*; fi -ENV PATH="/usr/lib/ccache:${PATH}" -RUN useradd -ms /bin/sh build +# The toolchain (compilers, kbuild deps, sccache + wrappers) comes from the +# published build environment image so it only changes via deliberate, +# reviewed digest bumps - see Dockerfile.buildenv and buildenv.yml. Dependabot +# keeps the pin current, with buildenv-diff.yml summarizing the package +# changes in each bump PR. +FROM --platform=$BUILDPLATFORM ghcr.io/edera-dev/kernel-buildenv:latest@sha256:c0bf191fcec390ef0b97522bcd35e6b776b8ec112676187e3208c933c28c8baa AS buildenv COPY --chown=build:build . /build USER build WORKDIR /build @@ -41,16 +33,16 @@ COPY --from=firmware --chown=build:build /firmware.tar.sign /build/override-firm FROM build-staged AS build-staged-nvidiagpu COPY --from=nvidia-modules --chown=build:build /nvidia-modules.tar.gz /build/override-nvidia-modules.tar.gz -FROM scratch AS kernel-ccachebuild -COPY --from=ccachebuild kernel /kernel/image -COPY --from=ccachebuild config.gz /kernel/config.gz -COPY --from=ccachebuild addons.squashfs /kernel/addons.squashfs -COPY --from=ccachebuild metadata /kernel/metadata +FROM scratch AS kernel-prebuilt +COPY --from=prebuilt kernel /kernel/image +COPY --from=prebuilt config.gz /kernel/config.gz +COPY --from=prebuilt addons.squashfs /kernel/addons.squashfs +COPY --from=prebuilt metadata /kernel/metadata -FROM alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS sdkbuild-ccachebuild +FROM alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS sdkbuild-prebuilt ARG KERNEL_FLAVOR=zone -COPY --from=ccachebuild sdk.tar.gz /sdk.tar.gz -COPY --from=ccachebuild metadata /metadata +COPY --from=prebuilt sdk.tar.gz /sdk.tar.gz +COPY --from=prebuilt metadata /metadata RUN KERNEL_UNAME_R=$(grep '^KERNEL_UNAME_R=' /metadata | cut -d= -f2) && \ mkdir -p /usr/src/kernel-sdk-${KERNEL_UNAME_R}-${KERNEL_FLAVOR} && \ tar -zx -C /usr/src/kernel-sdk-${KERNEL_UNAME_R}-${KERNEL_FLAVOR} -f /sdk.tar.gz && \ @@ -58,5 +50,5 @@ RUN KERNEL_UNAME_R=$(grep '^KERNEL_UNAME_R=' /metadata | cut -d= -f2) && \ ln -sf /usr/src/kernel-sdk-${KERNEL_UNAME_R}-${KERNEL_FLAVOR} /lib/modules/${KERNEL_UNAME_R}/build && \ rm -rf /sdk.tar.gz /metadata -FROM scratch AS sdk-ccachebuild -COPY --from=sdkbuild-ccachebuild /usr/src /usr/src +FROM scratch AS sdk-prebuilt +COPY --from=sdkbuild-prebuilt /usr/src /usr/src diff --git a/hack/build/docker-build-internal.sh b/hack/build/docker-build-internal.sh index c66458f..69e8768 100644 --- a/hack/build/docker-build-internal.sh +++ b/hack/build/docker-build-internal.sh @@ -10,3 +10,8 @@ if [ -z "${KERNEL_VERSION}" ] || [ -z "${KERNEL_FLAVOR}" ]; then fi ./hack/build/build.sh + +# Surface cache effectiveness (hits vs misses, backend errors) in build logs. +# || true: this may respawn an idle-timed-out server, and a transient backend +# error there must not fail an otherwise-successful build (set -e above). +sccache --show-stats || true diff --git a/hack/build/generate-docker-script.py b/hack/build/generate-docker-script.py index fed8687..3369832 100644 --- a/hack/build/generate-docker-script.py +++ b/hack/build/generate-docker-script.py @@ -9,10 +9,11 @@ from matrix import CONFIG from util import format_image_name, maybe, smart_script_split, parse_text_bool, get_branch_tag_suffix -# Targets that are handled via docker run + host CCACHE packaging stages. -CCACHE_TARGET_MAP = { - "kernel": "kernel-ccachebuild", - "sdk": "sdk-ccachebuild", +# Targets packaged from artifacts compiled out-of-band (the docker run compile +# phase) and imported into the image build through the `prebuilt` context. +PREBUILT_TARGET_MAP = { + "kernel": "kernel-prebuilt", + "sdk": "sdk-prebuilt", } # Targets skipped during the packaging phase (handled separately or not needed). @@ -110,7 +111,7 @@ def docker_compile( firmware_url: str, firmware_sig_url: str, ) -> list[str]: - """Generate docker run commands to compile the kernel with ccache.""" + """Generate docker run commands to compile the kernel with sccache.""" lines = [] has_firmware = flavor == "zone-amdgpu" has_nvidia = flavor == "zone-nvidiagpu" @@ -124,7 +125,7 @@ def docker_compile( staged_iidfile = "image-id-%s-%s-%s" % (version, flavor, stage_target) lines += ["", "rm -rf target && mkdir -p target && chmod a+rwX target"] - lines += ['mkdir -p "${HOME}/.cache/kernel-ccache" && chmod -R a+rwX "${HOME}/.cache/kernel-ccache"'] + lines += ['mkdir -p "${HOME}/.cache/kernel-sccache" && chmod -R a+rwX "${HOME}/.cache/kernel-sccache"'] for arch in archs: platform = arch_to_platform(arch) @@ -136,9 +137,19 @@ def docker_compile( "-e", quoted("KERNEL_VERSION=%s" % version), "-e", quoted("KERNEL_FLAVOR=%s" % flavor), "-e", quoted("KERNEL_SRC_URL=/build/override-kernel-src.tar.xz"), - "-e", quoted("CCACHE_DIR=/home/build/.cache/ccache"), - "-e", quoted("CCACHE_COMPRESS=1"), - "-v", quoted("${HOME}/.cache/kernel-ccache:/home/build/.cache/ccache"), + # The Azure sccache env is passed through by name only (no values), + # so the generated script stays free of secrets; docker omits any + # that are unset on the host. Without Azure config sccache falls + # back to the bind-mounted local disk cache below. This name list + # must match what the "configure sccache" step in + # .github/workflows/matrix.yml exports; a name missing here + # silently never reaches the build container. + "-e", quoted("SCCACHE_AZURE_CONNECTION_STRING"), + "-e", quoted("SCCACHE_AZURE_BLOB_CONTAINER"), + "-e", quoted("SCCACHE_AZURE_KEY_PREFIX"), + "-e", quoted("SCCACHE_AZURE_RW_MODE"), + "-e", quoted("SCCACHE_DIR=/home/build/.cache/sccache"), + "-v", quoted("${HOME}/.cache/kernel-sccache:/home/build/.cache/sccache"), "-v", quoted("${PWD}/target:/build/target"), ] if has_firmware: @@ -181,7 +192,7 @@ def docker_build( lines = [] version = dockerify_version(version) - actual_target = CCACHE_TARGET_MAP.get(target, target) + actual_target = PREBUILT_TARGET_MAP.get(target, target) # tag_version is the version string used for OCI tags; version is the # actual kernel version passed as a build arg and recorded in annotations. @@ -219,7 +230,7 @@ def docker_build( image_build_command += ["--platform", quoted(platform)] if actual_target != target: - image_build_command += ["--build-context", quoted("ccachebuild=target")] + image_build_command += ["--build-context", quoted("prebuilt=target")] if mark_format is not None: image_build_command += [ @@ -353,7 +364,8 @@ def generate_builds( firmware_sig_url=firmware_sig_url, ) - # Phase 2: Compile kernel via docker run with ccache bind-mounted from host. + # Phase 2: Compile kernel via docker run; sccache provides the compile + # cache (Azure Blob in CI, bind-mounted local disk otherwise). lines += docker_compile( version=kernel_version, flavor=kernel_flavor, @@ -362,7 +374,7 @@ def generate_builds( firmware_sig_url=firmware_sig_url, ) - # Phase 3: Package kernel and SDK images from the ccache-built artifacts. + # Phase 3: Package kernel and SDK images from the compiled artifacts. for image_config in image_configs: target = image_config["target"] if target in SKIP_PACKAGING_TARGETS: diff --git a/hack/build/matrix.py b/hack/build/matrix.py index 9909aae..ac9020c 100644 --- a/hack/build/matrix.py +++ b/hack/build/matrix.py @@ -1,5 +1,6 @@ import json import os +import re import yaml import subprocess @@ -9,7 +10,7 @@ from packaging.version import Version, parse -from util import matches_constraints, list_rsync_dir, format_image_name +from util import matches_constraints, list_remote_git_tags, format_image_name try: from yaml import CLoader as Loader @@ -48,44 +49,33 @@ def get_current_kernel_releases() -> dict[str, any]: @cache def get_all_kernel_releases() -> list[str]: + # Release tags (vX.Y[.Z]) map 1:1 to the published linux-X.Y[.Z].tar.xz + # artifacts; "-" excludes -rc and other pre-release tags. Tags only reach + # back to v2.6.11, but nothing anywhere near that old is buildable here. releases = [] - for maybe_release_major in list_rsync_dir( - "rsync://rsync.kernel.org/pub/linux/kernel/" + for tag in list_remote_git_tags( + "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git" ): - if not ( - maybe_release_major.startswith("v") and maybe_release_major.endswith("x") - ): + if not tag.startswith("v"): continue - for maybe_release_file in list_rsync_dir( - "rsync://rsync.kernel.org/pub/linux/kernel/%s/" % maybe_release_major - ): - if not ( - maybe_release_file.startswith("linux-") - and maybe_release_file.endswith(".tar.xz") - ): - continue - kernel_version = maybe_release_file.replace("linux-", "").replace( - ".tar.xz", "" - ) - if "-" in kernel_version: - continue - releases.append(kernel_version) + kernel_version = tag[1:] + if "-" in kernel_version: + continue + releases.append(kernel_version) return releases @cache def get_all_firmware_releases() -> list[str]: + # Snapshot tags are pure YYYYMMDD and map 1:1 to the published + # linux-firmware-YYYYMMDD.tar.xz artifacts, so lexicographic sort is + # chronological. snapshots = [] - for maybe_release_snapshot in list_rsync_dir( - "rsync://rsync.kernel.org/pub/linux/kernel/firmware/" + for tag in list_remote_git_tags( + "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git" ): - if not ( - maybe_release_snapshot.startswith("linux-firmware-") and maybe_release_snapshot.endswith(".xz") - ): + if not re.fullmatch(r"[0-9]{8}", tag): continue - firmware_snapshot_version = maybe_release_snapshot.replace("linux-firmware-", "").replace( - ".tar.xz", "" - ) - snapshots.append(firmware_snapshot_version) + snapshots.append(tag) snapshots.sort() snapshots.reverse() return snapshots diff --git a/hack/build/util.py b/hack/build/util.py index 15aff86..d09f574 100644 --- a/hack/build/util.py +++ b/hack/build/util.py @@ -1,5 +1,8 @@ import os +import random import re +import sys +import time from typing import Optional from packaging.version import Version @@ -114,24 +117,48 @@ def matches_constraints( return applies -def list_rsync_dir(url: str): - result = subprocess.run( - ["rsync", "--list-only", "--out-format='%n'", url], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - ) +def list_remote_git_tags(url: str, attempts: int = 6) -> list[str]: + # ls-remote fetches only the tag advertisement (protocol v2 filters it + # server-side), so this avoids both a clone and rsync.kernel.org's + # aggressive concurrent-connection cap. Retries cover transient network + # failures, with stderr surfaced so the failure mode shows up in CI logs. + for attempt in range(attempts): + try: + result = subprocess.run( + ["git", "ls-remote", "--tags", "--refs", url], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + # git has no network timeout of its own, so without this a + # hung connection would stall matrix generation until the CI + # job limit instead of falling through to the retry loop. + timeout=120, + ) + except subprocess.TimeoutExpired: + sys.stderr.write( + "listing tags of %s timed out (attempt %d/%d)\n" + % (url, attempt + 1, attempts) + ) + if attempt + 1 < attempts: + time.sleep(min(120, 10 * 2**attempt) + random.uniform(0, 5)) + continue + raise + if result.returncode == 0: + break + sys.stderr.write( + "listing tags of %s failed (attempt %d/%d):\n%s\n" + % (url, attempt + 1, attempts, result.stderr.decode("utf-8", "replace")) + ) + if attempt + 1 < attempts: + time.sleep(min(120, 10 * 2**attempt) + random.uniform(0, 5)) result.check_returncode() - files = [] + tags = [] for line in result.stdout.splitlines(keepends=False): - line = line.decode("utf-8") - if len(line.strip()) == 0: - continue - if line.startswith("MOTD:"): + # "\trefs/tags/" + parts = line.decode("utf-8").strip().split("\t") + if len(parts) != 2 or not parts[1].startswith("refs/tags/"): continue - file_name = str(line.split(" ")[-1]) - if file_name != ".": - files.append(file_name) - return files + tags.append(parts[1][len("refs/tags/"):]) + return tags def parse_text_bool(text: str) -> bool: