From d5e8d17f46a809347fedb5bacc63c893c4863f40 Mon Sep 17 00:00:00 2001 From: Steven Noonan Date: Tue, 14 Jul 2026 11:50:10 -0700 Subject: [PATCH 1/2] build: replace ccache + GHA cache with sccache backed by Azure Blob Storage The previous compile cache serialized the whole ccache directory into a single GitHub Actions cache blob per flavor/arch. Those blobs shared the repo-wide 10GB cache budget and were routinely evicted before any build could benefit from them, and the 7-day per-entry TTL needed a scheduled workflow (cache-refresh.yml) whose only job was to keep entries alive. Switch the compile phase to sccache with Azure Blob Storage as a shared backend (container `sccache` on the `ederasccache` storage account). Cache objects are stored and fetched individually, so there is no more monolithic save/restore step, no size cap to prune against, and all builders across all flavors, arches, and runs read and write the same namespace concurrently. Garbage collection is handled service-side by an Azure lifecycle rule that deletes objects unaccessed for 30 days, which lets cache-refresh.yml be deleted outright. Details: - The build environment now comes from the published kernel-buildenv image, pinned by digest (see Dockerfile.buildenv and buildenv.yml). That image carries the pinned, checksum-verified sccache release and the /usr/lib/sccache compiler wrapper scripts, so the Dockerfile's inline toolchain construction goes away entirely and the compilers can only change through a reviewed digest bump - which matters now that cache keys include the compiler binary's hash. - The generated docker.sh passes the SCCACHE_AZURE_* environment through by name only, so the script (which is uploaded as a workflow artifact) never contains credentials. - CI selects credentials from an explicit allowlist of maintainer-gated triggers: push, schedule, and workflow_dispatch get the read-write connection string, and everything else - pull requests today, any trigger added in the future - fails closed to a read-only token, so unreviewed code can never poison cache objects later consumed by release builds. Fork PRs (no secrets) fall back to a bind-mounted local disk cache, the same fallback that keeps local developer builds cached across runs. - docker-build-internal.sh prints `sccache --show-stats` after each compile so hit rates and backend errors are visible in build logs. - The Dockerfile packaging targets and build context are renamed ccachebuild -> prebuilt, since what they consume are artifacts compiled out-of-band in the docker run phase, not anything ccache-specific. Signed-off-by: Steven Noonan --- .github/workflows/build.yml | 1 + .github/workflows/cache-refresh.yml | 68 ---------------------------- .github/workflows/matrix.yml | 56 +++++++++++++---------- .github/workflows/test.yml | 1 + Dockerfile | 40 +++++++--------- hack/build/docker-build-internal.sh | 5 ++ hack/build/generate-docker-script.py | 38 ++++++++++------ 7 files changed, 81 insertions(+), 128 deletions(-) delete mode 100644 .github/workflows/cache-refresh.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c826f3d..2544671 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,6 +24,7 @@ permissions: jobs: build: uses: ./.github/workflows/matrix.yml + secrets: inherit with: spec: ${{ inputs.spec || 'lts' }} publish: ${{ inputs.publish == '' || inputs.publish }} diff --git a/.github/workflows/cache-refresh.yml b/.github/workflows/cache-refresh.yml deleted file mode 100644 index 40585f6..0000000 --- a/.github/workflows/cache-refresh.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Refresh ccache TTL -on: - # We use ccache with github save/restore to dramatically cut kernel build times. - # This works well, but GH has a 10GB limit for all cache entries, - # and a 7-day TTL for *each* cache entry. Which means that if we don't build a kernel for a week, - # we lose our cache benefit entirely, which stinks. The *correct* way to work around this is - # to replace GH's cache action with one that saves/restores directly from a dedicated S3 bucket - # we set up and manage. - # - # What *this* does is save/restore the cache every 4 days, well within the 7-day TTL, - # to keep GH from expiring them. Which is disgusting, but cheap. - schedule: - - cron: "0 0 */4 * *" - workflow_dispatch: -jobs: - discover: - name: discover cache keys - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.list.outputs.matrix }} - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 - with: - egress-policy: audit - - name: list ccache entries - id: list - env: - GH_TOKEN: ${{ github.token }} - run: | - # List main-scoped ccache-* cache keys, strip the run_id suffix to deduplicate - # by flavor/arch. We specifically skip non-main branches, becuse we only want to save/restore - # from main - in GH, PR branches can use `main` caches, but main cannot see PR branch caches, - # and saving duplicated PR branch caches counts against our 10GB github limit - matrix=$(gh api "/repos/${{ github.repository }}/actions/caches" --paginate \ - --jq '[.actions_caches[] - | select(.ref == "refs/heads/main" and (.key | startswith("ccache-"))) - | {prefix: (.key | gsub("-[0-9]+$"; ""))}] - | unique_by(.prefix) - | {entry: .}') - echo "matrix=$matrix" >> "$GITHUB_OUTPUT" - refresh: - name: "refresh ${{ matrix.entry.prefix }}" - needs: discover - if: needs.discover.outputs.matrix != '{"entry":[]}' - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.discover.outputs.matrix) }} - runs-on: ubuntu-latest - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 - with: - egress-policy: audit - - name: restore ccache - id: restore - uses: actions/cache/restore@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v5.0.2 - with: - path: ~/.cache/kernel-ccache - key: "${{ matrix.entry.prefix }}-${{ github.run_id }}" - restore-keys: | - ${{ matrix.entry.prefix }}- - - name: save ccache - if: steps.restore.outputs.cache-matched-key != '' - uses: actions/cache/save@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v5.0.2 - with: - path: ~/.cache/kernel-ccache - key: "${{ matrix.entry.prefix }}-${{ github.run_id }}" diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml index 34c18f5..4a4cce3 100644 --- a/.github/workflows/matrix.yml +++ b/.github/workflows/matrix.yml @@ -92,16 +92,39 @@ jobs: registry: ghcr.io username: "${{github.actor}}" password: "${{secrets.GITHUB_TOKEN}}" - - name: restore ccache - uses: actions/cache/restore@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v5.0.2 - with: - path: ~/.cache/kernel-ccache - # restore-keys is important here - it lets us restore the most recent cache key, - # *ignoring* the specific run ID, as a fuzzy match. So we can use previous build's - # caches for this flavor/arch even if the runid is not the same - key: "ccache-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}-${{ github.run_id }}" - restore-keys: | - ccache-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}- + - name: configure sccache + env: + SCCACHE_CONN_RW: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} + SCCACHE_CONN_RO: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING_RO }} + run: | + # Only explicitly trusted, maintainer-gated triggers get the + # read-write token. Everything else - pull requests today, and any + # trigger added later (e.g. merge_group) - fails closed to the + # read-only token, so code that hasn't been reviewed and merged can + # never poison cache objects later consumed by release builds. Fork + # PRs have no secrets at all and fall back to a runner-local disk + # cache. + case "${{ github.event_name }}" in + push|schedule|workflow_dispatch) + SCCACHE_CONN="${SCCACHE_CONN_RW}" + SCCACHE_MODE="READ_WRITE" + ;; + *) + SCCACHE_CONN="${SCCACHE_CONN_RO}" + SCCACHE_MODE="READ_ONLY" + ;; + esac + if [ -n "${SCCACHE_CONN}" ]; then + # These names must match the -e passthrough list in + # hack/build/generate-docker-script.py (docker_compile); a name + # missing there silently never reaches the build container. + { + echo "SCCACHE_AZURE_CONNECTION_STRING=${SCCACHE_CONN}" + echo "SCCACHE_AZURE_RW_MODE=${SCCACHE_MODE}" + echo "SCCACHE_AZURE_BLOB_CONTAINER=sccache" + echo "SCCACHE_AZURE_KEY_PREFIX=kernel" + } >> "${GITHUB_ENV}" + fi - name: generate docker script run: "./hack/build/generate-docker-script.sh" - name: upload docker script @@ -122,19 +145,6 @@ jobs: if-no-files-found: error compression-level: 0 retention-days: 1 - - name: save ccache - # Only save from main. GH's per-repo 10GB cache pool is shared across all refs, - # so PR-scoped saves evict main-scoped entries via LRU - and main is the only ref - # whose entries the next build can actually restore. PRs still benefit because - # they will fallthrough to/inherit parent branch caches - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v5.0.2 - with: - path: ~/.cache/kernel-ccache - # The run_id here is just for write-key uniqueness, as GH doesn't allow overwriting - # existing cache keys - the `restore` action will fuzzy-match and ignore the run_id - # for subsequent runs. - key: "ccache-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}-${{ github.run_id }}" merge: # Stitch the per-arch single-platform pushes from `build` into multi-arch # manifest lists. Only runs when publishing; no-op when nothing was pushed. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed17fa2..5dc1ba0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,6 +13,7 @@ concurrency: jobs: test: uses: ./.github/workflows/matrix.yml + secrets: inherit with: spec: "only-latest-lts:flavor=host,zone,zone-nvidiagpu" publish: false diff --git a/Dockerfile b/Dockerfile index aa7e01e..b104ed1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,20 +12,12 @@ FROM --platform=$BUILDPLATFORM scratch AS nvidia-modules ARG NV_MODULES_TARBALL_URL= ADD ${NV_MODULES_TARBALL_URL} /nvidia-modules.tar.gz -FROM --platform=$BUILDPLATFORM debian:bookworm@sha256:ed4fcc40bb1162b6d2d32e7bec15044d13963779abbe63f67f1cd62b06220519 AS buildenv -RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y \ - build-essential squashfs-tools python3-yaml \ - patch diffutils sed mawk findutils zstd \ - python3 python3-packaging curl rsync cpio gpg grep \ - flex bison pahole libssl-dev libelf-dev bc kmod ccache && \ - rm -rf /var/lib/apt/lists/* -ARG BUILDPLATFORM -RUN if [ "${BUILDPLATFORM}" = "linux/amd64" ]; then \ - apt-get update && apt-get install -y linux-headers-amd64 g++-aarch64-linux-gnu gcc-aarch64-linux-gnu && rm -rf /var/lib/apt/lists/*; fi -RUN if [ "${BUILDPLATFORM}" = "linux/arm64" ] || [ "${BUILDPLATFORM}" = "linux/aarch64" ]; then \ - apt-get update && apt-get install -y linux-headers-arm64 g++-x86-64-linux-gnu gcc-x86-64-linux-gnu && rm -rf /var/lib/apt/lists/*; fi -ENV PATH="/usr/lib/ccache:${PATH}" -RUN useradd -ms /bin/sh build +# The toolchain (compilers, kbuild deps, sccache + wrappers) comes from the +# published build environment image so it only changes via deliberate, +# reviewed digest bumps - see Dockerfile.buildenv and buildenv.yml. Dependabot +# keeps the pin current, with buildenv-diff.yml summarizing the package +# changes in each bump PR. +FROM --platform=$BUILDPLATFORM ghcr.io/edera-dev/kernel-buildenv:latest@sha256:c0bf191fcec390ef0b97522bcd35e6b776b8ec112676187e3208c933c28c8baa AS buildenv COPY --chown=build:build . /build USER build WORKDIR /build @@ -41,16 +33,16 @@ COPY --from=firmware --chown=build:build /firmware.tar.sign /build/override-firm FROM build-staged AS build-staged-nvidiagpu COPY --from=nvidia-modules --chown=build:build /nvidia-modules.tar.gz /build/override-nvidia-modules.tar.gz -FROM scratch AS kernel-ccachebuild -COPY --from=ccachebuild kernel /kernel/image -COPY --from=ccachebuild config.gz /kernel/config.gz -COPY --from=ccachebuild addons.squashfs /kernel/addons.squashfs -COPY --from=ccachebuild metadata /kernel/metadata +FROM scratch AS kernel-prebuilt +COPY --from=prebuilt kernel /kernel/image +COPY --from=prebuilt config.gz /kernel/config.gz +COPY --from=prebuilt addons.squashfs /kernel/addons.squashfs +COPY --from=prebuilt metadata /kernel/metadata -FROM alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS sdkbuild-ccachebuild +FROM alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS sdkbuild-prebuilt ARG KERNEL_FLAVOR=zone -COPY --from=ccachebuild sdk.tar.gz /sdk.tar.gz -COPY --from=ccachebuild metadata /metadata +COPY --from=prebuilt sdk.tar.gz /sdk.tar.gz +COPY --from=prebuilt metadata /metadata RUN KERNEL_UNAME_R=$(grep '^KERNEL_UNAME_R=' /metadata | cut -d= -f2) && \ mkdir -p /usr/src/kernel-sdk-${KERNEL_UNAME_R}-${KERNEL_FLAVOR} && \ tar -zx -C /usr/src/kernel-sdk-${KERNEL_UNAME_R}-${KERNEL_FLAVOR} -f /sdk.tar.gz && \ @@ -58,5 +50,5 @@ RUN KERNEL_UNAME_R=$(grep '^KERNEL_UNAME_R=' /metadata | cut -d= -f2) && \ ln -sf /usr/src/kernel-sdk-${KERNEL_UNAME_R}-${KERNEL_FLAVOR} /lib/modules/${KERNEL_UNAME_R}/build && \ rm -rf /sdk.tar.gz /metadata -FROM scratch AS sdk-ccachebuild -COPY --from=sdkbuild-ccachebuild /usr/src /usr/src +FROM scratch AS sdk-prebuilt +COPY --from=sdkbuild-prebuilt /usr/src /usr/src diff --git a/hack/build/docker-build-internal.sh b/hack/build/docker-build-internal.sh index c66458f..69e8768 100644 --- a/hack/build/docker-build-internal.sh +++ b/hack/build/docker-build-internal.sh @@ -10,3 +10,8 @@ if [ -z "${KERNEL_VERSION}" ] || [ -z "${KERNEL_FLAVOR}" ]; then fi ./hack/build/build.sh + +# Surface cache effectiveness (hits vs misses, backend errors) in build logs. +# || true: this may respawn an idle-timed-out server, and a transient backend +# error there must not fail an otherwise-successful build (set -e above). +sccache --show-stats || true diff --git a/hack/build/generate-docker-script.py b/hack/build/generate-docker-script.py index fed8687..3369832 100644 --- a/hack/build/generate-docker-script.py +++ b/hack/build/generate-docker-script.py @@ -9,10 +9,11 @@ from matrix import CONFIG from util import format_image_name, maybe, smart_script_split, parse_text_bool, get_branch_tag_suffix -# Targets that are handled via docker run + host CCACHE packaging stages. -CCACHE_TARGET_MAP = { - "kernel": "kernel-ccachebuild", - "sdk": "sdk-ccachebuild", +# Targets packaged from artifacts compiled out-of-band (the docker run compile +# phase) and imported into the image build through the `prebuilt` context. +PREBUILT_TARGET_MAP = { + "kernel": "kernel-prebuilt", + "sdk": "sdk-prebuilt", } # Targets skipped during the packaging phase (handled separately or not needed). @@ -110,7 +111,7 @@ def docker_compile( firmware_url: str, firmware_sig_url: str, ) -> list[str]: - """Generate docker run commands to compile the kernel with ccache.""" + """Generate docker run commands to compile the kernel with sccache.""" lines = [] has_firmware = flavor == "zone-amdgpu" has_nvidia = flavor == "zone-nvidiagpu" @@ -124,7 +125,7 @@ def docker_compile( staged_iidfile = "image-id-%s-%s-%s" % (version, flavor, stage_target) lines += ["", "rm -rf target && mkdir -p target && chmod a+rwX target"] - lines += ['mkdir -p "${HOME}/.cache/kernel-ccache" && chmod -R a+rwX "${HOME}/.cache/kernel-ccache"'] + lines += ['mkdir -p "${HOME}/.cache/kernel-sccache" && chmod -R a+rwX "${HOME}/.cache/kernel-sccache"'] for arch in archs: platform = arch_to_platform(arch) @@ -136,9 +137,19 @@ def docker_compile( "-e", quoted("KERNEL_VERSION=%s" % version), "-e", quoted("KERNEL_FLAVOR=%s" % flavor), "-e", quoted("KERNEL_SRC_URL=/build/override-kernel-src.tar.xz"), - "-e", quoted("CCACHE_DIR=/home/build/.cache/ccache"), - "-e", quoted("CCACHE_COMPRESS=1"), - "-v", quoted("${HOME}/.cache/kernel-ccache:/home/build/.cache/ccache"), + # The Azure sccache env is passed through by name only (no values), + # so the generated script stays free of secrets; docker omits any + # that are unset on the host. Without Azure config sccache falls + # back to the bind-mounted local disk cache below. This name list + # must match what the "configure sccache" step in + # .github/workflows/matrix.yml exports; a name missing here + # silently never reaches the build container. + "-e", quoted("SCCACHE_AZURE_CONNECTION_STRING"), + "-e", quoted("SCCACHE_AZURE_BLOB_CONTAINER"), + "-e", quoted("SCCACHE_AZURE_KEY_PREFIX"), + "-e", quoted("SCCACHE_AZURE_RW_MODE"), + "-e", quoted("SCCACHE_DIR=/home/build/.cache/sccache"), + "-v", quoted("${HOME}/.cache/kernel-sccache:/home/build/.cache/sccache"), "-v", quoted("${PWD}/target:/build/target"), ] if has_firmware: @@ -181,7 +192,7 @@ def docker_build( lines = [] version = dockerify_version(version) - actual_target = CCACHE_TARGET_MAP.get(target, target) + actual_target = PREBUILT_TARGET_MAP.get(target, target) # tag_version is the version string used for OCI tags; version is the # actual kernel version passed as a build arg and recorded in annotations. @@ -219,7 +230,7 @@ def docker_build( image_build_command += ["--platform", quoted(platform)] if actual_target != target: - image_build_command += ["--build-context", quoted("ccachebuild=target")] + image_build_command += ["--build-context", quoted("prebuilt=target")] if mark_format is not None: image_build_command += [ @@ -353,7 +364,8 @@ def generate_builds( firmware_sig_url=firmware_sig_url, ) - # Phase 2: Compile kernel via docker run with ccache bind-mounted from host. + # Phase 2: Compile kernel via docker run; sccache provides the compile + # cache (Azure Blob in CI, bind-mounted local disk otherwise). lines += docker_compile( version=kernel_version, flavor=kernel_flavor, @@ -362,7 +374,7 @@ def generate_builds( firmware_sig_url=firmware_sig_url, ) - # Phase 3: Package kernel and SDK images from the ccache-built artifacts. + # Phase 3: Package kernel and SDK images from the compiled artifacts. for image_config in image_configs: target = image_config["target"] if target in SKIP_PACKAGING_TARGETS: From f15fdc2b0ed9ed086046718e7a32d33ca3f5591d Mon Sep 17 00:00:00 2001 From: Steven Noonan Date: Tue, 14 Jul 2026 12:13:23 -0700 Subject: [PATCH 2/2] build: enumerate kernel.org releases via git tags instead of rsync listings rsync.kernel.org caps concurrent connections (currently 100) and refuses new sessions with exit code 5 when saturated, and matrix generation made one rsync connection for the firmware listing plus one per kernel major series - a connection-slot lottery on every run. Worse, rsync's stderr was discarded, so CI logs never showed the "@ERROR: max connections" message; the failure had to be reproduced locally to be diagnosed. Enumerate releases from git tags instead: `git ls-remote --tags --refs` against git.kernel.org fetches only the server-side-filtered tag advertisement (no clone), and git.kernel.org does not impose the rsync daemon's aggressive connection cap. Release tags map 1:1 to published artifacts: vX.Y[.Z] in stable/linux.git to linux-X.Y[.Z].tar.xz, and YYYYMMDD in firmware/linux-firmware.git to linux-firmware-YYYYMMDD.tar.xz (the live tag sets were verified to match the rsync directory listings exactly, including the firmware range 20190312..present). The listing helper retries with exponential backoff and jitter, and prints the underlying stderr on each failed attempt, so any residual network failure is both tolerated and visible in CI logs. One behavioral note: linux.git tags only reach back to v2.6.11 while the rsync tree lists tarballs to v1.0; nothing anywhere near that old is buildable from this repo. Signed-off-by: Steven Noonan --- hack/build/matrix.py | 48 +++++++++++++++---------------------- hack/build/util.py | 57 ++++++++++++++++++++++++++++++++------------ 2 files changed, 61 insertions(+), 44 deletions(-) diff --git a/hack/build/matrix.py b/hack/build/matrix.py index 9909aae..ac9020c 100644 --- a/hack/build/matrix.py +++ b/hack/build/matrix.py @@ -1,5 +1,6 @@ import json import os +import re import yaml import subprocess @@ -9,7 +10,7 @@ from packaging.version import Version, parse -from util import matches_constraints, list_rsync_dir, format_image_name +from util import matches_constraints, list_remote_git_tags, format_image_name try: from yaml import CLoader as Loader @@ -48,44 +49,33 @@ def get_current_kernel_releases() -> dict[str, any]: @cache def get_all_kernel_releases() -> list[str]: + # Release tags (vX.Y[.Z]) map 1:1 to the published linux-X.Y[.Z].tar.xz + # artifacts; "-" excludes -rc and other pre-release tags. Tags only reach + # back to v2.6.11, but nothing anywhere near that old is buildable here. releases = [] - for maybe_release_major in list_rsync_dir( - "rsync://rsync.kernel.org/pub/linux/kernel/" + for tag in list_remote_git_tags( + "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git" ): - if not ( - maybe_release_major.startswith("v") and maybe_release_major.endswith("x") - ): + if not tag.startswith("v"): continue - for maybe_release_file in list_rsync_dir( - "rsync://rsync.kernel.org/pub/linux/kernel/%s/" % maybe_release_major - ): - if not ( - maybe_release_file.startswith("linux-") - and maybe_release_file.endswith(".tar.xz") - ): - continue - kernel_version = maybe_release_file.replace("linux-", "").replace( - ".tar.xz", "" - ) - if "-" in kernel_version: - continue - releases.append(kernel_version) + kernel_version = tag[1:] + if "-" in kernel_version: + continue + releases.append(kernel_version) return releases @cache def get_all_firmware_releases() -> list[str]: + # Snapshot tags are pure YYYYMMDD and map 1:1 to the published + # linux-firmware-YYYYMMDD.tar.xz artifacts, so lexicographic sort is + # chronological. snapshots = [] - for maybe_release_snapshot in list_rsync_dir( - "rsync://rsync.kernel.org/pub/linux/kernel/firmware/" + for tag in list_remote_git_tags( + "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git" ): - if not ( - maybe_release_snapshot.startswith("linux-firmware-") and maybe_release_snapshot.endswith(".xz") - ): + if not re.fullmatch(r"[0-9]{8}", tag): continue - firmware_snapshot_version = maybe_release_snapshot.replace("linux-firmware-", "").replace( - ".tar.xz", "" - ) - snapshots.append(firmware_snapshot_version) + snapshots.append(tag) snapshots.sort() snapshots.reverse() return snapshots diff --git a/hack/build/util.py b/hack/build/util.py index 15aff86..d09f574 100644 --- a/hack/build/util.py +++ b/hack/build/util.py @@ -1,5 +1,8 @@ import os +import random import re +import sys +import time from typing import Optional from packaging.version import Version @@ -114,24 +117,48 @@ def matches_constraints( return applies -def list_rsync_dir(url: str): - result = subprocess.run( - ["rsync", "--list-only", "--out-format='%n'", url], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - ) +def list_remote_git_tags(url: str, attempts: int = 6) -> list[str]: + # ls-remote fetches only the tag advertisement (protocol v2 filters it + # server-side), so this avoids both a clone and rsync.kernel.org's + # aggressive concurrent-connection cap. Retries cover transient network + # failures, with stderr surfaced so the failure mode shows up in CI logs. + for attempt in range(attempts): + try: + result = subprocess.run( + ["git", "ls-remote", "--tags", "--refs", url], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + # git has no network timeout of its own, so without this a + # hung connection would stall matrix generation until the CI + # job limit instead of falling through to the retry loop. + timeout=120, + ) + except subprocess.TimeoutExpired: + sys.stderr.write( + "listing tags of %s timed out (attempt %d/%d)\n" + % (url, attempt + 1, attempts) + ) + if attempt + 1 < attempts: + time.sleep(min(120, 10 * 2**attempt) + random.uniform(0, 5)) + continue + raise + if result.returncode == 0: + break + sys.stderr.write( + "listing tags of %s failed (attempt %d/%d):\n%s\n" + % (url, attempt + 1, attempts, result.stderr.decode("utf-8", "replace")) + ) + if attempt + 1 < attempts: + time.sleep(min(120, 10 * 2**attempt) + random.uniform(0, 5)) result.check_returncode() - files = [] + tags = [] for line in result.stdout.splitlines(keepends=False): - line = line.decode("utf-8") - if len(line.strip()) == 0: - continue - if line.startswith("MOTD:"): + # "\trefs/tags/" + parts = line.decode("utf-8").strip().split("\t") + if len(parts) != 2 or not parts[1].startswith("refs/tags/"): continue - file_name = str(line.split(" ")[-1]) - if file_name != ".": - files.append(file_name) - return files + tags.append(parts[1][len("refs/tags/"):]) + return tags def parse_text_bool(text: str) -> bool: