diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml index 16bf40f66..adb7d085b 100644 --- a/.github/workflows/lre.yaml +++ b/.github/workflows/lre.yaml @@ -58,211 +58,57 @@ jobs: with: nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} -# remote: -# strategy: -# fail-fast: false -# matrix: -# os: [xlarge-ubuntu-24.04] -# toolchain: [lre-cc, lre-rs] -# name: Remote / ${{ matrix.toolchain }} / ${{ matrix.os }} -# runs-on: ${{ matrix.os }} -# timeout-minutes: 45 -# steps: -# - name: Checkout -# uses: >- # v6.0.2 -# actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - -# - name: Prepare Worker -# uses: ./.github/actions/prepare-nix -# with: -# nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} - -# - name: Start Kubernetes cluster -# run: > -# nix run .#native up - -# - name: Start NativeLink operator -# env: -# REPO_URL: ${{ github.event.pull_request.head.repo.clone_url || format('https://github.com/{0}.git', github.repository) }} -# BRANCH: ${{ github.event.pull_request.head.ref || github.ref_name }} -# COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} -# TOOLCHAIN: ${{ matrix.toolchain }} -# run: | -# nix develop --fallback --impure --command bash -c 'cat > kustomization.yaml << EOF -# apiVersion: kustomize.config.k8s.io/v1beta1 -# kind: Kustomization -# resources: -# - kubernetes/resources/flux -# - kubernetes/resources/nativelink-core -# - kubernetes/resources/gateway-routes -# - kubernetes/workers/resources/worker-init -# - kubernetes/workers/resources/${TOOLCHAIN} -# patches: -# - patch: |- -# - op: replace -# path: /spec/url -# value: ${REPO_URL} -# - op: replace -# path: /spec/ref/branch -# value: ${BRANCH} -# - op: replace -# path: /spec/ref/commit -# value: ${COMMIT} -# target: -# kind: GitRepository -# name: nativelink-infra -# - patch: |- -# - op: replace -# path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT -# value: ./src_root#image -# target: -# kind: Kustomization -# name: nativelink-alert-core -# - patch: |- -# - op: replace -# path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT -# value: ./src_root#nativelink-worker-init -# target: -# kind: Kustomization -# name: nativelink-alert-worker-init -# - patch: |- -# - op: replace -# path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT -# value: ./src_root#nativelink-worker-${TOOLCHAIN} -# target: -# kind: Kustomization -# name: nativelink-alert-${TOOLCHAIN} -# EOF -# kubectl apply -k . && -# rm kustomization.yaml' - -# - name: Wait for Tekton resources -# run: > -# nix develop --impure --command -# bash -c "flux reconcile kustomization \ -# --timeout=15m \ -# nativelink-tekton-resources" - -# - name: Wait for alerts -# env: -# TOOLCHAIN: ${{ matrix.toolchain }} -# run: > -# nix develop --impure --command -# bash -c "flux reconcile kustomization \ -# --timeout=15m \ -# nativelink-alert-core && \ -# flux reconcile kustomization \ -# --timeout=15m \ -# nativelink-alert-worker-init && \ -# flux reconcile kustomization \ -# --timeout=15m \ -# nativelink-alert-${TOOLCHAIN}" - -# - name: Trigger pipelines -# env: -# REPO_URL: ${{ github.event.pull_request.head.repo.clone_url || format('https://github.com/{0}.git', github.repository) }} -# BRANCH: ${{ github.event.pull_request.head.ref || github.ref_name }} -# run: | -# nix develop --impure --command bash -c 'cat > nativelink-repo.yaml << EOF -# apiVersion: source.toolkit.fluxcd.io/v1 -# kind: GitRepository -# metadata: -# name: nativelink -# namespace: default -# spec: -# interval: 2m -# url: ${REPO_URL} -# ref: -# branch: ${BRANCH} -# EOF -# kubectl apply -f nativelink-repo.yaml && -# rm nativelink-repo.yaml' - -# - name: Wait for Tekton pipelines -# run: > -# nix develop --impure --command bash << 'EOF' -# until pr=$(kubectl get pipelinerun -n ci -o name | \ -# grep rebuild-nativelink-run-); do -# echo "Waiting for pipeline to be created..." -# sleep 1 -# done - -# echo "Found pipelinerun: $pr" -# kubectl wait --for=create -n ci $pr - -# echo "Waiting for pipelinerun to succeed..." -# kubectl wait \ -# --for=condition=Succeeded \ -# --timeout=45m \ -# pipelinerun \ -# -n ci \ -# -l tekton.dev/pipeline=rebuild-nativelink -# EOF - -# - name: Wait for NativeLink Kustomization -# run: > -# nix develop --impure --command -# bash -c "flux reconcile kustomization -n default \ -# --timeout=15m \ -# nativelink-core" - -# - name: Debug Kustomizations -# run: > -# nix develop --impure --command -# bash -c "kubectl get kustomizations.kustomize.toolkit.fluxcd.io -A && kubectl events --all-namespaces && kubectl describe kustomizations.kustomize.toolkit.fluxcd.io -A" -# if: always() - -# - name: Wait for Worker Kustomization -# env: -# TOOLCHAIN: ${{ matrix.toolchain }} -# run: > -# nix develop --impure --command -# bash -c "flux reconcile kustomization -n default \ -# --timeout=15m \ -# nativelink-${TOOLCHAIN}" - -# - name: Debug Kustomizations -# run: > -# nix develop --impure --command -# bash -c "kubectl get kustomizations.kustomize.toolkit.fluxcd.io -A && kubectl events --all-namespaces && kubectl describe kustomizations.kustomize.toolkit.fluxcd.io -A" -# if: always() - -# - name: Wait for NativeLink -# run: > -# nix develop --impure --command -# bash -c "kubectl rollout status deploy/nativelink" - -# - name: Wait for worker -# env: -# TOOLCHAIN: ${{ matrix.toolchain }} -# run: > -# nix develop --impure --command -# bash -c "kubectl rollout status deploy/nativelink-worker-${TOOLCHAIN}" - -# - name: Get gateway IPs -# id: gateway-ips -# run: | -# echo "nativelink_ip=$(kubectl get gtw nativelink-gateway -o=jsonpath='{.status.addresses[0].value}')" >> "$GITHUB_ENV" - -# - name: Print cluster state -# run: | -# kubectl get svc -A -# kubectl get pod -A -# kubectl get svc -A -# kubectl get deployments -A -# kubectl describe gtw -# echo "nativelink" -# kubectl logs -l app=nativelink -# echo "worker" -# kubectl logs -l app=nativelink-worker - -# - name: Build example with ${{ matrix.toolchain }} toolchain -# env: -# TOOLCHAIN: ${{ matrix.toolchain }} -# run: > -# nix develop --impure --command -# bash -c "bazel run \ -# --remote_cache=grpc://$nativelink_ip \ -# --remote_executor=grpc://$nativelink_ip \ -# --verbose_failures \ -# @local-remote-execution//examples:${TOOLCHAIN}" + remote-docker-compose: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: >- # v6.0.2 + actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Set up Docker Buildx + uses: >- # v3.9.0 + docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca + + - name: Prepare Worker + uses: ./.github/actions/prepare-nix + with: + nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} + + - name: Build Nativelink images + run: | + nix run .#image.copyTo docker-daemon:local-nativelink:latest + nix run .#nativelink-worker-lre-rs.copyTo docker-daemon:local-worker-rs:latest + nix run .#nativelink-worker-init.copyTo docker-daemon:local-worker-init:latest + + - name: Compile NativeLink with NativeLink + timeout-minutes: 5 # FIXME: Reset to 60 before merge + run: | + set -eu + mkdir .cache + ls -l .cache + chmod 777 .cache + ls -l .cache + cd deployment-examples/docker-compose + (docker compose -f docker-compose-lre.yml up 2>&1 | tee -i docker-compose.log) & + cd ../../ + nix develop --impure --fallback --command \ + bash -c "bazel test //... \ + --lockfile_mode=error \ + --verbose_failures \ + --config self_test --config self_execute" + env: + NATIVELINK_IMAGE: local-nativelink:latest + NATIVELINK_WORKER_INIT_IMAGE: local-worker-init:latest + NATIVELINK_LRE_IMAGE: local-worker-rs:latest + NATIVELINK_DIR: ${{ github.workspace }}/.cache + RUST_LOG: debug + + - name: Check Docker compose + run: cat deployment-examples/docker-compose/docker-compose.log + if: always() + + - name: Teardown Worker + uses: ./.github/actions/end-nix + if: always() + with: + nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2fa5d424d..1992b609f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -20,72 +20,6 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - # TODO(palfrey): Flaky. Fix. - # docker-compose-compiles-nativelink: - # # The type of runner that the job will run on. - # runs-on: ubuntu-24.04 - # strategy: - # matrix: - # # Which OS versions we will test on. - # os_version: [ 24.04 ] - # steps: - # - name: Checkout - # uses: >- # v6.0.2 - # actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - # - name: Set up Docker Buildx - # uses: >- # v3.9.0 - # docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca - - # - name: Build Nativelink image - # uses: >- # v6.13.0 - # docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 - # with: - # context: . - # file: ./deployment-examples/docker-compose/Dockerfile - # build-args: | - # OPT_LEVEL=opt - # OS_VERSION=${{ matrix.os_version }} - # ADDITIONAL_SETUP_WORKER_CMD=apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y gcc g++ lld pkg-config python3 - # load: true # This brings the build into `docker images` from buildx. - # tags: trace_machina/nativelink:latest - - # - name: Build builder image - # uses: >- # v6.13.0 - # docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 - # with: - # context: . - # file: ./deployment-examples/docker-compose/Dockerfile - # build-args: | - # OPT_LEVEL=opt - # OS_VERSION=${{ matrix.os_version }} - # load: true # This brings the build into `docker images` from buildx. - # tags: trace_machina/nativelink:builder - # target: builder - - # - name: Compile NativeLink with NativeLink - # run: | - # mkdir -p ~/.cache && \ - # cd deployment-examples/docker-compose && \ - # (docker-compose up -d || docker compose up -d) && \ - # cd ../../ && \ - # docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \ - # bazel clean && \ - # bazel test //... \ - # --extra_toolchains=@rust_toolchains//:all \ - # --remote_cache=grpc://127.0.0.1:50051 \ - # --remote_executor=grpc://127.0.0.1:50052 \ - # --remote_default_exec_properties=cpu_count=2 \ - # ' && \ - # docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \ - # bazel clean && \ - # bazel test //... \ - # --extra_toolchains=@rust_toolchains//:all \ - # --remote_cache=grpc://127.0.0.1:50051 \ - # --remote_executor=grpc://127.0.0.1:50052 \ - # --remote_default_exec_properties=cpu_count=2 \ - # ' 2>&1 | ( ! grep ' PASSED in ' ) # If we get PASSED without (cache) it means there's a cache issue. - integration-tests: runs-on: ubuntu-24.04 timeout-minutes: 60 diff --git a/MODULE.bazel b/MODULE.bazel index 7d3bb4878..3f278eca6 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,7 +15,7 @@ bazel_dep(name = "rules_shell", version = "0.6.1") # via --extra_toolchains, which outranks the MODULE.bazel registration below, # so Nix users keep getting LRE-CC unchanged. Outside Nix (no lre.bazelrc), # these hermetic toolchains replace the host autodetect path. -bazel_dep(name = "llvm", version = "0.7.7") +bazel_dep(name = "llvm", version = "0.8.0") register_toolchains("@llvm//toolchain:all") @@ -50,9 +50,10 @@ register_toolchains( dev_dependency = True, ) -bazel_dep(name = "rules_rs", version = "0.0.76") +bazel_dep(name = "rules_rs", version = "0.0.77") -# Pin rules_rust to the hermeticbuild fork (the same commit rules_rs provisions) +# Pin rules_rust to the hermeticbuild fork (the same commit rules_rs provisions +# see https://github.com/hermeticbuild/rules_rs/blob/v0.0.77/rs/rules_rust.bzl) # so all `@rules_rust//...` references resolve to the patched ruleset. The # bazel_dep + archive_override is the form that keeps `@rules_rust` visible to # subsequent `use_extension(...)` calls in this MODULE.bazel — the alternative @@ -62,15 +63,15 @@ bazel_dep(name = "rules_rs", version = "0.0.76") # The local musl-platforms patch is still applied because the hermeticbuild # fork does not list x86_64/aarch64-unknown-linux-musl as supported triples in # rust/platform/triple_mappings.bzl. -bazel_dep(name = "rules_rust", version = "0.68.1") +bazel_dep(name = "rules_rust", version = "0.70.0") archive_override( module_name = "rules_rust", - integrity = "sha256-HG4cSGKVIoZTn0zpUNKhJbGvFfD2UVPJqKRqgTqLOQQ=", + integrity = "sha256-aROt7MxNPUOELpgM4BZEQpptsoT6t2qjFpAzyZRF0LE=", patch_strip = 1, patches = ["//tools:rules_rust-musl-platforms.diff"], - strip_prefix = "rules_rust-cf176d81c12d9c8f6420c7d433b0af0f08d2abb1", + strip_prefix = "rules_rust-1c00622c3ba0269a6d5306772fe2e9aff4bf89a0", urls = [ - "https://github.com/hermeticbuild/rules_rust/archive/cf176d81c12d9c8f6420c7d433b0af0f08d2abb1.tar.gz", + "https://github.com/hermeticbuild/rules_rust/archive/1c00622c3ba0269a6d5306772fe2e9aff4bf89a0.tar.gz", ], ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index d6608223b..cddd3115e 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -83,7 +83,8 @@ "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/llvm/0.7.7/MODULE.bazel": "0eeaf1814feca77abc7af3523e2b9d3735f92e2583043f8d6a2cc0fb5f479a28", - "https://bcr.bazel.build/modules/llvm/0.7.7/source.json": "7c8910307329462a21b7bdcc52710360da3de8284738dca52241bb15302a00dc", + "https://bcr.bazel.build/modules/llvm/0.8.0/MODULE.bazel": "72e043f9ad3cdd721659c74010fff87a4ce970abf8d47c04f48a0371234191e3", + "https://bcr.bazel.build/modules/llvm/0.8.0/source.json": "bbe84a7c74b4bbb964a5dad5071c80e4ea36da42f798f70844658b8fd3b236af", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83", "https://bcr.bazel.build/modules/package_metadata/0.0.7/MODULE.bazel": "7adb03933fc8401f495800cf4eafcff0edc6da0ff55c7db223ef69d19f689486", @@ -193,7 +194,8 @@ "https://bcr.bazel.build/modules/rules_python/2.0.0/MODULE.bazel": "1459089e2d4194d2a49e07896f5334fb230a8f2966ae945b1f793bef87a292fd", "https://bcr.bazel.build/modules/rules_python/2.0.0/source.json": "b8e25661f58c573e5e27af21295867e87766e89211f326fcb84034e6e6b6794b", "https://bcr.bazel.build/modules/rules_rs/0.0.76/MODULE.bazel": "461dcf664f368fdc921f67ea20ec1bc78c73f65a0a20b6e2a6d4b1c77fbde8c1", - "https://bcr.bazel.build/modules/rules_rs/0.0.76/source.json": "167eb1122e0f74848fc995b581061155dda1dfd600a38c253a85ef46d0523221", + "https://bcr.bazel.build/modules/rules_rs/0.0.77/MODULE.bazel": "709814c4dc438fe77234ffa04a0ffe12e511e7e7d051590a0e8f43db678e712d", + "https://bcr.bazel.build/modules/rules_rs/0.0.77/source.json": "6af335041ccb8a949dad0cf55e4785a68c622012646b820f87a9c40038bf1936", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", @@ -209,8 +211,9 @@ "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/MODULE.bazel": "e8f9ff79199e8d9eaad7f1b0a77ad74b30bb82d794b87d8ca942bead5de83ae9", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/source.json": "20143442376c03426f6135292ba02d825cb75308aa47e6bf42dd4cc5a435c2ff", "https://bcr.bazel.build/modules/tar.bzl/0.9.0/MODULE.bazel": "452a22d7f02b1c9d7a22ab25edf20f46f3e1101f0f67dc4bfbf9a474ddf02445", - "https://bcr.bazel.build/modules/tar.bzl/0.9.0/source.json": "c732760a374831a2cf5b08839e4be75017196b4d796a5aa55235272ee17cd839", "https://bcr.bazel.build/modules/toolchains_protoc/0.4.3/MODULE.bazel": "54daf5468a9c3e52f6c8a96c8e0b867f7b30029dfe1e74f5a59bf081921d91a3", "https://bcr.bazel.build/modules/toolchains_protoc/0.4.3/source.json": "fbf3886395e08c407caca84f92f8c9ad92b05ce126a94883def1e150edd6b417", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", @@ -226,7 +229,7 @@ "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", - "usagesDigest": "ibiT5THLv67F8GNR92fjXCqmyYBFE4ZBphBLqWLTFBI=", + "usagesDigest": "QyL2lVFExcQwxFR4MIRIQssq2OMzBZKS9pogrUzpqwY=", "recordedInputs": [ "REPO_MAPPING:aspect_tools_telemetry+,bazel_lib bazel_lib+", "REPO_MAPPING:aspect_tools_telemetry+,bazel_skylib bazel_skylib+" @@ -236,7 +239,7 @@ "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", "attributes": { "deps": { - "rules_rs": "0.0.76", + "rules_rs": "0.0.77", "aspect_tools_telemetry": "0.3.3" } } @@ -1518,6 +1521,7 @@ "rustc-1.92.0-aarch64-apple-darwin.tar.xz": "15dee753c9217dff4cf45d734b29dc13ce6017d8a55fe34eed75022b39a63ff0", "rustc-1.92.0-aarch64-pc-windows-msvc.tar.xz": "07ba5606143de3bc916455714f67732fca05805591396a8190d5e88b8863eea3", "rustc-1.92.0-aarch64-unknown-linux-gnu.tar.xz": "7c8706fad4c038b5eacab0092e15db54d2b365d5f3323ca046fe987f814e7826", + "rustc-1.92.0-src.tar.xz": "ebee170bfe4c4dfc59521a101de651e5534f4dae889756a5c97ca9ea40d0c307", "rustc-1.92.0-x86_64-apple-darwin.tar.xz": "0facbd5d2742c8e97c53d59c9b5b81db6088cfc285d9ecb99523a50d6765fc5c", "rustc-1.92.0-x86_64-pc-windows-msvc.tar.xz": "6aec5aba5384ce1041538e71ebebf671d82e29f4fe697bc7844626d94b0dbfe6", "rustc-1.92.0-x86_64-unknown-linux-gnu.tar.xz": "78b2dd9c6b1fcd2621fa81c611cf5e2d6950690775038b585c64f364422886e0", diff --git a/deployment-examples/docker-compose/Dockerfile b/deployment-examples/docker-compose/Dockerfile index c95796932..268cf336c 100644 --- a/deployment-examples/docker-compose/Dockerfile +++ b/deployment-examples/docker-compose/Dockerfile @@ -66,4 +66,4 @@ RUN apt-get update \ && mkdir -p /root/.cache/nativelink EXPOSE 50051/tcp 50052/tcp -CMD ["nativelink"] +ENTRYPOINT ["nativelink"] diff --git a/deployment-examples/docker-compose/docker-compose-lre.yml b/deployment-examples/docker-compose/docker-compose-lre.yml new file mode 100644 index 000000000..a9ea33af4 --- /dev/null +++ b/deployment-examples/docker-compose/docker-compose-lre.yml @@ -0,0 +1,102 @@ +# Copyright 2022 The NativeLink Authors. All rights reserved. +# +# Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# See LICENSE file for details +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + nativelink_local_cas: + image: ${NATIVELINK_IMAGE:-trace_machina/nativelink:latest} + build: + context: ../.. + dockerfile: ./deployment-examples/docker-compose/Dockerfile + network: host + args: + - ADDITIONAL_SETUP_WORKER_CMD=${ADDITIONAL_SETUP_WORKER_CMD:-} + volumes: + - ${NATIVELINK_DIR:-~/.cache/nativelink}:/root/.cache/nativelink + - type: bind + source: . + target: /root + environment: + RUST_LOG: ${RUST_LOG:-warn} + NL_LOG: json + ports: [ "50051:50051/tcp", "127.0.0.1:50061:50061", "50071:50071/tcp", ] + command: | + /root/local-storage-cas.json5 + + nativelink_scheduler: + image: ${NATIVELINK_IMAGE:-trace_machina/nativelink:latest} + build: + context: ../.. + dockerfile: ./deployment-examples/docker-compose/Dockerfile + network: host + args: + - ADDITIONAL_SETUP_WORKER_CMD=${ADDITIONAL_SETUP_WORKER_CMD:-} + volumes: + - type: bind + source: . + target: /root + environment: + RUST_LOG: ${RUST_LOG:-warn} + CAS_ENDPOINT: nativelink_local_cas + NL_LOG: json + ports: [ "50052:50052/tcp" ] + command: | + /root/scheduler.json5 + depends_on: + - nativelink_local_cas + + executor-init: + image: ${NATIVELINK_WORKER_INIT_IMAGE:-trace_machina/nativelink:latest} + build: + context: ../.. + dockerfile: ./deployment-examples/docker-compose/Dockerfile + network: host + args: + - ADDITIONAL_SETUP_WORKER_CMD=${ADDITIONAL_SETUP_WORKER_CMD:-} + command: /shared + volumes: + - exec-shared:/shared + + nativelink_executor: + image: ${NATIVELINK_LRE_IMAGE:-trace_machina/nativelink:latest} + privileged: true # So we can do use_namespaces + build: + context: ../.. + dockerfile: ./deployment-examples/docker-compose/Dockerfile + network: host + args: + - ADDITIONAL_SETUP_WORKER_CMD=${ADDITIONAL_SETUP_WORKER_CMD:-} + volumes: + - ${NATIVELINK_DIR:-~/.cache/nativelink}:/root/.cache/nativelink + - type: bind + source: . + target: /root + - exec-shared:/shared + environment: + RUST_LOG: ${RUST_LOG:-warn} + NL_LOG: json + CAS_ENDPOINT: nativelink_local_cas + SCHEDULER_ENDPOINT: nativelink_scheduler + entrypoint: /shared/nativelink + command: | + /root/worker.json5 + depends_on: + nativelink_local_cas: + condition: service_started + nativelink_scheduler: + condition: service_started + executor-init: + condition: service_completed_successfully + +volumes: + exec-shared: diff --git a/deployment-examples/docker-compose/docker-compose.yml b/deployment-examples/docker-compose/docker-compose.yml index 3eb071aa1..bb9cdece6 100644 --- a/deployment-examples/docker-compose/docker-compose.yml +++ b/deployment-examples/docker-compose/docker-compose.yml @@ -14,7 +14,7 @@ services: nativelink_local_cas: - image: trace_machina/nativelink:latest + image: ${NATIVELINK_IMAGE:-trace_machina/nativelink:latest} build: context: ../.. dockerfile: ./deployment-examples/docker-compose/Dockerfile @@ -30,10 +30,10 @@ services: RUST_LOG: ${RUST_LOG:-warn} ports: [ "50051:50051/tcp", "127.0.0.1:50061:50061", "50071:50071/tcp", ] command: | - nativelink /root/local-storage-cas.json5 + /root/local-storage-cas.json5 nativelink_scheduler: - image: trace_machina/nativelink:latest + image: ${NATIVELINK_IMAGE:-trace_machina/nativelink:latest} build: context: ../.. dockerfile: ./deployment-examples/docker-compose/Dockerfile @@ -49,12 +49,12 @@ services: CAS_ENDPOINT: nativelink_local_cas ports: [ "50052:50052/tcp" ] command: | - nativelink /root/scheduler.json5 + /root/scheduler.json5 depends_on: - nativelink_local_cas nativelink_executor: - image: trace_machina/nativelink:latest + image: ${NATIVELINK_LRE_IMAGE:-trace_machina/nativelink:latest} privileged: true # So we can do use_namespaces build: context: ../.. @@ -72,7 +72,9 @@ services: CAS_ENDPOINT: nativelink_local_cas SCHEDULER_ENDPOINT: nativelink_scheduler command: | - nativelink /root/worker.json5 + /root/worker.json5 depends_on: - - nativelink_local_cas - - nativelink_scheduler + nativelink_local_cas: + condition: service_started + nativelink_scheduler: + condition: service_started diff --git a/deployment-examples/docker-compose/local-storage-cas.json5 b/deployment-examples/docker-compose/local-storage-cas.json5 index 6d4acdeed..428f36d8f 100644 --- a/deployment-examples/docker-compose/local-storage-cas.json5 +++ b/deployment-examples/docker-compose/local-storage-cas.json5 @@ -54,13 +54,13 @@ }, ], capabilities: [], - bytestream: { - cas_stores: { - "": "CAS_MAIN_STORE", + bytestream: [ + { + cas_store: "CAS_MAIN_STORE", }, - }, - fetch: {}, - push: {}, + ], + fetch: [], + push: [], }, }, { @@ -91,12 +91,12 @@ }, ], capabilities: [], - bytestream: { - cas_stores: { - "": "CAS_MAIN_STORE", + bytestream: [ + { + cas_store: "CAS_MAIN_STORE", }, - }, - health: {}, + ], + health: [], }, }, ], diff --git a/deployment-examples/docker-compose/worker.json5 b/deployment-examples/docker-compose/worker.json5 index 7e55b0f17..fff3cd1aa 100644 --- a/deployment-examples/docker-compose/worker.json5 +++ b/deployment-examples/docker-compose/worker.json5 @@ -63,7 +63,7 @@ work_directory: "/root/.cache/nativelink/work", platform_properties: { cpu_count: { - query_cmd: "nproc", + values: [], }, OSFamily: { values: [ diff --git a/flake.nix b/flake.nix index ad1e68fac..effc6375f 100644 --- a/flake.nix +++ b/flake.nix @@ -164,6 +164,9 @@ (craneLibFor p).buildPackage ((commonArgsFor p) // { cargoArtifacts = cargoArtifactsFor p; + # If you're testing Nativelink images, doing a dev profile will + # massively speedup build times. Just don't commit/push them! + # CARGO_PROFILE = "dev"; }); nativeTargetPkgs = diff --git a/local-remote-execution/MODULE.bazel b/local-remote-execution/MODULE.bazel index 8c95cf2ec..868f70510 100644 --- a/local-remote-execution/MODULE.bazel +++ b/local-remote-execution/MODULE.bazel @@ -14,8 +14,8 @@ bazel_dep(name = "rules_cc", version = "0.2.18") # Use the starlark implementation of Java rules instead of the builtin ones. bazel_dep(name = "rules_java", version = "8.11.0") -bazel_dep(name = "rules_rs", version = "0.0.76") -bazel_dep(name = "rules_rust", version = "0.68.1") +bazel_dep(name = "rules_rs", version = "0.0.77") +bazel_dep(name = "rules_rust", version = "0.70.0") bazel_dep(name = "bazel_skylib", version = "1.9.0") # rules_rust is pinned to the hermeticbuild fork in the parent MODULE.bazel via # archive_override; this submodule resolves to the same archive automatically. diff --git a/local-remote-execution/MODULE.bazel.lock b/local-remote-execution/MODULE.bazel.lock index a20eb23b9..2244c9a1a 100644 --- a/local-remote-execution/MODULE.bazel.lock +++ b/local-remote-execution/MODULE.bazel.lock @@ -82,8 +82,8 @@ "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", - "https://bcr.bazel.build/modules/llvm/0.7.7/MODULE.bazel": "0eeaf1814feca77abc7af3523e2b9d3735f92e2583043f8d6a2cc0fb5f479a28", - "https://bcr.bazel.build/modules/llvm/0.7.7/source.json": "7c8910307329462a21b7bdcc52710360da3de8284738dca52241bb15302a00dc", + "https://bcr.bazel.build/modules/llvm/0.8.0/MODULE.bazel": "72e043f9ad3cdd721659c74010fff87a4ce970abf8d47c04f48a0371234191e3", + "https://bcr.bazel.build/modules/llvm/0.8.0/source.json": "bbe84a7c74b4bbb964a5dad5071c80e4ea36da42f798f70844658b8fd3b236af", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83", "https://bcr.bazel.build/modules/package_metadata/0.0.7/MODULE.bazel": "7adb03933fc8401f495800cf4eafcff0edc6da0ff55c7db223ef69d19f689486", @@ -190,10 +190,10 @@ "https://bcr.bazel.build/modules/rules_python/1.6.3/MODULE.bazel": "a7b80c42cb3de5ee2a5fa1abc119684593704fcd2fec83165ebe615dec76574f", "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", "https://bcr.bazel.build/modules/rules_python/1.7.0/source.json": "028a084b65dcf8f4dc4f82f8778dbe65df133f234b316828a82e060d81bdce32", - "https://bcr.bazel.build/modules/rules_rs/0.0.76/MODULE.bazel": "461dcf664f368fdc921f67ea20ec1bc78c73f65a0a20b6e2a6d4b1c77fbde8c1", - "https://bcr.bazel.build/modules/rules_rs/0.0.76/source.json": "167eb1122e0f74848fc995b581061155dda1dfd600a38c253a85ef46d0523221", - "https://bcr.bazel.build/modules/rules_rust/0.68.1/MODULE.bazel": "8d3332ef4079673385eb81f8bd68b012decc04ac00c9d5a01a40eff90301732c", - "https://bcr.bazel.build/modules/rules_rust/0.68.1/source.json": "3378e746f81b62457fdfd37391244fa8ff075ba85c05931ee4f3a20ac1efe963", + "https://bcr.bazel.build/modules/rules_rs/0.0.77/MODULE.bazel": "709814c4dc438fe77234ffa04a0ffe12e511e7e7d051590a0e8f43db678e712d", + "https://bcr.bazel.build/modules/rules_rs/0.0.77/source.json": "6af335041ccb8a949dad0cf55e4785a68c622012646b820f87a9c40038bf1936", + "https://bcr.bazel.build/modules/rules_rust/0.70.0/MODULE.bazel": "5b1407b11c305bc2522e204e7f170faf8399e836e49b6afef9074dfe532e6c3f", + "https://bcr.bazel.build/modules/rules_rust/0.70.0/source.json": "24ae6d23425359db1c3148aa22c389970fce9a06102b2b3a329a2800f9569de2", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", @@ -210,8 +210,8 @@ "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", - "https://bcr.bazel.build/modules/tar.bzl/0.9.0/MODULE.bazel": "452a22d7f02b1c9d7a22ab25edf20f46f3e1101f0f67dc4bfbf9a474ddf02445", - "https://bcr.bazel.build/modules/tar.bzl/0.9.0/source.json": "c732760a374831a2cf5b08839e4be75017196b4d796a5aa55235272ee17cd839", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/MODULE.bazel": "e8f9ff79199e8d9eaad7f1b0a77ad74b30bb82d794b87d8ca942bead5de83ae9", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/source.json": "20143442376c03426f6135292ba02d825cb75308aa47e6bf42dd4cc5a435c2ff", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/source.json": "3f3fbaeafecaf629877ad152a2c9def21f8d330d91aa94c5dc75bbb98c10b8b8", diff --git a/local-remote-execution/flake-module.nix b/local-remote-execution/flake-module.nix index f548553a7..575dc4132 100644 --- a/local-remote-execution/flake-module.nix +++ b/local-remote-execution/flake-module.nix @@ -100,9 +100,9 @@ in { "aarch64-darwin" = "aarch64-apple-darwin"; - "aarch64-linux" = "aarch64-unknown-linux-gnu"; + "aarch64-linux" = "aarch64-unknown-linux-musl"; "x86_64-darwin" = "x86_64-apple-darwin"; - "x86_64-linux" = "x86_64-unknown-linux-gnu"; + "x86_64-linux" = "x86_64-unknown-linux-musl"; }.${ system } or (throw "Unsupported Nix exec platform: ${system}"); diff --git a/local-remote-execution/rust/BUILD.bazel b/local-remote-execution/rust/BUILD.bazel index be3a63088..637cca645 100644 --- a/local-remote-execution/rust/BUILD.bazel +++ b/local-remote-execution/rust/BUILD.bazel @@ -98,8 +98,14 @@ config_setting( }), env = {}, exec_triple = { - "x86_64-linux": "x86_64-unknown-linux-gnu", - "aarch64-linux": "aarch64-unknown-linux-gnu", + "x86_64-linux": select({ + "@local-remote-execution//libc:glibc": "x86_64-unknown-linux-gnu", + "//conditions:default": "x86_64-unknown-linux-musl", + }), + "aarch64-linux": select({ + "@local-remote-execution//libc:glibc": "aarch64-unknown-linux-gnu", + "//conditions:default": "aarch64-unknown-linux-musl", + }), "x86_64-darwin": "x86_64-apple-darwin", "aarch64-darwin": "aarch64-apple-darwin", }[nix_system], diff --git a/templates/bazel/MODULE.bazel b/templates/bazel/MODULE.bazel index a65d6c89d..9895b120a 100644 --- a/templates/bazel/MODULE.bazel +++ b/templates/bazel/MODULE.bazel @@ -13,4 +13,4 @@ archive_override( urls = ["https://github.com/TraceMachina/nativelink/archive/f9ff630e09a3c22d6a3abea68d1bacc775eac6bb.zip"], ) -bazel_dep(name = "rules_cc", version = "0.2.8") +bazel_dep(name = "rules_cc", version = "0.2.18") diff --git a/toolchain-examples/MODULE.bazel b/toolchain-examples/MODULE.bazel index c37339476..c3517011e 100644 --- a/toolchain-examples/MODULE.bazel +++ b/toolchain-examples/MODULE.bazel @@ -38,8 +38,8 @@ bazel_dep(name = "rules_go", version = "0.60.0") # Rust # -bazel_dep(name = "rules_rs", version = "0.0.76") -bazel_dep(name = "llvm", version = "0.7.7") +bazel_dep(name = "rules_rs", version = "0.0.77") +bazel_dep(name = "llvm", version = "0.8.0") rust_toolchains = use_extension("@rules_rs//rs/toolchains:module_extension.bzl", "toolchains") rust_toolchains.toolchain( diff --git a/tools/public/create-worker.nix b/tools/public/create-worker.nix index 23308f7e8..9725ef66d 100644 --- a/tools/public/create-worker.nix +++ b/tools/public/create-worker.nix @@ -5,7 +5,6 @@ nix2container, coreutils, gnused, - lib, runCommand, runtimeShell, self, @@ -31,25 +30,13 @@ ln -s /bin/env $out/usr/bin/env ''; - user = "nativelink"; - group = "nativelink"; - uid = "1000"; - gid = "1000"; - mkUser = runCommand "mkUser" {} '' mkdir -p $out/etc/pam.d echo "root:x:0:0::/root:${runtimeShell}" > $out/etc/passwd - echo "${user}:x:${uid}:${gid}:::" >> $out/etc/passwd - echo "root:!x:::::::" > $out/etc/shadow - echo "${user}:!x:::::::" >> $out/etc/shadow - echo "root:x:0:" > $out/etc/group - echo "${group}:x:${gid}:" >> $out/etc/group - echo "root:x::" > $out/etc/gshadow - echo "${group}:x::" >> $out/etc/gshadow cat > $out/etc/pam.d/other <