diff --git a/.github/actions/bazel/action.yaml b/.github/actions/bazel/action.yaml index e71245bfadce..a5550998c4ba 100644 --- a/.github/actions/bazel/action.yaml +++ b/.github/actions/bazel/action.yaml @@ -158,9 +158,6 @@ runs: ^@@rules_rust..crate.crate_index__cranelift-isle # has a non reproducible isle_tests.rs in its OUT_DIR. ^@@rules_rust..crate.crate_index__secp256k1-sys # has non reproducible object files, like lax_der_parsing.o, in it OUT_DIR. ^@@rules_rust..crate.crate_index__sev # build.rs depends on the presence of /dev/sev and /dev/sev-guest. See: https://github.com/virtee/sev/issues/315 - # TODO: fix the following ASAP: - ^//ic-os/hostos/envs/prod:rootfs-tree.tar # became non-reproducible because of https://github.com/dfinity/ic/pull/10208 - ^//ic-os/setupos/envs/prod:rootfs-tree.tar # idem ) for execlog_zst in $(find '${{ steps.metrics-tmpdir.outputs.dir }}' -name 'execlog-*.zst'); do zstd -f -d "$execlog_zst" -o "$execlog" diff --git a/MODULE.bazel b/MODULE.bazel index 82f472ca3237..edc8867101ca 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -322,6 +322,18 @@ http_archive( url = "https://github.com/apalache-mc/apalache/releases/download/v0.52.2/apalache-0.52.2.tgz", ) +# Android SDK platform-tools, for the hermetic `e2fsdroid` binary used while +# assembling ext4 filesystem images. This is the same statically-linked binary +# that Ubuntu's `google-android-platform-tools-installer` package downloads at +# install time; pinning it here removes the host dependence. +http_archive( + name = "android_platform_tools", + build_file_content = """exports_files(["e2fsdroid"], visibility = ["//visibility:public"])""", + sha256 = "defcee9da1f22fe5c2324ec0edf612122f1c6ffe01a7b124191e07fcc74f8fff", + strip_prefix = "platform-tools", + url = "https://dl.google.com/android/repository/platform-tools_r33.0.2-linux.zip", +) + # Official WebAssembly test suite. # To be used for testing libraries that handle canister Wasm code. http_archive( diff --git a/bazel/noble.lock.json b/bazel/noble.lock.json index 786843c37bfe..076a8d9d4885 100755 --- a/bazel/noble.lock.json +++ b/bazel/noble.lock.json @@ -983,6 +983,84 @@ "url": "https://snapshot.ubuntu.com/ubuntu/20260131T000000Z/pool/main/u/util-linux/libblkid1_2.39.3-9ubuntu6.4_amd64.deb", "version": "2.39.3-9ubuntu6.4" }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.39-0ubuntu8.6_amd64", + "name": "libc6", + "version": "2.39-0ubuntu8.6" + }, + { + "key": "libgcc-s1_14.2.0-4ubuntu2_24.04_amd64", + "name": "libgcc-s1", + "version": "14.2.0-4ubuntu2~24.04" + }, + { + "key": "gcc-14-base_14.2.0-4ubuntu2_24.04_amd64", + "name": "gcc-14-base", + "version": "14.2.0-4ubuntu2~24.04" + }, + { + "key": "libfakeroot_1.33-1_amd64", + "name": "libfakeroot", + "version": "1.33-1" + } + ], + "key": "fakeroot_1.33-1_amd64", + "name": "fakeroot", + "sha256": "d58b7fc73f0b7ae8b9441b0aa41e44ead0d7e4deedf5aaf5696db182eac71031", + "url": "https://snapshot.ubuntu.com/ubuntu/20260131T000000Z/pool/main/f/fakeroot/fakeroot_1.33-1_amd64.deb", + "version": "1.33-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libfakeroot_1.33-1_amd64", + "name": "libfakeroot", + "sha256": "a78087f50586595375f850b4445261f7a47a83005b05ce521331929fab49df4b", + "url": "https://snapshot.ubuntu.com/ubuntu/20260131T000000Z/pool/main/f/fakeroot/libfakeroot_1.33-1_amd64.deb", + "version": "1.33-1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.39-0ubuntu8.6_amd64", + "name": "libc6", + "version": "2.39-0ubuntu8.6" + }, + { + "key": "libgcc-s1_14.2.0-4ubuntu2_24.04_amd64", + "name": "libgcc-s1", + "version": "14.2.0-4ubuntu2~24.04" + }, + { + "key": "gcc-14-base_14.2.0-4ubuntu2_24.04_amd64", + "name": "gcc-14-base", + "version": "14.2.0-4ubuntu2~24.04" + }, + { + "key": "libfaketime_0.9.10-2.1_amd64", + "name": "libfaketime", + "version": "0.9.10-2.1" + } + ], + "key": "faketime_0.9.10-2.1_amd64", + "name": "faketime", + "sha256": "665f136637004d2f1c9af0cef03b9070ea6a68087c167b83ec6417095530d3f5", + "url": "https://snapshot.ubuntu.com/ubuntu/20260131T000000Z/pool/universe/f/faketime/faketime_0.9.10-2.1_amd64.deb", + "version": "0.9.10-2.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libfaketime_0.9.10-2.1_amd64", + "name": "libfaketime", + "sha256": "25fc8987f1f700c58603f68edd93ac03a1b7dea35da1e84509f29f444d5b11e9", + "url": "https://snapshot.ubuntu.com/ubuntu/20260131T000000Z/pool/universe/f/faketime/libfaketime_0.9.10-2.1_amd64.deb", + "version": "0.9.10-2.1" + }, { "arch": "amd64", "dependencies": [ diff --git a/bazel/noble.yaml b/bazel/noble.yaml index 186559f652f2..e372c9cb2a2d 100644 --- a/bazel/noble.yaml +++ b/bazel/noble.yaml @@ -27,6 +27,8 @@ packages: - "dosfstools" - "dpkg" # for apt list --installed - "e2fsprogs" # for mkfs.ext4 used in ICOS device tests + - "fakeroot" # for hermetic fakeroot used in ICOS image builds + - "faketime" # for hermetic faketime used in ICOS image builds - "gawk" # for build-bootstrap-config-image - "gzip" # for tar-ing up ic regsitry store in systests - "libcryptsetup-dev" diff --git a/toolchains/sysimage/BUILD.bazel b/toolchains/sysimage/BUILD.bazel index 4e9502b55f79..da77035950d4 100644 --- a/toolchains/sysimage/BUILD.bazel +++ b/toolchains/sysimage/BUILD.bazel @@ -130,6 +130,68 @@ filegroup( srcs = [":mkfs_ext4_files"], ) +# Hermetic libfaketime.so.1 extracted from the apt-snapshot-pinned libfaketime +# .deb. We deliberately do NOT use the `faketime` wrapper binary because it +# hard-codes /usr/$LIB/faketime/libfaketime.so.1 and so cannot be relocated. +# Instead build_ext4_image.py LD_PRELOADs this library and sets FAKETIME +# directly, matching what the wrapper would do. +genrule( + name = "libfaketime_files", + srcs = ["@noble//libfaketime/amd64:data"], + outs = ["faketime.d/libfaketime.so.1"], + cmd = "tar -xzOf $(location @noble//libfaketime/amd64:data) ./usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1 > $@", +) + +filegroup( + name = "libfaketime_runfiles", + srcs = [":libfaketime_files"], +) + +# Hermetic fakeroot bits extracted from the apt-snapshot-pinned fakeroot and +# libfakeroot .debs: the faked-sysv daemon and the libfakeroot-sysv.so +# LD_PRELOAD library. We deliberately do NOT use the /usr/bin/fakeroot shell +# wrapper because it does platform-specific library lookups against the host +# filesystem. Instead build_ext4_image.py spawns faked-sysv itself and sets +# FAKEROOTKEY + LD_PRELOAD directly. +genrule( + name = "fakeroot_files", + srcs = [ + "@noble//fakeroot/amd64:data", + "@noble//libfakeroot/amd64:data", + ], + outs = [ + "fakeroot.d/faked-sysv", + "fakeroot.d/libfakeroot-sysv.so", + ], + cmd = """ +set -euo pipefail +tmp=$$(mktemp -d) +trap 'rm -rf $$tmp' EXIT +for f in $(SRCS); do tar -xzf $$f -C $$tmp; done +cp $$tmp/usr/bin/faked-sysv $(execpath fakeroot.d/faked-sysv) +chmod +x $(execpath fakeroot.d/faked-sysv) +cp $$tmp/usr/lib/x86_64-linux-gnu/libfakeroot/libfakeroot-sysv.so $(execpath fakeroot.d/libfakeroot-sysv.so) +""", +) + +filegroup( + name = "fakeroot_runfiles", + srcs = [":fakeroot_files"], +) + +# Hermetic e2fsdroid: re-export the statically-linked binary from the pinned +# Android SDK platform-tools zip. The host's /usr/bin/e2fsdroid (provided by +# Ubuntu's google-android-platform-tools-installer) comes from the very same +# upstream zip, so pinning it via http_archive removes the host dependence +# without changing the binary or its output. +genrule( + name = "e2fsdroid_bin", + srcs = ["@android_platform_tools//:e2fsdroid"], + outs = ["e2fsdroid"], + cmd = "cp $(location @android_platform_tools//:e2fsdroid) $@ && chmod +x $@", + executable = True, +) + py_binary( name = "verity_sign", srcs = ["verity_sign.py"], diff --git a/toolchains/sysimage/build_ext4_image.py b/toolchains/sysimage/build_ext4_image.py index f33ed7a4cbb9..f9b44bdcf7c1 100755 --- a/toolchains/sysimage/build_ext4_image.py +++ b/toolchains/sysimage/build_ext4_image.py @@ -8,10 +8,87 @@ # build_ext4_image -s 10M -o partition.img.tzst -p boot -i dockerimg.tar -S file_contexts # import argparse +import contextlib import os +import signal import subprocess import sys import tempfile +import time + + +def faketime_env(libfaketime, base_env=None): + """ + Return an env that LD_PRELOADs our hermetic libfaketime.so.1 and pins + the wall clock to the Unix epoch. Equivalent to what the `faketime + -f '1970-1-1 0:0:0'` wrapper does, but without invoking the wrapper + binary, which hard-codes /usr/$LIB/faketime/libfaketime.so.1. + """ + env = dict(base_env if base_env is not None else os.environ) + # The `x0` speed multiplier freezes the fake clock so the time does not + # advance during the program's lifetime; otherwise e2fsdroid would record + # different i_crtime values for inodes created in different real seconds. + env["FAKETIME"] = "@1970-01-01 00:00:00 x0" + existing = env.get("LD_PRELOAD", "") + env["LD_PRELOAD"] = f"{libfaketime}:{existing}" if existing else libfaketime + return env + + +@contextlib.contextmanager +def fakeroot_session(faked_sysv, libfakeroot, statefile=None, load_state=False): + """ + Spawn the hermetic faked-sysv daemon and yield (env, statefile). + + The env contains FAKEROOTKEY + LD_PRELOAD set so that subprocess calls + appear to run as root and have their stat/chmod/chown operations + intercepted and recorded by the daemon. If `statefile` is given the + daemon's state will be written there when the daemon exits; if + `load_state` is True it will additionally be pre-loaded from the same + file. This mirrors the `-s` and `-i` options of the `fakeroot` wrapper + but bypasses the wrapper itself, which performs host-specific library + lookups. + """ + faked_cmd = [faked_sysv] + faked_input = None + if statefile is not None: + faked_cmd += ["--save-file", statefile] + if load_state: + assert statefile is not None + faked_cmd += ["--load"] + with open(statefile, "rb") as f: + faked_input = f.read() + # faked-sysv prints "KEY:PID\n" then forks into the background. + proc = subprocess.run(faked_cmd, input=faked_input, capture_output=True, check=True) + key, pid_str = proc.stdout.decode().strip().split(":") + pid = int(pid_str) + env = dict(os.environ) + env["FAKEROOTKEY"] = key + env["LD_PRELOAD"] = libfakeroot + try: + yield env + # When asked to save state, force the daemon to flush by issuing a + # final stat() call against it (matches what the fakeroot wrapper's + # WAITINTRAP trap does to ensure --save-file is written). + if statefile is not None: + subprocess.run( + ["/bin/ls", "-l", "/"], + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + finally: + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + # Wait for the daemon to actually exit so --save-file is complete. + for _ in range(100): + try: + os.kill(pid, 0) + except ProcessLookupError: + break + time.sleep(0.05) def limit_file_contexts(file_contexts, base_path): @@ -81,7 +158,7 @@ def read_fakeroot_state(statefile): return entry_by_inode -def strip_files(fs_basedir, fakeroot_statefile, strip_paths): +def strip_files(faked_sysv, libfakeroot, fs_basedir, fakeroot_statefile, strip_paths): flattened_paths = [] for path in strip_paths: if path[0] == "/": @@ -99,14 +176,17 @@ def strip_files(fs_basedir, fakeroot_statefile, strip_paths): BATCH_SIZE = 100 for batch_start in range(0, len(flattened_paths), BATCH_SIZE): batch_end = min(batch_start + BATCH_SIZE, len(flattened_paths)) - subprocess.run( - ["fakeroot", "-s", fakeroot_statefile, "-i", fakeroot_statefile, "rm", "-rf"] - + flattened_paths[batch_start:batch_end], - check=True, - ) + with fakeroot_session(faked_sysv, libfakeroot, statefile=fakeroot_statefile, load_state=True) as env: + subprocess.run( + ["rm", "-rf"] + flattened_paths[batch_start:batch_end], + env=env, + check=True, + ) -def prepare_tree_from_tar(in_file, fakeroot_statefile, fs_basedir, dir_to_extract, extra_files): +def prepare_tree_from_tar( + faked_sysv, libfakeroot, in_file, fakeroot_statefile, fs_basedir, dir_to_extract, extra_files +): # We batch all commands together and run them under bash. This is significantly faster than invoking fakeroot # multiple times. commands = "set -euo pipefail\n" @@ -123,7 +203,8 @@ def prepare_tree_from_tar(in_file, fakeroot_statefile, fs_basedir, dir_to_extrac else: commands += f"""chown root:root "{fs_basedir}";\n""" - subprocess.run(["fakeroot", "-s", fakeroot_statefile, "bash"], input=commands.encode(), check=True) + with fakeroot_session(faked_sysv, libfakeroot, statefile=fakeroot_statefile) as env: + subprocess.run(["bash"], input=commands.encode(), env=env, check=True) def make_argparser(): @@ -170,6 +251,30 @@ def make_argparser(): type=str, required=True, ) + parser.add_argument( + "--libfaketime", + help="Path to our hermetic libfaketime.so.1, LD_PRELOADed in place of the faketime wrapper.", + type=str, + required=True, + ) + parser.add_argument( + "--faked-sysv", + help="Path to our hermetic faked-sysv daemon, used in place of the fakeroot wrapper.", + type=str, + required=True, + ) + parser.add_argument( + "--libfakeroot", + help="Path to our hermetic libfakeroot-sysv.so, LD_PRELOADed in place of the fakeroot wrapper.", + type=str, + required=True, + ) + parser.add_argument( + "--e2fsdroid", + help="Path to our hermetic e2fsdroid binary (pinned Android platform-tools).", + type=str, + required=True, + ) return parser @@ -206,23 +311,37 @@ def main(): # Prepare a filesystem tree that represents what will go into # the fs image. Wrap everything in fakeroot so permissions and # ownership will be preserved while unpacking (see below). - prepare_tree_from_tar(in_file, fakeroot_statefile, fs_basedir, limit_prefix, extra_files) - strip_files(fs_basedir, fakeroot_statefile, strip_paths) + prepare_tree_from_tar( + args.faked_sysv, + args.libfakeroot, + in_file, + fakeroot_statefile, + fs_basedir, + limit_prefix, + extra_files, + ) + strip_files( + args.faked_sysv, + args.libfakeroot, + fs_basedir, + fakeroot_statefile, + strip_paths, + ) # Now build the basic filesystem image. Wrap again in fakeroot # so correct permissions are read for all files etc. mke2fs_dir = os.path.dirname(args.mkfs_ext4) - mke2fs_env = { - "E2FSPROGS_FAKE_TIME": "0", - # Point mke2fs at our bundled libraries and config so its output does - # not depend on the host's installed e2fsprogs version. - "LD_LIBRARY_PATH": os.path.join(mke2fs_dir, "lib"), - "MKE2FS_CONFIG": os.path.join(mke2fs_dir, "mke2fs.conf"), - } + mke2fs_env = faketime_env( + args.libfaketime, + base_env={ + "E2FSPROGS_FAKE_TIME": "0", + # Point mke2fs at our bundled libraries and config so its output does + # not depend on the host's installed e2fsprogs version. + "LD_LIBRARY_PATH": os.path.join(mke2fs_dir, "lib"), + "MKE2FS_CONFIG": os.path.join(mke2fs_dir, "mke2fs.conf"), + }, + ) mke2fs_args = [ - "faketime", - "-f", - "1970-1-1 0:0:0", args.mkfs_ext4, "-E", "hash_seed=c61251eb-100b-48fe-b089-57dea7368612", @@ -251,13 +370,7 @@ def main(): subprocess.run(diroid_args, check=True) e2fsdroid_args = [ - "faketime", - "-f", - "1970-1-1 0:0:0", - "fakeroot", - "-i", - fakeroot_statefile, - "e2fsdroid", + args.e2fsdroid, "-e", "-a", "/", @@ -268,7 +381,10 @@ def main(): if file_contexts_file: e2fsdroid_args += ["-S", file_contexts_file] e2fsdroid_args += [image_file] - subprocess.run(e2fsdroid_args, check=True, env={"E2FSPROGS_FAKE_TIME": "0"}) + with fakeroot_session(args.faked_sysv, args.libfakeroot, statefile=fakeroot_statefile, load_state=True) as fr_env: + e2fsdroid_env = faketime_env(args.libfaketime, base_env=fr_env) + e2fsdroid_env["E2FSPROGS_FAKE_TIME"] = "0" + subprocess.run(e2fsdroid_args, check=True, env=e2fsdroid_env) # We use our tool, dflate, to quickly create a sparse, deterministic, tar. # If dflate is ever misbehaving, it can be replaced with: diff --git a/toolchains/sysimage/toolchain.bzl b/toolchains/sysimage/toolchain.bzl index a3bdb67e52e7..ba09e73a1f64 100644 --- a/toolchains/sysimage/toolchain.bzl +++ b/toolchains/sysimage/toolchain.bzl @@ -345,9 +345,20 @@ def _ext4_image_impl(ctx): ctx.executable._dflate.path, "--mkfs-ext4", ctx.file._mkfs_ext4.path, + "--libfaketime", + ctx.file._libfaketime.path, + "--faked-sysv", + ctx.file._faked_sysv.path, + "--libfakeroot", + ctx.file._libfakeroot.path, + "--e2fsdroid", + ctx.file._e2fsdroid.path, ]) inputs.extend(ctx.files._mkfs_ext4_runfiles) + inputs.extend(ctx.files._libfaketime_runfiles) + inputs.extend(ctx.files._fakeroot_runfiles) + inputs.append(ctx.file._e2fsdroid) if ctx.attr.file_contexts: args.extend(["-S", ctx.files.file_contexts[0].path]) @@ -415,6 +426,30 @@ ext4_image = _icos_build_rule( default = "//toolchains/sysimage:mkfs_ext4_runfiles", allow_files = True, ), + "_libfaketime": attr.label( + default = "//toolchains/sysimage:faketime.d/libfaketime.so.1", + allow_single_file = True, + ), + "_libfaketime_runfiles": attr.label( + default = "//toolchains/sysimage:libfaketime_runfiles", + allow_files = True, + ), + "_faked_sysv": attr.label( + default = "//toolchains/sysimage:fakeroot.d/faked-sysv", + allow_single_file = True, + ), + "_libfakeroot": attr.label( + default = "//toolchains/sysimage:fakeroot.d/libfakeroot-sysv.so", + allow_single_file = True, + ), + "_fakeroot_runfiles": attr.label( + default = "//toolchains/sysimage:fakeroot_runfiles", + allow_files = True, + ), + "_e2fsdroid": attr.label( + default = "//toolchains/sysimage:e2fsdroid", + allow_single_file = True, + ), }, )