fix(docker-in-docker): disable containerd erofs snapshotter to fix dockerd startup on Debian 12 / Ubuntu 22.04/ Ubuntu 24.04 (closes #1642)

.github/workflows/test-pr-arm64.yaml

@@ -8,6 +8,8 @@ on:
     paths:
       - "src/powershell/**"
       - "test/powershell/**"
+      - "src/docker-in-docker/**"
+      - "test/docker-in-docker/**"      
 
 jobs:
   detect-changes:
@@ -23,6 +25,7 @@ jobs:
           #   <feature-name>: ./**/<feature-name>/**
           filters: |
             powershell: ./**/powershell/**
+            docker-in-docker: ./**/docker-in-docker/**
 
   test:
     needs: [detect-changes]
@@ -42,9 +45,15 @@ jobs:
             "mcr.microsoft.com/devcontainers/base:debian",
             "mcr.microsoft.com/devcontainers/base:noble"
           ]
+        exclude:
+          - features: docker-in-docker
+            baseImage: mcr.microsoft.com/devcontainers/base:debian           
     steps:
       - uses: actions/checkout@v6
 
+      - name: "Load erofs module and verify"
+        run: sudo modprobe erofs && grep erofs /proc/filesystems
+
       - name: "Install latest devcontainer CLI"
         run: npm install -g @devcontainers/cli
 

src/docker-in-docker/NOTES.md

@@ -4,7 +4,7 @@ This docker-in-docker Dev Container Feature is roughly based on the [official do
 * As the name implies, the Feature is expected to work when the host is running Docker (or the OSS Moby container engine it is built on). It may be possible to get running in other container engines, but it has not been tested with them.
 * The host and the container must be running on the same chip architecture. You will not be able to use it with an emulated x86 image with Docker Desktop on an Apple Silicon Mac, like in this example:
   ```
-  FROM --platform=linux/amd64 mcr.microsoft.com/devcontainers/typescript-node:16
+  FROM --platform=linux/amd64 mcr.microsoft.com/devcontainers/typescript-node:24
   ```
   See [Issue #219](https://github.com/devcontainers/features/issues/219) for more details.
 

src/docker-in-docker/README.md

@@ -7,7 +7,7 @@ Create child containers *inside* a container, independent from the host's docker
 
 ```json
 "features": {
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {}
+    "ghcr.io/devcontainers/features/docker-in-docker:3": {}
 }
 ```
 

src/docker-in-docker/devcontainer-feature.json

@@ -1,6 +1,6 @@
 {
   "id": "docker-in-docker",
-  "version": "2.17.0",
+  "version": "3.0.0",
   "name": "Docker (Docker-in-Docker)",
   "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker",
   "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.",

src/docker-in-docker/install.sh

@@ -287,10 +287,13 @@ else
 fi
 
 # Install base dependencies
+# Note: erofs-utils provides mkfs.erofs, required by containerd >= 2.3.x snapshotter
+# to avoid "failed to check mkfs.erofs availability" errors at dockerd startup
+# (see https://github.com/devcontainers/features/issues/1642).
 base_packages="curl ca-certificates pigz iptables gnupg2 wget jq"
 case ${ADJUSTED_ID} in
     debian)
-        check_packages apt-transport-https $base_packages dirmngr
+        check_packages apt-transport-https $base_packages dirmngr erofs-utils
         ;;
     rhel)
         check_packages $base_packages tar gawk shadow-utils policycoreutils  procps-ng systemd-libs systemd-devel
@@ -731,11 +734,21 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then
             err "Docker compose v1 is unavailable for 'bookworm' on Arm64. Kindly switch to use v2"
             exit 1
         else
-            # Use pip to get a version that runs on this architecture
+            # Use pip (inside an isolated venv) to get a version that runs on this architecture.
+            # A dedicated venv avoids PEP 668 "externally-managed-environment" errors on newer
+            # distros (Debian trixie, Ubuntu noble, etc.) and guarantees we do not modify or
+            # shadow the distro-managed system Python site-packages.
             check_packages python3-minimal python3-pip libffi-dev python3-venv
-            echo "(*) Installing docker compose v1 via pip..."
-            export PYTHONUSERBASE=/usr/local
-            pip3 install --disable-pip-version-check --no-cache-dir --user "Cython<3.0" pyyaml wheel docker-compose --no-build-isolation
+            echo "(*) Installing docker compose v1 via pip into an isolated virtualenv..."
+
+            compose_v1_venv="/usr/local/share/docker-compose-v1-venv"
+            python3 -m venv "${compose_v1_venv}"
+            "${compose_v1_venv}/bin/pip" install --disable-pip-version-check --no-cache-dir --upgrade pip setuptools wheel
+            "${compose_v1_venv}/bin/pip" install --disable-pip-version-check --no-cache-dir "Cython<3.0" pyyaml docker-compose --no-build-isolation
+
+            # Expose the venv's docker-compose entrypoint on PATH at the expected location.
+            ln -sf "${compose_v1_venv}/bin/docker-compose" "${docker_compose_path}"
+            chmod +x "${docker_compose_path}"
         fi
     else
         compose_version=${DOCKER_DASH_COMPOSE_VERSION#v}
@@ -876,6 +889,58 @@ if [ "$DISABLE_IP6_TABLES" == true ]; then
     fi
 fi
 
+# Workaround for https://github.com/devcontainers/features/issues/1642
+# containerd >= 2.3 ships an erofs snapshotter that requires mkfs.erofs >= 1.7.
+# Older distros (Debian 12, Ubuntu 22.04) ship erofs-utils 1.4/1.5, so when the
+# host kernel exposes the 'erofs' filesystem the snapshotter fails to
+# initialize and dockerd times out waiting for containerd. Disable the plugin
+# via the top-level `disabled_plugins` list so containerd always uses
+# overlayfs, regardless of distro / mkfs.erofs version.
+mkdir -p /etc/containerd
+if [ ! -s /etc/containerd/config.toml ]; then
+    if command -v containerd >/dev/null 2>&1; then
+        containerd config default > /etc/containerd/config.toml 2>/dev/null || : > /etc/containerd/config.toml
+    elif [ -x /usr/sbin/containerd ]; then
+        /usr/sbin/containerd config default > /etc/containerd/config.toml 2>/dev/null || : > /etc/containerd/config.toml
+    elif [ -x /usr/bin/containerd ]; then
+        /usr/bin/containerd config default > /etc/containerd/config.toml 2>/dev/null || : > /etc/containerd/config.toml
+    else
+        : > /etc/containerd/config.toml
+    fi
+fi
+
+EROFS_PLUGIN_URI='io.containerd.snapshotter.v1.erofs'
+EROFS_MARKER='# devcontainers-features:disable-erofs'
+
+if ! grep -qF "${EROFS_MARKER}" /etc/containerd/config.toml; then
+    if grep -qE '^[[:space:]]*disabled_plugins[[:space:]]*=' /etc/containerd/config.toml; then
+        # Add erofs URI to the existing top-level disabled_plugins list.
+        # Branches are mutually exclusive and guarded so we never produce
+        # duplicate entries on re-runs.
+        if grep -qE '^[[:space:]]*disabled_plugins[[:space:]]*=[[:space:]]*\[[[:space:]]*\]' /etc/containerd/config.toml; then
+            # disabled_plugins = []  ->  insert URI as the only entry.
+            sed -i -E \
+              "s|^([[:space:]]*disabled_plugins[[:space:]]*=[[:space:]]*\[)[[:space:]]*\]|\1\"${EROFS_PLUGIN_URI}\"]|" \
+              /etc/containerd/config.toml
+        elif ! grep -qF "\"${EROFS_PLUGIN_URI}\"" /etc/containerd/config.toml; then
+            # disabled_plugins = ["existing", ...]  ->  append URI.
+            sed -i -E \
+              "s|^([[:space:]]*disabled_plugins[[:space:]]*=[[:space:]]*\[)([^]]*[^],[:space:]])[[:space:]]*\]|\1\2, \"${EROFS_PLUGIN_URI}\"]|" \
+              /etc/containerd/config.toml
+        fi
+    else
+        # No disabled_plugins key in the config: prepend one.
+        tmp_cfg="$(mktemp)"
+        {
+            printf 'disabled_plugins = ["%s"]\n\n' "${EROFS_PLUGIN_URI}"
+            cat /etc/containerd/config.toml
+        } > "${tmp_cfg}"
+        mv "${tmp_cfg}" /etc/containerd/config.toml
+    fi
+    # Idempotency marker
+    printf '\n%s\n' "${EROFS_MARKER}" >> /etc/containerd/config.toml
+fi
+
 if [ ! -d /usr/local/share ]; then
     mkdir -p /usr/local/share
 fi
@@ -974,8 +1039,42 @@ dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAU
         DEFAULT_ADDRESS_POOL="--default-address-pool $DOCKER_DEFAULT_ADDRESS_POOL"
     fi
 
+
+    # Start our own containerd so it picks up /etc/containerd/config.toml
+    # (notably the disabled_plugins entry for the erofs snapshotter, see
+    # https://github.com/devcontainers/features/issues/1642). dockerd's
+    # built-in containerd child uses an auto-generated config that ignores
+    # /etc/containerd/config.toml, so we must run containerd ourselves and
+    # point dockerd at it via --containerd.
+    CONTAINERD_SOCK="/run/containerd/containerd.sock"
+    CONTAINERD_BIN=""
+    for candidate in /usr/local/bin/containerd /usr/bin/containerd /usr/sbin/containerd; do
+        if [ -x "$candidate" ]; then
+            CONTAINERD_BIN="$candidate"
+            break
+        fi
+    done
+    DOCKERD_CONTAINERD_ARG=""
+    if [ -n "$CONTAINERD_BIN" ] && [ -f /etc/containerd/config.toml ]; then
+        mkdir -p /run/containerd
+        if ! pgrep -x containerd > /dev/null 2>&1; then
+            ( "$CONTAINERD_BIN" --config /etc/containerd/config.toml > /tmp/containerd.log 2>&1 ) &
+        fi
+        # Wait up to ~5s for the socket to appear
+        i=0
+        while [ $i -lt 50 ] && [ ! -S "$CONTAINERD_SOCK" ]; do
+            sleep 0.1
+            i=$((i + 1))
+        done
+        if [ -S "$CONTAINERD_SOCK" ]; then
+            DOCKERD_CONTAINERD_ARG="--containerd $CONTAINERD_SOCK"
+        else
+            echo "(*) containerd socket not ready; letting dockerd spawn its own containerd."
+        fi
+    fi
+
     # Start docker/moby engine
-    ( dockerd $CUSTOMDNS $DEFAULT_ADDRESS_POOL $DOCKER_DEFAULT_IP6_TABLES > /tmp/dockerd.log 2>&1 ) &
+    ( dockerd $DOCKERD_CONTAINERD_ARG $CUSTOMDNS $DEFAULT_ADDRESS_POOL $DOCKER_DEFAULT_IP6_TABLES > /tmp/dockerd.log 2>&1 ) &
 INNEREOF
 )"