Skip to content

AMI: Clean up messy apt handling #2351

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
mmlb wants to merge 2 commits into
base: develop
Choose a base branch
from
Draft

AMI: Clean up messy apt handling #2351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
324a5c9
packer: Simplify apt mirror fallback
mmlb Aug 14, 2026
0c3fa9f
packer: Minimize and improve apt-get calls
mmlb Jul 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions amazon-amd64-nix.pkr.hcl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,11 +170,6 @@ source "amazon-ebssurrogate" "source" {
build {
sources = ["source.amazon-ebssurrogate.source"]

provisioner "file" {
source = "ebssurrogate/files/sources.cfg"
destination = "/tmp/sources.list"
}

provisioner "file" {
source = "ebssurrogate/files/ebsnvme-id"
destination = "/tmp/ebsnvme-id"
Expand Down
5 changes: 0 additions & 5 deletions amazon-arm64-nix.pkr.hcl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,11 +170,6 @@ source "amazon-ebssurrogate" "source" {
build {
sources = ["source.amazon-ebssurrogate.source"]

provisioner "file" {
source = "ebssurrogate/files/sources-arm64.cfg"
destination = "/tmp/sources.list"
}

provisioner "file" {
source = "ebssurrogate/files/ebsnvme-id"
destination = "/tmp/ebsnvme-id"
Expand Down
11 changes: 0 additions & 11 deletions ansible/playbook.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,17 +141,6 @@
when: not stage2
import_tasks: tasks/setup-fail2ban.yml

# Install EC2 instance connect
# Only for AWS images
- name: install EC2 instance connect
when: not qemu
become: yes
apt:
pkg:
- ec2-instance-connect
tags:
- aws-only

# Install this at the end to prevent it from kicking in during the apt process, causing conflicts
- name: Install security tools
become: yes
Expand Down
10 changes: 0 additions & 10 deletions ebssurrogate/files/sources-arm64.cfg
View file
Open in desktop

This file was deleted.

10 changes: 0 additions & 10 deletions ebssurrogate/files/sources.cfg
View file
Open in desktop

This file was deleted.

265 changes: 53 additions & 212 deletions ebssurrogate/scripts/chroot-bootstrap-nix.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,238 +8,88 @@ set -o errexit
set -o pipefail
set -o xtrace

# Switch to a different mirror
function switch_mirror {
local new_mirror=$1
local sources_file=/etc/apt/sources.list

echo "Switching to mirror: $new_mirror"
if [[ $ARCH == amd64 ]]; then
sed -i "s|http://[^/]*/ubuntu/|http://$new_mirror/ubuntu/|g" "$sources_file"
else
sed -i "s|http://[^/]*/ubuntu-ports/|http://$new_mirror/ubuntu-ports/|g" "$sources_file"
fi

# Show what we're using
echo "Current sources.list configuration:"
grep -E '^deb ' "$sources_file" | head -3
}

# Get list of mirrors to try
function get_mirror_list {
local sources_file=/etc/apt/sources.list
local -a mirrors=()

# Priority order:
# 1. Country-specific mirror (most reliable)
# 2. Regional CDN (can be inconsistent)
# 3. Global fallback

local current_region
if [[ $ARCH == amd64 ]]; then
current_region=$(grep -oP '(?<=http://)[^.]+(?=\.ec2\.archive\.ubuntu\.com)' "$sources_file" | head -1 || echo "")

if [[ -n $current_region ]]; then
mirrors+=("$current_region.ec2.archive.ubuntu.com")
fi

mirrors+=("archive.ubuntu.com")
else
current_region=$(grep -oP '(?<=http://)[^.]+(?=\.clouds\.ports\.ubuntu\.com)' "$sources_file" | head -1 || echo "")

# Singapore country mirror for ap-southeast-1
if [[ $current_region == "ap-southeast-1" ]]; then
mirrors+=("sg.ports.ubuntu.com")
fi

if [[ -n $current_region ]]; then
mirrors+=("$current_region.clouds.ports.ubuntu.com")
fi
mirrors+=("ports.ubuntu.com")
fi

echo "${mirrors[@]}"
}

# Mirror fallback function for resilient apt-get update
# The following 2 functions don'treally do much since we are now using deb822 formatted sources with fallbacks in the URIs.
# This means apt-get handles fallback on its own, much better and cleaner than we are doing.
# Leaving the functions as is for now to make the diff smaller, soon will go away.
function apt_update_with_fallback {
local sources_file=/etc/apt/sources.list
local -a mirror_list
readarray mirror_list < <(get_mirror_list)
local attempt=1
local max_attempts=${#mirror_list[@]}

for mirror in "${mirror_list[@]}"; do
echo "========================================="
echo "Attempting apt-get update with mirror: $mirror"
echo "Attempt $attempt of $max_attempts"
echo "========================================="

switch_mirror "$mirror"

# Attempt update with timeout (5 minutes)
if timeout 300 apt-get "${APT_OPTIONS[@]}" update 2>&1; then
echo "========================================="
echo "✓ Successfully updated apt cache using mirror: $mirror"
echo "========================================="
return 0
else
local ret=$?
echo "========================================="
echo "✗ Failed to update using mirror: $mirror"
echo "Exit code: $ret"
echo "========================================="

# Clean partial downloads
apt-get clean
rm -rf /var/lib/apt/lists/*

# Exponential backoff before next attempt
if [[ $attempt -lt $max_attempts ]]; then
local sleep_time=$((attempt * 5))
echo "Waiting $sleep_time seconds before trying next mirror..."
sleep $sleep_time
fi
fi

attempt=$((attempt + 1))
done

echo "========================================="
echo "ERROR: All mirror tiers failed after $max_attempts attempts"
echo "========================================="
return 1
timeout 300 apt-get "${APT_OPTIONS[@]}" update 2>&1
}

# Wrapper for apt-get install with mirror fallback on 404 errors
function apt_install_with_fallback {
local -a mirror_list
readarray mirror_list < <(get_mirror_list)
local attempt=1
local max_attempts=${#mirror_list[@]}

for mirror in "${mirror_list[@]}"; do
echo "========================================="
echo "Attempting apt-get install with mirror: $mirror"
echo "Attempt $attempt of $max_attempts"
echo "========================================="

switch_mirror "$mirror"

# Re-run apt-get update to get package lists from new mirror
if ! timeout 300 apt-get "${APT_OPTIONS[@]}" update 2>&1; then
echo "Warning: apt-get update failed for mirror $mirror, trying next..."
attempt=$((attempt + 1))
continue
fi

# Run apt-get install directly (no output capture to avoid buffering/timeout issues)
if apt-get "$@"; then
echo "========================================="
echo "✓ Successfully installed packages using mirror: ${mirror}"
echo "========================================="
return 0
else
local ret=$?
# On failure, check if it's a mirror issue worth retrying
echo "========================================="
echo "✗ apt-get failed with exit code: $ret"
echo "========================================="
fi

# Clean apt cache before potential retry
apt-get clean

if ((attempt < max_attempts)); then
local sleep_time=$((attempt * 5))
echo "Waiting $sleep_time seconds before trying next mirror..."
sleep $sleep_time
fi

attempt=$((attempt + 1))
done

echo "========================================="
echo "ERROR: All mirror tiers failed for apt-get install after $max_attempts attempts"
echo "========================================="
return 1
apt-get "$@"
}

function update_install_packages {
# Update APT with new sources (using fallback mechanism)
cat /etc/apt/sources.list
tail -n+1 /etc/apt/sources.list /etc/apt/sources.list.d/*
if ! apt_update_with_fallback; then
echo "FATAL: Failed to update package lists with any mirror tier"
exit 1
fi
apt-get "${APT_OPTIONS[@]}" --yes dist-upgrade

# Do not configure grub during package install
local packages=(
e2fsprogs
initramfs-tools
linux-aws
)
# Install various packages needed for a booting system (with mirror fallback)
if [[ $ARCH == amd64 ]]; then
# Do not configure grub during package install
echo 'grub-pc grub-pc/install_devices_empty select true' | debconf-set-selections
echo 'grub-pc grub-pc/install_devices select' | debconf-set-selections
# Install various packages needed for a booting system (with mirror fallback)
if ! apt_install_with_fallback install -y linux-aws grub-pc e2fsprogs; then
echo "FATAL: Failed to install boot packages"
exit 1
fi
packages+=(
grub-pc
)
else
if ! apt_install_with_fallback install -y e2fsprogs; then
echo "FATAL: Failed to install e2fsprogs"
exit 1
fi
packages+=(
cloud-guest-utils
dosfstools
efibootmgr
fdisk
grub-efi-arm64
)
fi
# Install standard packages (with mirror fallback)
# Note: ec2-hibinit-agent, ec2-instance-connect, hibagent moved to stage 2
# because their post-install scripts try to access EC2 metadata service
# which doesn't work in a chroot and causes long hangs
if ! apt_install_with_fallback install -y \
bzip2 \
sudo \
wget \
cloud-init \
acpid \
ncurses-term \
ssh-import-id; then

packages+=(
acpid
apparmor
apparmor-utils
auditd
bzip2
cloud-init
ec2-hibinit-agent
ec2-instance-connect
hibagent
ncurses-term
ssh-import-id
sudo
wget
)
if ! apt_install_with_fallback install -y "${packages[@]}"; then
echo "FATAL: Failed to install standard packages"
exit 1
fi

# apt upgrade
apt-get upgrade -y

# Install OpenSSH and other packages
add-apt-repository --yes universe
if ! apt_update_with_fallback; then
echo "FATAL: Failed to update package lists after adding universe repository"
exit 1
fi
if ! apt_install_with_fallback install -y --no-install-recommends \
openssh-server \
git \
ufw \
cron \
logrotate \
fail2ban \
locales \
at \
less \
python3-systemd; then
packages=(
at
cron
fail2ban
git
less
locales
logrotate
openssh-server
python3-systemd
ufw
)
if ! apt_install_with_fallback install -y --no-install-recommends "${packages[@]}"; then
echo "FATAL: Failed to install universe packages"
exit 1
fi

if [[ $ARCH == arm64 ]]; then
if ! apt_install_with_fallback "${APT_OPTIONS[@]}" --yes install linux-aws initramfs-tools dosfstools; then
echo "FATAL: Failed to install arm64 boot packages"
exit 1
fi
else
if ! apt_install_with_fallback "${APT_OPTIONS[@]}" --yes install initramfs-tools; then
echo "FATAL: Failed to install amd64 boot packages"
exit 1
fi
fi
}

function setup_locale {
Expand Down Expand Up @@ -268,11 +118,6 @@ function setup_postgesql_env {
}

function setup_apparmor {
if ! apt_install_with_fallback install -y apparmor apparmor-utils auditd; then
echo "FATAL: Failed to install apparmor packages"
exit 1
fi

# Copy apparmor profiles
cp -rv /tmp/apparmor_profiles/* /etc/apparmor.d/
}
Expand All @@ -290,10 +135,6 @@ function setup_grub {
EOF

if [[ $ARCH == arm64 ]]; then
if ! apt_install_with_fallback "${APT_OPTIONS[@]}" --yes install cloud-guest-utils fdisk grub-efi-arm64 efibootmgr; then
echo "FATAL: Failed to install grub packages for arm64"
exit 1
fi
rm -rf /etc/grub.d/30_os-prober
sleep 1
fi
Expand Down
Loading
Loading