From 63485bea34e5a824b230e61e89db327b45cfcca4 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 21 Aug 2026 09:07:21 +1000 Subject: [PATCH 1/3] CI: add timeout and retry to apt-get install steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bare `sudo apt-get install -y` with no `timeout-minutes` and no retry let a stalled Ubuntu mirror hang CI for hours — two 358-minute hangs on 2026-08-18 across two edition repos ten minutes apart, and four more hangs on 2026-08-19 each cleared only by cancel-and-rerun. Every apt-installing step now carries a step-level `timeout-minutes` and runs up to three attempts with a per-attempt `timeout`, so a stalled fetch fails fast and a transient mirror outage is ridden out rather than burning the job. Package lists are unchanged. Tracked in QuantEcon/project-translation#43. --- .github/workflows/cache.yml | 21 +++++++++------------ .github/workflows/ci.yml | 31 ++++++++++++++++++------------- .github/workflows/publish.yml | 20 +++++++++----------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index d7d4f35a..0012729f 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -23,19 +23,16 @@ jobs: run: | sudo apt-get -qq update && sudo apt-get install -y graphviz - name: Install latex dependencies + timeout-minutes: 30 run: | - sudo apt-get -qq update - sudo apt-get install -y \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-xetex \ - latexmk \ - xindy \ - dvipng \ - cm-super \ - latex-cjk-all + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super latex-cjk-all" + for attempt in 1 2 3; do + timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Build HTML shell: bash -l {0} run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6d2481b..9892f822 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,22 +18,27 @@ jobs: environment-file: environment.yml activate-environment: quantecon - name: Graphics Support #TODO: Review if graphviz is needed + timeout-minutes: 10 run: | - sudo apt-get -qq update && sudo apt-get install -y graphviz + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="graphviz" + for attempt in 1 2 3; do + timeout 3m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Install latex dependencies + timeout-minutes: 30 run: | - sudo apt-get -qq update - sudo apt-get install -y \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-xetex \ - latexmk \ - xindy \ - dvipng \ - cm-super \ - latex-cjk-all + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super latex-cjk-all" + for attempt in 1 2 3; do + timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Display Conda Environment Versions shell: bash -l {0} run: conda list diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5685ef15..bae71eb8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,18 +22,16 @@ jobs: environment-file: environment.yml activate-environment: quantecon - name: Install latex dependencies + timeout-minutes: 30 run: | - sudo apt-get -qq update - sudo apt-get install -y \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-xetex \ - latexmk \ - xindy \ - dvipng \ - cm-super + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" + for attempt in 1 2 3; do + timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Display Conda Environment Versions shell: bash -l {0} run: conda list From 949fc01d750aa5fe28b621a8d3eecde3406c3537 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 21 Aug 2026 09:40:32 +1000 Subject: [PATCH 2/3] CI: harden the cache.yml graphviz step too; two attempts with wider per-attempt timeouts (4m/14m) Addresses Copilot review on QuantEcon/lecture-python.zh-cn#263: the cache.yml graphviz step was missed by a name match, and three 3m/9m attempts could never let a slow-but-progressing install finish. Now two attempts at 4m (graphviz) / 14m (texlive) inside the unchanged 10/30-minute step budgets. --- .github/workflows/cache.yml | 16 ++++++++++++---- .github/workflows/ci.yml | 12 ++++++------ .github/workflows/publish.yml | 6 +++--- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 0012729f..4dec663c 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -20,17 +20,25 @@ jobs: environment-file: environment.yml activate-environment: quantecon - name: graphviz Support # TODO: required? + timeout-minutes: 10 run: | - sudo apt-get -qq update && sudo apt-get install -y graphviz + # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # hangs this step for hours (two 358-minute hangs on 2026-08-18). + pkgs="graphviz" + for attempt in 1 2; do + timeout 4m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi + echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 + done - name: Install latex dependencies timeout-minutes: 30 run: | # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super latex-cjk-all" - for attempt in 1 2 3; do - timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break - if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + for attempt in 1 2; do + timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Build HTML diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9892f822..12239160 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,9 +23,9 @@ jobs: # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="graphviz" - for attempt in 1 2 3; do - timeout 3m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break - if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + for attempt in 1 2; do + timeout 4m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Install latex dependencies @@ -34,9 +34,9 @@ jobs: # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super latex-cjk-all" - for attempt in 1 2 3; do - timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break - if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + for attempt in 1 2; do + timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Display Conda Environment Versions diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bae71eb8..53f008a9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -27,9 +27,9 @@ jobs: # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" - for attempt in 1 2 3; do - timeout 9m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break - if [ "$attempt" = 3 ]; then echo "apt-get failed after 3 attempts"; exit 1; fi + for attempt in 1 2; do + timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Display Conda Environment Versions From bcb15602a022cfb1839fef204e7bebd2ad84d87e Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 21 Aug 2026 09:43:50 +1000 Subject: [PATCH 3/3] CI: run timeout inside sudo, per apt command, so the kill reaches apt-get Addresses Copilot review on QuantEcon/lecture-intro.zh-cn#301: with timeout wrapping sudo, a timed-out attempt terminated sudo and the bash -c shell but could orphan apt-get holding the dpkg lock, making the retry block instead of retry. timeout is now the direct parent of each apt-get (inside sudo, -k 30s escalation): update 1m/2m, install 3m/12m for graphviz/texlive. Step budgets unchanged. --- .github/workflows/cache.yml | 10 ++++++---- .github/workflows/ci.yml | 10 ++++++---- .github/workflows/publish.yml | 5 +++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 4dec663c..8b225720 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -22,22 +22,24 @@ jobs: - name: graphviz Support # TODO: required? timeout-minutes: 10 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="graphviz" for attempt in 1 2; do - timeout 4m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 1m apt-get -qq update && sudo timeout -k 30s 3m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Install latex dependencies timeout-minutes: 30 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super latex-cjk-all" for attempt in 1 2; do - timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 2m apt-get -qq update && sudo timeout -k 30s 12m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12239160..7d3697cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,22 +20,24 @@ jobs: - name: Graphics Support #TODO: Review if graphviz is needed timeout-minutes: 10 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="graphviz" for attempt in 1 2; do - timeout 4m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 1m apt-get -qq update && sudo timeout -k 30s 3m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done - name: Install latex dependencies timeout-minutes: 30 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super latex-cjk-all" for attempt in 1 2; do - timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 2m apt-get -qq update && sudo timeout -k 30s 12m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 53f008a9..2956eb93 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,11 +24,12 @@ jobs: - name: Install latex dependencies timeout-minutes: 30 run: | - # Retried with a per-attempt timeout: a stalled Ubuntu mirror otherwise + # Retried with per-command timeouts (timeout inside sudo, so the kill reaches + # apt-get and the dpkg lock is released): a stalled Ubuntu mirror otherwise # hangs this step for hours (two 358-minute hangs on 2026-08-18). pkgs="texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-fonts-extra texlive-xetex latexmk xindy dvipng cm-super" for attempt in 1 2; do - timeout 14m sudo bash -c "apt-get -qq update && apt-get install -y $pkgs" && break + sudo timeout -k 30s 2m apt-get -qq update && sudo timeout -k 30s 12m apt-get install -y $pkgs && break if [ "$attempt" = 2 ]; then echo "apt-get failed after 2 attempts"; exit 1; fi echo "apt-get attempt $attempt failed; retrying in 30s"; sleep 30 done