fix(ci): kill the remaining transient-failure classes in workflows + Dockerfile

From the workflow reliability audit:
- tests.yml: duration-cache restore had NO restore-keys while saves use
  run_id-suffixed keys — the cache never matched once, so LPT slicing
  always ran blind and unbalanced slices pushed heavy files toward the
  per-file timeout. One-line restore-keys fixes slice balancing.
- Label gates (lint ci-reviewed, supply-chain mcp-catalog-reviewed):
  'gh pr view || true' turned an API blip into 'label absent' → false
  BLOCKING failure. Now 3x retry, and API failure is reported as an API
  failure instead of a missing label.
- detect-changes action: compare API retried before failing open (was
  silently running all lanes on any blip).
- uv-lockfile-check: 'uv lock --check' resolves against PyPI — retried
  so registry blips don't read as 'lockfile stale'.
- docker.yml merge job: imagetools create retried (Docker Hub eventual
  consistency on just-pushed digests).
- Dockerfile: apt-get Acquire::Retries=3; s6-overlay ADDs converted to
  curl --retry 3 (ADD cannot retry; checksums still enforced); npm
  --fetch-retries=5; playwright chromium fetch retried 3x.
- Advisory artifact uploads (per-slice durations, ci-timings report)
  get continue-on-error so an artifact-service blip can't fail a green
  test slice.

.github/actions/detect-changes/action.yml

@@ -57,10 +57,26 @@ runs:
           # event payload instead of the "current PR files" endpoint. The SHAs
           # are frozen at trigger time, so the file list is deterministic even
           # if the PR receives a new push between trigger and detect.
-          CHANGED="$(gh api \
-            --paginate \
-            "repos/${REPO}/compare/${BASE_SHA}...${HEAD_SHA}" \
-            --jq '.files[].filename' || true)"
+          #
+          # Retried: a rate-limit blip or eventual-consistency 404 on a
+          # freshly-pushed HEAD would otherwise silently fall open (all lanes
+          # run — safe, but wasteful and it masks the API failure).
+          CHANGED=""
+          for i in 1 2 3; do
+            if CHANGED="$(gh api \
+              --paginate \
+              "repos/${REPO}/compare/${BASE_SHA}...${HEAD_SHA}" \
+              --jq '.files[].filename')"; then
+              break
+            fi
+            if [ "$i" = 3 ]; then
+              echo "::warning::compare API failed after 3 attempts — failing open (all lanes run)"
+              CHANGED=""
+              break
+            fi
+            echo "::warning::compare API failed (attempt $i); retrying in 10s"
+            sleep 10
+          done
         fi
 
         echo "Changed files:"

.github/workflows/ci.yml

@@ -220,6 +220,8 @@ jobs:
             --summary-out ci-timings-summary.md
 
       - name: Upload HTML report
+        # Advisory report — artifact-service blips must not fail the job.
+        continue-on-error: true
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         id: ci-timings-artifact
         with:

.github/workflows/docker.yml

@@ -189,15 +189,23 @@ jobs:
             args+=("${IMAGE_NAME}@sha256:${digest_file}")
           done
           if [ "${{ github.event_name }}" = "release" ]; then
-            docker buildx imagetools create \
-              -t "${IMAGE_NAME}:${RELEASE_TAG}" \
-              "${args[@]}"
+            tags=(-t "${IMAGE_NAME}:${RELEASE_TAG}")
           else
-            docker buildx imagetools create \
-              -t "${IMAGE_NAME}:main" \
-              -t "${IMAGE_NAME}:latest" \
-              "${args[@]}"
+            tags=(-t "${IMAGE_NAME}:main" -t "${IMAGE_NAME}:latest")
           fi
+          # Retry: Docker Hub API + just-pushed digest eventual consistency
+          # can transiently fail the create; the operation is idempotent.
+          for i in 1 2 3; do
+            if docker buildx imagetools create "${tags[@]}" "${args[@]}"; then
+              break
+            fi
+            if [ "$i" = 3 ]; then
+              echo "::error::imagetools create failed after 3 attempts"
+              exit 1
+            fi
+            echo "::warning::imagetools create failed (attempt $i); retrying in 20s"
+            sleep 20
+          done
 
       - name: Inspect image
         env:

.github/workflows/lint.yml

@@ -184,7 +184,21 @@ jobs:
         run: |
           set -euo pipefail
           PR="${{ github.event.pull_request.number }}"
-          LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name' || true)
+          # Retry the label fetch: a transient API blip must not read as
+          # "label absent" (which hard-fails the job below on a PR that
+          # actually carries the label).
+          LABELS=""
+          for i in 1 2 3; do
+            if LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name'); then
+              break
+            fi
+            if [ "$i" = 3 ]; then
+              echo "::error::Could not fetch PR labels after 3 attempts (GitHub API failure — re-run this job)."
+              exit 1
+            fi
+            echo "::warning::gh pr view failed (attempt $i); retrying in 10s"
+            sleep 10
+          done
           if echo "$LABELS" | grep -Fxq 'ci-reviewed'; then
             echo "reviewed=true" >> "$GITHUB_OUTPUT"
             echo "ci-reviewed label present."

.github/workflows/supply-chain-audit.yml

@@ -244,7 +244,20 @@ jobs:
         run: |
           set -euo pipefail
           PR="${{ github.event.pull_request.number }}"
-          LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name' || true)
+          # Retry the label fetch so an API blip doesn't read as "label
+          # absent" and falsely block the PR.
+          LABELS=""
+          for i in 1 2 3; do
+            if LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name'); then
+              break
+            fi
+            if [ "$i" = 3 ]; then
+              echo "::error::Could not fetch PR labels after 3 attempts (GitHub API failure — re-run this job)."
+              exit 1
+            fi
+            echo "::warning::gh pr view failed (attempt $i); retrying in 10s"
+            sleep 10
+          done
           if echo "$LABELS" | grep -Fxq 'mcp-catalog-reviewed'; then
             echo "MCP catalog review label present."
             exit 0

.github/workflows/tests.yml

@@ -32,6 +32,12 @@ jobs:
         with:
           path: test_durations.json
           key: test-durations
+          # Saves use test-durations-${run_id}, so the exact key above never
+          # matches — without this prefix fallback the cache ALWAYS missed,
+          # LPT slicing ran on no data, and unbalanced slices pushed heavy
+          # files toward the per-file timeout under load.
+          restore-keys: |
+            test-durations-
 
       - name: Generate test slices
         id: matrix
@@ -115,6 +121,9 @@ jobs:
           NOUS_API_KEY: ""
 
       - name: Upload per-slice durations
+        # Advisory artifact (feeds slice balancing) — a transient artifact-
+        # service blip must not fail an otherwise-green test slice.
+        continue-on-error: true
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: test-durations-slice-${{ matrix.slice.index }}

.github/workflows/uv-lockfile-check.yml

@@ -74,7 +74,20 @@ jobs:
       # rebase and regenerate uv.lock."
       - name: Verify uv.lock is up-to-date
         run: |
-          if ! uv lock --check; then
+          # uv lock --check re-resolves against PyPI (network). Retry so a
+          # registry blip doesn't read as "lockfile stale". A genuinely stale
+          # lockfile fails all attempts (deterministic), costing only seconds.
+          ok=false
+          for i in 1 2 3; do
+            if uv lock --check; then
+              ok=true
+              break
+            fi
+            [ "$i" = 3 ] && break
+            echo "::warning::uv lock --check failed (attempt $i); retrying in 10s"
+            sleep 10
+          done
+          if [ "$ok" != true ]; then
             cat <<'EOF' >> "$GITHUB_STEP_SUMMARY"
           ## ❌ uv.lock is out of sync with pyproject.toml
 

Dockerfile

@@ -26,8 +26,8 @@ ENV PLAYWRIGHT_BROWSERS_PATH=/opt/hermes/.playwright
 # replaces tini with s6-overlay's /init (PID 1 = s6-svscan), which reaps
 # zombies non-blockingly on SIGCHLD and additionally supervises the main
 # hermes process, the dashboard, and per-profile gateways.
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
+RUN apt-get -o Acquire::Retries=3 update && \
+    apt-get -o Acquire::Retries=3 install -y --no-install-recommends \
     ca-certificates curl iputils-ping python3 python-is-python3 ripgrep ffmpeg gcc g++ make cmake python3-dev python3-venv libffi-dev libolm-dev procps git openssh-client docker-cli xz-utils && \
     rm -rf /var/lib/apt/lists/*
 
@@ -40,33 +40,30 @@ RUN apt-get update && \
 # we map between them inline. The noarch + symlinks tarballs are
 # architecture-independent and reused as-is.
 #
-# We use `curl` instead of `ADD` for the per-arch tarball because `ADD`
-# evaluates its URL at parse time, before any ARG / TARGETARCH substitution
-# — splitting one URL per arch into two ADDs would download both on every
-# build and leave dead bytes in the cache. A single curl + arch-keyed URL
-# is simpler and cache-friendlier.
-#
-# Supply-chain integrity: every tarball is checksum-verified against the
-# upstream-published SHA256. To bump S6_OVERLAY_VERSION, fetch the four
-# `.sha256` files from the corresponding release and update the ARGs. The
-# checksum lookup happens during build, so a compromised release artifact
-# fails the build loudly instead of silently producing a tampered image.
+# We use `curl` instead of `ADD` for ALL three tarballs: `ADD` evaluates its
+# URL at parse time (no ARG / TARGETARCH substitution) and — critically for
+# CI reliability — cannot retry, so a single GitHub-release CDN blip fails
+# the whole 15-45 min build. curl -fsSL --retry 3 self-heals those blips,
+# and every tarball is still checksum-verified below before extraction.
 ARG TARGETARCH
 ARG S6_OVERLAY_VERSION=3.2.3.0
 ARG S6_OVERLAY_NOARCH_SHA256=b720f9d9340efc8bb07528b9743813c836e4b02f8693d90241f047998b4c53cf
 ARG S6_OVERLAY_X86_64_SHA256=a93f02882c6ed46b21e7adb5c0add86154f01236c93cd82c7d682722e8840563
 ARG S6_OVERLAY_AARCH64_SHA256=0952056ff913482163cc30e35b2e944b507ba1025d78f5becbb89367bf344581
 ARG S6_OVERLAY_SYMLINKS_SHA256=a60dc5235de3ecbcf874b9c1f18d73263ab99b289b9329aa950e8729c4789f0e
-ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz /tmp/
-ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-symlinks-noarch.tar.xz /tmp/
 RUN set -eu; \
     case "${TARGETARCH:-amd64}" in \
         amd64) s6_arch="x86_64"; s6_arch_sha="${S6_OVERLAY_X86_64_SHA256}" ;; \
         arm64) s6_arch="aarch64"; s6_arch_sha="${S6_OVERLAY_AARCH64_SHA256}" ;; \
         *) echo "Unsupported TARGETARCH=${TARGETARCH} for s6-overlay" >&2; exit 1 ;; \
     esac; \
+    base="https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}"; \
+    curl -fsSL --retry 3 -o /tmp/s6-overlay-noarch.tar.xz \
+        "${base}/s6-overlay-noarch.tar.xz"; \
+    curl -fsSL --retry 3 -o /tmp/s6-overlay-symlinks-noarch.tar.xz \
+        "${base}/s6-overlay-symlinks-noarch.tar.xz"; \
     curl -fsSL --retry 3 -o /tmp/s6-overlay-arch.tar.xz \
-        "https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-${s6_arch}.tar.xz"; \
+        "${base}/s6-overlay-${s6_arch}.tar.xz"; \
     { \
         printf '%s  %s\n' "${S6_OVERLAY_NOARCH_SHA256}" /tmp/s6-overlay-noarch.tar.xz; \
         printf '%s  %s\n' "${s6_arch_sha}" /tmp/s6-overlay-arch.tar.xz; \
@@ -135,8 +132,11 @@ COPY apps/shared/ apps/shared/
 # guards against a future regression if the source npm version changes.
 ENV npm_config_install_links=false
 
-RUN npm install --prefer-offline --no-audit && \
-    npx playwright install --with-deps chromium --only-shell && \
+RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \
+    for i in 1 2 3; do \
+        npx playwright install --with-deps chromium --only-shell && break || \
+        { [ "$i" = 3 ] && exit 1; echo "playwright install failed (attempt $i); retrying in 10s"; sleep 10; }; \
+    done && \
     npm cache clean --force
 
 # ---------- Layer-cached Python dependency install ----------