diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml index 0b990491ff4..2b6a0fa7d25 100644 --- a/.github/actions/detect-changes/action.yml +++ b/.github/actions/detect-changes/action.yml @@ -57,10 +57,26 @@ runs: # event payload instead of the "current PR files" endpoint. The SHAs # are frozen at trigger time, so the file list is deterministic even # if the PR receives a new push between trigger and detect. - CHANGED="$(gh api \ - --paginate \ - "repos/${REPO}/compare/${BASE_SHA}...${HEAD_SHA}" \ - --jq '.files[].filename' || true)" + # + # Retried: a rate-limit blip or eventual-consistency 404 on a + # freshly-pushed HEAD would otherwise silently fall open (all lanes + # run — safe, but wasteful and it masks the API failure). + CHANGED="" + for i in 1 2 3; do + if CHANGED="$(gh api \ + --paginate \ + "repos/${REPO}/compare/${BASE_SHA}...${HEAD_SHA}" \ + --jq '.files[].filename')"; then + break + fi + if [ "$i" = 3 ]; then + echo "::warning::compare API failed after 3 attempts — failing open (all lanes run)" + CHANGED="" + break + fi + echo "::warning::compare API failed (attempt $i); retrying in 10s" + sleep 10 + done fi echo "Changed files:" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc0219f57d8..05593a8ba95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,6 +220,8 @@ jobs: --summary-out ci-timings-summary.md - name: Upload HTML report + # Advisory report — artifact-service blips must not fail the job. + continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 id: ci-timings-artifact with: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e92a085f781..f500aca9953 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -189,15 +189,23 @@ jobs: args+=("${IMAGE_NAME}@sha256:${digest_file}") done if [ "${{ github.event_name }}" = "release" ]; then - docker buildx imagetools create \ - -t "${IMAGE_NAME}:${RELEASE_TAG}" \ - "${args[@]}" + tags=(-t "${IMAGE_NAME}:${RELEASE_TAG}") else - docker buildx imagetools create \ - -t "${IMAGE_NAME}:main" \ - -t "${IMAGE_NAME}:latest" \ - "${args[@]}" + tags=(-t "${IMAGE_NAME}:main" -t "${IMAGE_NAME}:latest") fi + # Retry: Docker Hub API + just-pushed digest eventual consistency + # can transiently fail the create; the operation is idempotent. + for i in 1 2 3; do + if docker buildx imagetools create "${tags[@]}" "${args[@]}"; then + break + fi + if [ "$i" = 3 ]; then + echo "::error::imagetools create failed after 3 attempts" + exit 1 + fi + echo "::warning::imagetools create failed (attempt $i); retrying in 20s" + sleep 20 + done - name: Inspect image env: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e88294dd94e..e1c42a50e63 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -184,7 +184,21 @@ jobs: run: | set -euo pipefail PR="${{ github.event.pull_request.number }}" - LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name' || true) + # Retry the label fetch: a transient API blip must not read as + # "label absent" (which hard-fails the job below on a PR that + # actually carries the label). + LABELS="" + for i in 1 2 3; do + if LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name'); then + break + fi + if [ "$i" = 3 ]; then + echo "::error::Could not fetch PR labels after 3 attempts (GitHub API failure — re-run this job)." + exit 1 + fi + echo "::warning::gh pr view failed (attempt $i); retrying in 10s" + sleep 10 + done if echo "$LABELS" | grep -Fxq 'ci-reviewed'; then echo "reviewed=true" >> "$GITHUB_OUTPUT" echo "ci-reviewed label present." diff --git a/.github/workflows/supply-chain-audit.yml b/.github/workflows/supply-chain-audit.yml index ff2df8e077f..3dd7888546a 100644 --- a/.github/workflows/supply-chain-audit.yml +++ b/.github/workflows/supply-chain-audit.yml @@ -244,7 +244,20 @@ jobs: run: | set -euo pipefail PR="${{ github.event.pull_request.number }}" - LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name' || true) + # Retry the label fetch so an API blip doesn't read as "label + # absent" and falsely block the PR. + LABELS="" + for i in 1 2 3; do + if LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name'); then + break + fi + if [ "$i" = 3 ]; then + echo "::error::Could not fetch PR labels after 3 attempts (GitHub API failure — re-run this job)." + exit 1 + fi + echo "::warning::gh pr view failed (attempt $i); retrying in 10s" + sleep 10 + done if echo "$LABELS" | grep -Fxq 'mcp-catalog-reviewed'; then echo "MCP catalog review label present." exit 0 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index faa4b072504..1f29b25008e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,6 +32,12 @@ jobs: with: path: test_durations.json key: test-durations + # Saves use test-durations-${run_id}, so the exact key above never + # matches — without this prefix fallback the cache ALWAYS missed, + # LPT slicing ran on no data, and unbalanced slices pushed heavy + # files toward the per-file timeout under load. + restore-keys: | + test-durations- - name: Generate test slices id: matrix @@ -115,6 +121,9 @@ jobs: NOUS_API_KEY: "" - name: Upload per-slice durations + # Advisory artifact (feeds slice balancing) — a transient artifact- + # service blip must not fail an otherwise-green test slice. + continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: test-durations-slice-${{ matrix.slice.index }} diff --git a/.github/workflows/uv-lockfile-check.yml b/.github/workflows/uv-lockfile-check.yml index 8a7f52e899a..27b072a9941 100644 --- a/.github/workflows/uv-lockfile-check.yml +++ b/.github/workflows/uv-lockfile-check.yml @@ -74,7 +74,20 @@ jobs: # rebase and regenerate uv.lock." - name: Verify uv.lock is up-to-date run: | - if ! uv lock --check; then + # uv lock --check re-resolves against PyPI (network). Retry so a + # registry blip doesn't read as "lockfile stale". A genuinely stale + # lockfile fails all attempts (deterministic), costing only seconds. + ok=false + for i in 1 2 3; do + if uv lock --check; then + ok=true + break + fi + [ "$i" = 3 ] && break + echo "::warning::uv lock --check failed (attempt $i); retrying in 10s" + sleep 10 + done + if [ "$ok" != true ]; then cat <<'EOF' >> "$GITHUB_STEP_SUMMARY" ## ❌ uv.lock is out of sync with pyproject.toml diff --git a/Dockerfile b/Dockerfile index 6f957f77967..6803adc2e1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,8 @@ ENV PLAYWRIGHT_BROWSERS_PATH=/opt/hermes/.playwright # replaces tini with s6-overlay's /init (PID 1 = s6-svscan), which reaps # zombies non-blockingly on SIGCHLD and additionally supervises the main # hermes process, the dashboard, and per-profile gateways. -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ +RUN apt-get -o Acquire::Retries=3 update && \ + apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ ca-certificates curl iputils-ping python3 python-is-python3 ripgrep ffmpeg gcc g++ make cmake python3-dev python3-venv libffi-dev libolm-dev procps git openssh-client docker-cli xz-utils && \ rm -rf /var/lib/apt/lists/* @@ -40,33 +40,30 @@ RUN apt-get update && \ # we map between them inline. The noarch + symlinks tarballs are # architecture-independent and reused as-is. # -# We use `curl` instead of `ADD` for the per-arch tarball because `ADD` -# evaluates its URL at parse time, before any ARG / TARGETARCH substitution -# — splitting one URL per arch into two ADDs would download both on every -# build and leave dead bytes in the cache. A single curl + arch-keyed URL -# is simpler and cache-friendlier. -# -# Supply-chain integrity: every tarball is checksum-verified against the -# upstream-published SHA256. To bump S6_OVERLAY_VERSION, fetch the four -# `.sha256` files from the corresponding release and update the ARGs. The -# checksum lookup happens during build, so a compromised release artifact -# fails the build loudly instead of silently producing a tampered image. +# We use `curl` instead of `ADD` for ALL three tarballs: `ADD` evaluates its +# URL at parse time (no ARG / TARGETARCH substitution) and — critically for +# CI reliability — cannot retry, so a single GitHub-release CDN blip fails +# the whole 15-45 min build. curl -fsSL --retry 3 self-heals those blips, +# and every tarball is still checksum-verified below before extraction. ARG TARGETARCH ARG S6_OVERLAY_VERSION=3.2.3.0 ARG S6_OVERLAY_NOARCH_SHA256=b720f9d9340efc8bb07528b9743813c836e4b02f8693d90241f047998b4c53cf ARG S6_OVERLAY_X86_64_SHA256=a93f02882c6ed46b21e7adb5c0add86154f01236c93cd82c7d682722e8840563 ARG S6_OVERLAY_AARCH64_SHA256=0952056ff913482163cc30e35b2e944b507ba1025d78f5becbb89367bf344581 ARG S6_OVERLAY_SYMLINKS_SHA256=a60dc5235de3ecbcf874b9c1f18d73263ab99b289b9329aa950e8729c4789f0e -ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz /tmp/ -ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-symlinks-noarch.tar.xz /tmp/ RUN set -eu; \ case "${TARGETARCH:-amd64}" in \ amd64) s6_arch="x86_64"; s6_arch_sha="${S6_OVERLAY_X86_64_SHA256}" ;; \ arm64) s6_arch="aarch64"; s6_arch_sha="${S6_OVERLAY_AARCH64_SHA256}" ;; \ *) echo "Unsupported TARGETARCH=${TARGETARCH} for s6-overlay" >&2; exit 1 ;; \ esac; \ + base="https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}"; \ + curl -fsSL --retry 3 -o /tmp/s6-overlay-noarch.tar.xz \ + "${base}/s6-overlay-noarch.tar.xz"; \ + curl -fsSL --retry 3 -o /tmp/s6-overlay-symlinks-noarch.tar.xz \ + "${base}/s6-overlay-symlinks-noarch.tar.xz"; \ curl -fsSL --retry 3 -o /tmp/s6-overlay-arch.tar.xz \ - "https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-${s6_arch}.tar.xz"; \ + "${base}/s6-overlay-${s6_arch}.tar.xz"; \ { \ printf '%s %s\n' "${S6_OVERLAY_NOARCH_SHA256}" /tmp/s6-overlay-noarch.tar.xz; \ printf '%s %s\n' "${s6_arch_sha}" /tmp/s6-overlay-arch.tar.xz; \ @@ -135,8 +132,11 @@ COPY apps/shared/ apps/shared/ # guards against a future regression if the source npm version changes. ENV npm_config_install_links=false -RUN npm install --prefer-offline --no-audit && \ - npx playwright install --with-deps chromium --only-shell && \ +RUN npm install --prefer-offline --no-audit --fetch-retries=5 && \ + for i in 1 2 3; do \ + npx playwright install --with-deps chromium --only-shell && break || \ + { [ "$i" = 3 ] && exit 1; echo "playwright install failed (attempt $i); retrying in 10s"; sleep 10; }; \ + done && \ npm cache clean --force # ---------- Layer-cached Python dependency install ----------