mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
fix(ci): kill the remaining transient-failure classes in workflows + Dockerfile
From the workflow reliability audit: - tests.yml: duration-cache restore had NO restore-keys while saves use run_id-suffixed keys — the cache never matched once, so LPT slicing always ran blind and unbalanced slices pushed heavy files toward the per-file timeout. One-line restore-keys fixes slice balancing. - Label gates (lint ci-reviewed, supply-chain mcp-catalog-reviewed): 'gh pr view || true' turned an API blip into 'label absent' → false BLOCKING failure. Now 3x retry, and API failure is reported as an API failure instead of a missing label. - detect-changes action: compare API retried before failing open (was silently running all lanes on any blip). - uv-lockfile-check: 'uv lock --check' resolves against PyPI — retried so registry blips don't read as 'lockfile stale'. - docker.yml merge job: imagetools create retried (Docker Hub eventual consistency on just-pushed digests). - Dockerfile: apt-get Acquire::Retries=3; s6-overlay ADDs converted to curl --retry 3 (ADD cannot retry; checksums still enforced); npm --fetch-retries=5; playwright chromium fetch retried 3x. - Advisory artifact uploads (per-slice durations, ci-timings report) get continue-on-error so an artifact-service blip can't fail a green test slice.
This commit is contained in:
parent
3edcf23a44
commit
007ca57dc8
8 changed files with 107 additions and 32 deletions
24
.github/actions/detect-changes/action.yml
vendored
24
.github/actions/detect-changes/action.yml
vendored
|
|
@ -57,10 +57,26 @@ runs:
|
|||
# event payload instead of the "current PR files" endpoint. The SHAs
|
||||
# are frozen at trigger time, so the file list is deterministic even
|
||||
# if the PR receives a new push between trigger and detect.
|
||||
CHANGED="$(gh api \
|
||||
--paginate \
|
||||
"repos/${REPO}/compare/${BASE_SHA}...${HEAD_SHA}" \
|
||||
--jq '.files[].filename' || true)"
|
||||
#
|
||||
# Retried: a rate-limit blip or eventual-consistency 404 on a
|
||||
# freshly-pushed HEAD would otherwise silently fall open (all lanes
|
||||
# run — safe, but wasteful and it masks the API failure).
|
||||
CHANGED=""
|
||||
for i in 1 2 3; do
|
||||
if CHANGED="$(gh api \
|
||||
--paginate \
|
||||
"repos/${REPO}/compare/${BASE_SHA}...${HEAD_SHA}" \
|
||||
--jq '.files[].filename')"; then
|
||||
break
|
||||
fi
|
||||
if [ "$i" = 3 ]; then
|
||||
echo "::warning::compare API failed after 3 attempts — failing open (all lanes run)"
|
||||
CHANGED=""
|
||||
break
|
||||
fi
|
||||
echo "::warning::compare API failed (attempt $i); retrying in 10s"
|
||||
sleep 10
|
||||
done
|
||||
fi
|
||||
|
||||
echo "Changed files:"
|
||||
|
|
|
|||
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
|
|
@ -220,6 +220,8 @@ jobs:
|
|||
--summary-out ci-timings-summary.md
|
||||
|
||||
- name: Upload HTML report
|
||||
# Advisory report — artifact-service blips must not fail the job.
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
id: ci-timings-artifact
|
||||
with:
|
||||
|
|
|
|||
22
.github/workflows/docker.yml
vendored
22
.github/workflows/docker.yml
vendored
|
|
@ -189,15 +189,23 @@ jobs:
|
|||
args+=("${IMAGE_NAME}@sha256:${digest_file}")
|
||||
done
|
||||
if [ "${{ github.event_name }}" = "release" ]; then
|
||||
docker buildx imagetools create \
|
||||
-t "${IMAGE_NAME}:${RELEASE_TAG}" \
|
||||
"${args[@]}"
|
||||
tags=(-t "${IMAGE_NAME}:${RELEASE_TAG}")
|
||||
else
|
||||
docker buildx imagetools create \
|
||||
-t "${IMAGE_NAME}:main" \
|
||||
-t "${IMAGE_NAME}:latest" \
|
||||
"${args[@]}"
|
||||
tags=(-t "${IMAGE_NAME}:main" -t "${IMAGE_NAME}:latest")
|
||||
fi
|
||||
# Retry: Docker Hub API + just-pushed digest eventual consistency
|
||||
# can transiently fail the create; the operation is idempotent.
|
||||
for i in 1 2 3; do
|
||||
if docker buildx imagetools create "${tags[@]}" "${args[@]}"; then
|
||||
break
|
||||
fi
|
||||
if [ "$i" = 3 ]; then
|
||||
echo "::error::imagetools create failed after 3 attempts"
|
||||
exit 1
|
||||
fi
|
||||
echo "::warning::imagetools create failed (attempt $i); retrying in 20s"
|
||||
sleep 20
|
||||
done
|
||||
|
||||
- name: Inspect image
|
||||
env:
|
||||
|
|
|
|||
16
.github/workflows/lint.yml
vendored
16
.github/workflows/lint.yml
vendored
|
|
@ -184,7 +184,21 @@ jobs:
|
|||
run: |
|
||||
set -euo pipefail
|
||||
PR="${{ github.event.pull_request.number }}"
|
||||
LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name' || true)
|
||||
# Retry the label fetch: a transient API blip must not read as
|
||||
# "label absent" (which hard-fails the job below on a PR that
|
||||
# actually carries the label).
|
||||
LABELS=""
|
||||
for i in 1 2 3; do
|
||||
if LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name'); then
|
||||
break
|
||||
fi
|
||||
if [ "$i" = 3 ]; then
|
||||
echo "::error::Could not fetch PR labels after 3 attempts (GitHub API failure — re-run this job)."
|
||||
exit 1
|
||||
fi
|
||||
echo "::warning::gh pr view failed (attempt $i); retrying in 10s"
|
||||
sleep 10
|
||||
done
|
||||
if echo "$LABELS" | grep -Fxq 'ci-reviewed'; then
|
||||
echo "reviewed=true" >> "$GITHUB_OUTPUT"
|
||||
echo "ci-reviewed label present."
|
||||
|
|
|
|||
15
.github/workflows/supply-chain-audit.yml
vendored
15
.github/workflows/supply-chain-audit.yml
vendored
|
|
@ -244,7 +244,20 @@ jobs:
|
|||
run: |
|
||||
set -euo pipefail
|
||||
PR="${{ github.event.pull_request.number }}"
|
||||
LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name' || true)
|
||||
# Retry the label fetch so an API blip doesn't read as "label
|
||||
# absent" and falsely block the PR.
|
||||
LABELS=""
|
||||
for i in 1 2 3; do
|
||||
if LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name'); then
|
||||
break
|
||||
fi
|
||||
if [ "$i" = 3 ]; then
|
||||
echo "::error::Could not fetch PR labels after 3 attempts (GitHub API failure — re-run this job)."
|
||||
exit 1
|
||||
fi
|
||||
echo "::warning::gh pr view failed (attempt $i); retrying in 10s"
|
||||
sleep 10
|
||||
done
|
||||
if echo "$LABELS" | grep -Fxq 'mcp-catalog-reviewed'; then
|
||||
echo "MCP catalog review label present."
|
||||
exit 0
|
||||
|
|
|
|||
9
.github/workflows/tests.yml
vendored
9
.github/workflows/tests.yml
vendored
|
|
@ -32,6 +32,12 @@ jobs:
|
|||
with:
|
||||
path: test_durations.json
|
||||
key: test-durations
|
||||
# Saves use test-durations-${run_id}, so the exact key above never
|
||||
# matches — without this prefix fallback the cache ALWAYS missed,
|
||||
# LPT slicing ran on no data, and unbalanced slices pushed heavy
|
||||
# files toward the per-file timeout under load.
|
||||
restore-keys: |
|
||||
test-durations-
|
||||
|
||||
- name: Generate test slices
|
||||
id: matrix
|
||||
|
|
@ -115,6 +121,9 @@ jobs:
|
|||
NOUS_API_KEY: ""
|
||||
|
||||
- name: Upload per-slice durations
|
||||
# Advisory artifact (feeds slice balancing) — a transient artifact-
|
||||
# service blip must not fail an otherwise-green test slice.
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: test-durations-slice-${{ matrix.slice.index }}
|
||||
|
|
|
|||
15
.github/workflows/uv-lockfile-check.yml
vendored
15
.github/workflows/uv-lockfile-check.yml
vendored
|
|
@ -74,7 +74,20 @@ jobs:
|
|||
# rebase and regenerate uv.lock."
|
||||
- name: Verify uv.lock is up-to-date
|
||||
run: |
|
||||
if ! uv lock --check; then
|
||||
# uv lock --check re-resolves against PyPI (network). Retry so a
|
||||
# registry blip doesn't read as "lockfile stale". A genuinely stale
|
||||
# lockfile fails all attempts (deterministic), costing only seconds.
|
||||
ok=false
|
||||
for i in 1 2 3; do
|
||||
if uv lock --check; then
|
||||
ok=true
|
||||
break
|
||||
fi
|
||||
[ "$i" = 3 ] && break
|
||||
echo "::warning::uv lock --check failed (attempt $i); retrying in 10s"
|
||||
sleep 10
|
||||
done
|
||||
if [ "$ok" != true ]; then
|
||||
cat <<'EOF' >> "$GITHUB_STEP_SUMMARY"
|
||||
## ❌ uv.lock is out of sync with pyproject.toml
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue