mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
e4fe99987d
Swap all `runs-on: ubuntu-latest` to `runs-on: arc-runner-set` all jobs.
The ARM docker build job in docker.yml uses `${{ matrix.runner }}`
and is left untouched since the GKE runner pool is x86_64 only.
Runners are backed by ARC (Actions Runner Controller) on a GKE cluster
with a spot preemptible node pool that scales based on job demand.
Use the baked Electron dependencies for the desktop E2E job.
249 lines
10 KiB
YAML
249 lines
10 KiB
YAML
name: Tests
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
slice_count:
|
|
description: Number of parallel test slices
|
|
type: number
|
|
default: 8
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Cancel in-progress runs for the same ref
|
|
concurrency:
|
|
group: tests-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
generate:
|
|
name: "Generate slices"
|
|
runs-on: arc-runner-set
|
|
timeout-minutes: 10
|
|
outputs:
|
|
matrix: ${{ steps.matrix.outputs.matrix }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Restore duration cache
|
|
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
|
with:
|
|
path: test_durations.json
|
|
key: test-durations
|
|
# Saves use test-durations-${run_id}, so the exact key above never
|
|
# matches — without this prefix fallback the cache ALWAYS missed,
|
|
# LPT slicing ran on no data, and unbalanced slices pushed heavy
|
|
# files toward the per-file timeout under load.
|
|
restore-keys: |
|
|
test-durations-
|
|
|
|
- name: Generate test slices
|
|
id: matrix
|
|
run: |
|
|
MATRIX=$(python3 scripts/run_tests_parallel.py --generate-slices ${{ inputs.slice_count }})
|
|
echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"
|
|
|
|
test:
|
|
name: Run tests slice ${{ matrix.slice.index }}/${{ inputs.slice_count }}
|
|
needs: generate
|
|
runs-on: arc-runner-set
|
|
timeout-minutes: 30
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{ fromJSON(needs.generate.outputs.matrix) }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Install ripgrep (prebuilt binary)
|
|
run: |
|
|
set -euo pipefail
|
|
RG_VERSION=15.1.0
|
|
RG_SHA256=1c9297be4a084eea7ecaedf93eb03d058d6faae29bbc57ecdaf5063921491599
|
|
RG_TARBALL=ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz
|
|
curl -sSfL --retry 3 --retry-delay 5 -o "$RG_TARBALL" \
|
|
"https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${RG_TARBALL}"
|
|
echo "${RG_SHA256} ${RG_TARBALL}" | sha256sum -c -
|
|
tar -xzf "$RG_TARBALL"
|
|
sudo mv "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg
|
|
rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
|
|
rg --version
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0
|
|
with:
|
|
# Pin the uv version: unpinned, setup-uv resolves "latest" by
|
|
# fetching a manifest from raw.githubusercontent.com on EVERY job —
|
|
# a transient fetch failure fails the whole job (2026-07-28 slice-5
|
|
# incident). Pinned, the binary downloads directly; no manifest hop.
|
|
version: "0.9.28"
|
|
# Persist uv's download/wheel cache (~/.cache/uv) across runs.
|
|
# Keyed on the dependency manifests, so the cache is reused until
|
|
# pyproject.toml or uv.lock changes. `uv sync` still runs every
|
|
# time, but resolves from the warm cache instead of re-downloading
|
|
# and re-building wheels.
|
|
enable-cache: true
|
|
cache-dependency-glob: |
|
|
pyproject.toml
|
|
uv.lock
|
|
|
|
- name: Set up Python 3.11
|
|
run: uv python install 3.11
|
|
|
|
- name: Install dependencies
|
|
# `uv sync --locked` installs the exact pinned set from uv.lock (and
|
|
# fails if the lock is out of sync with pyproject.toml), giving a
|
|
# reproducible env. It also creates .venv itself, so no separate
|
|
# `uv venv` step is needed.
|
|
#
|
|
# The trailing extras beyond all/dev are the lazy-install features
|
|
# (tools/lazy_deps.py) that tests exercise for real: provider.anthropic,
|
|
# stt/tts.mistral, image.fal, terminal.modal, terminal.daytona,
|
|
# memory.hindsight, search.parallel. The hermetic test env forbids
|
|
# mid-run pip installs (HERMES_DISABLE_LAZY_INSTALLS=1 in
|
|
# tests/conftest.py), so the SDKs those tests need must be in the
|
|
# venv up front — resolved from uv.lock like everything else, which
|
|
# also honors the exact supply-chain pins these extras carry.
|
|
uses: ./.github/actions/retry
|
|
with:
|
|
command: uv sync --locked --python 3.11 --extra all --extra dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra hindsight --extra parallel-web
|
|
|
|
- name: Minimize uv cache
|
|
# Optimized for CI: prunes pre-built wheels that are cheap to
|
|
# re-download, keeping the persisted cache small and fast to restore.
|
|
run: uv cache prune --ci
|
|
|
|
- name: Run tests (slice ${{ matrix.slice.index }}/${{ inputs.slice_count }})
|
|
# Per-file isolation via scripts/run_tests.sh: each test file runs
|
|
# in its own freshly-spawned `python -m pytest <file>` subprocess
|
|
# with bounded parallelism. No xdist, no shared workers, no
|
|
# module-level state leakage between files.
|
|
#
|
|
# File list is pre-computed by the generate job (--generate-slices)
|
|
# which runs LPT distribution once and passes the file list to each
|
|
# matrix job via --files. Previously each job re-discovered files and
|
|
# re-ran LPT independently — redundant N times.
|
|
run: |
|
|
source .venv/bin/activate
|
|
scripts/run_tests.sh --files '${{ matrix.slice.files }}'
|
|
env:
|
|
# Ensure tests don't accidentally call real APIs
|
|
OPENROUTER_API_KEY: ""
|
|
OPENAI_API_KEY: ""
|
|
NOUS_API_KEY: ""
|
|
|
|
- name: Upload per-slice durations
|
|
# Advisory artifact (feeds slice balancing) — a transient artifact-
|
|
# service blip must not fail an otherwise-green test slice.
|
|
continue-on-error: true
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: test-durations-slice-${{ matrix.slice.index }}
|
|
path: test_durations.json
|
|
retention-days: 1
|
|
|
|
# Merge per-slice duration data into a single cache, so future runs
|
|
# (including PRs) get balanced slicing.
|
|
save-durations:
|
|
needs: test
|
|
if: needs.test.result == 'success' && github.ref == 'refs/heads/main'
|
|
runs-on: arc-runner-set
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Download all slice durations
|
|
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
with:
|
|
pattern: test-durations-slice-*
|
|
path: durations
|
|
merge-multiple: true
|
|
|
|
- name: Merge into single durations file
|
|
run: |
|
|
python3 -c "
|
|
import json, glob, os
|
|
merged = {}
|
|
for f in glob.glob('durations/*test_durations.json'):
|
|
with open(f) as fh:
|
|
merged.update(json.load(fh))
|
|
with open('test_durations.json', 'w') as fh:
|
|
json.dump(merged, fh, indent=2, sort_keys=True)
|
|
print(f'Merged {len(merged)} file durations')
|
|
"
|
|
|
|
- name: Save merged duration cache
|
|
uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
|
with:
|
|
path: test_durations.json
|
|
key: test-durations-${{ github.run_id }}
|
|
|
|
e2e:
|
|
runs-on: arc-runner-set
|
|
timeout-minutes: 15
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Install ripgrep (prebuilt binary)
|
|
run: |
|
|
set -euo pipefail
|
|
RG_VERSION=15.1.0
|
|
RG_SHA256=1c9297be4a084eea7ecaedf93eb03d058d6faae29bbc57ecdaf5063921491599
|
|
RG_TARBALL=ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz
|
|
curl -sSfL --retry 3 --retry-delay 5 -o "$RG_TARBALL" \
|
|
"https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${RG_TARBALL}"
|
|
echo "${RG_SHA256} ${RG_TARBALL}" | sha256sum -c -
|
|
tar -xzf "$RG_TARBALL"
|
|
sudo mv "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg
|
|
rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
|
|
rg --version
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0
|
|
with:
|
|
# Pin the uv version: unpinned, setup-uv resolves "latest" by
|
|
# fetching a manifest from raw.githubusercontent.com on EVERY job —
|
|
# a transient fetch failure fails the whole job (2026-07-28 slice-5
|
|
# incident). Pinned, the binary downloads directly; no manifest hop.
|
|
version: "0.9.28"
|
|
# Persist uv's download/wheel cache (~/.cache/uv) across runs.
|
|
# Keyed on the dependency manifests, so the cache is reused until
|
|
# pyproject.toml or uv.lock changes. `uv sync` still runs every
|
|
# time, but resolves from the warm cache instead of re-downloading
|
|
# and re-building wheels.
|
|
enable-cache: true
|
|
cache-dependency-glob: |
|
|
pyproject.toml
|
|
uv.lock
|
|
|
|
- name: Set up Python 3.11
|
|
run: uv python install 3.11
|
|
|
|
- name: Install dependencies
|
|
# `uv sync --locked` installs the exact pinned set from uv.lock (and
|
|
# fails if the lock is out of sync with pyproject.toml), giving a
|
|
# reproducible env. It also creates .venv itself, so no separate
|
|
# `uv venv` step is needed.
|
|
#
|
|
# Same extras as the test job's sync above: the hermetic test env
|
|
# forbids mid-run pip installs (HERMES_DISABLE_LAZY_INSTALLS=1 in
|
|
# tests/conftest.py), so lazy-install SDKs exercised by tests must be
|
|
# in the venv up front.
|
|
uses: ./.github/actions/retry
|
|
with:
|
|
command: uv sync --locked --python 3.11 --extra all --extra dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra hindsight --extra parallel-web
|
|
|
|
- name: Minimize uv cache
|
|
# Optimized for CI: prunes pre-built wheels that are cheap to
|
|
# re-download, keeping the persisted cache small and fast to restore.
|
|
run: uv cache prune --ci
|
|
|
|
- name: Run e2e tests
|
|
run: |
|
|
source .venv/bin/activate
|
|
python -m pytest tests/e2e/ -v --tb=short
|
|
env:
|
|
OPENROUTER_API_KEY: ""
|
|
OPENAI_API_KEY: ""
|
|
NOUS_API_KEY: ""
|