mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-28 01:21:43 +00:00
50f23ea522
Target: <2min CI test wall time. Runs the Tests workflow as a 4-way matrix instead of one job. Each shard runs ~3,000 tests on its own ubuntu-latest runner (4 cores) with -n auto xdist inside. Total effective parallelism: 16 workers across 4 machines (vs 4 workers on 1 machine today). Was previously tried in #11566 and closed — shard 3 hung at 97% complete for 100+ seconds with dozens of E/F markers. Root cause was cross-test pollution exposed by splitting test files across shards (e.g. the three test files that mutated sys.modules['dotenv'] at import time poisoned whichever shard they landed in). That's now fixed by #11453 and #11577: conftest is hermetic, the dotenv stub bombs are removed, and tests no longer depend on each other's env-var side effects. Changes: - pyproject.toml: add pytest-split>=0.9,<1 to dev extras - .github/workflows/tests.yml: 'test' job becomes matrix-split into 4 groups with fail-fast: false. Runs 'pytest --splits 4 --group N'. pytest-split composes with -n auto from pyproject addopts. e2e job is unchanged (already small, 20s). Expected timing: Before: ~4m total (243s test step + ~25s setup) After: ~90-115s total (shard wall time ~60-90s + ~25s setup) Hash-based split is deterministic; no .test_durations file needed yet. Can add one later via --store-durations for better shard balance.
82 lines
2.1 KiB
YAML
82 lines
2.1 KiB
YAML
name: Tests
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
pull_request:
|
|
branches: [main]
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Cancel in-progress runs for the same PR/branch
|
|
concurrency:
|
|
group: tests-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
test:
|
|
name: test (${{ matrix.group }}/4)
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
group: [1, 2, 3, 4]
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
|
|
|
- name: Install system dependencies
|
|
run: sudo apt-get update && sudo apt-get install -y ripgrep
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
|
|
|
|
- name: Set up Python 3.11
|
|
run: uv python install 3.11
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
uv venv .venv --python 3.11
|
|
source .venv/bin/activate
|
|
uv pip install -e ".[all,dev]"
|
|
|
|
- name: Run tests (shard ${{ matrix.group }}/4)
|
|
run: |
|
|
source .venv/bin/activate
|
|
python -m pytest tests/ -q --ignore=tests/integration --ignore=tests/e2e --tb=short \
|
|
--splits 4 --group ${{ matrix.group }}
|
|
env:
|
|
# Ensure tests don't accidentally call real APIs
|
|
OPENROUTER_API_KEY: ""
|
|
OPENAI_API_KEY: ""
|
|
NOUS_API_KEY: ""
|
|
|
|
e2e:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
|
|
|
|
- name: Set up Python 3.11
|
|
run: uv python install 3.11
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
uv venv .venv --python 3.11
|
|
source .venv/bin/activate
|
|
uv pip install -e ".[all,dev]"
|
|
|
|
- name: Run e2e tests
|
|
run: |
|
|
source .venv/bin/activate
|
|
python -m pytest tests/e2e/ -v --tb=short
|
|
env:
|
|
OPENROUTER_API_KEY: ""
|
|
OPENAI_API_KEY: ""
|
|
NOUS_API_KEY: ""
|