hermes-agent/.github/workflows/tests.yml

name: Tests

on:
  workflow_call:
    inputs:
      slice_count:
        description: Number of parallel test slices
        type: number
        default: 8

permissions:
  contents: read

# Cancel in-progress runs for the same ref
concurrency:
  group: tests-${{ github.ref }}
  cancel-in-progress: true

jobs:
  generate:
    name: "Generate slices"
    runs-on: arc-runner-set
    timeout-minutes: 10
    outputs:
      matrix: ${{ steps.matrix.outputs.matrix }}
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          # This job only needs test file *paths* — it computes the LPT
          # distribution and never opens a test file. So skip the blobs
          # (~212MB working tree -> ~3MB) and materialize only the script
          # itself. `--discover-from-git` below lists the test paths from
          # the git index, which a sparse checkout leaves fully populated.
          # Checkout was ~90% of this job's wall time, and every test slice
          # waits on it.
          filter: blob:none
          sparse-checkout: scripts/run_tests_parallel.py
          sparse-checkout-cone-mode: false

      - name: Restore duration cache
        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
        with:
          path: test_durations.json
          key: test-durations
          # Saves use test-durations-${run_id}, so the exact key above never
          # matches — without this prefix fallback the cache ALWAYS missed,
          # LPT slicing ran on no data, and unbalanced slices pushed heavy
          # files toward the per-file timeout under load.
          restore-keys: |
            test-durations-

      - name: Generate test slices
        id: matrix
        run: |
          MATRIX=$(python3 scripts/run_tests_parallel.py --generate-slices ${{ inputs.slice_count }} --discover-from-git)
          echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"

  test:
    name: Run tests slice ${{ matrix.slice.index }}/${{ inputs.slice_count }}
    needs: generate
    runs-on: arc-runner-set
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.generate.outputs.matrix) }}
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      # uv and CPython 3.11 are baked into the nousresearch/nous-gke-runner
      # image (hermes-agent-ci-infra runner/Dockerfile) — no setup-uv, no
      # `uv python install`. Only the wheel cache still needs restoring.
      - name: Restore uv cache
        uses: ./.github/actions/uv-cache

      - name: Install dependencies
        # `uv sync --locked` installs the exact pinned set from uv.lock (and
        # fails if the lock is out of sync with pyproject.toml), giving a
        # reproducible env. It also creates .venv itself, so no separate
        # `uv venv` step is needed.
        #
        # The trailing extras beyond all/dev are the lazy-install features
        # (tools/lazy_deps.py) that tests exercise for real: provider.anthropic,
        # stt/tts.mistral, image.fal, terminal.modal, terminal.daytona,
        # memory.hindsight, search.parallel. The hermetic test env forbids
        # mid-run pip installs (HERMES_DISABLE_LAZY_INSTALLS=1 in
        # tests/conftest.py), so the SDKs those tests need must be in the
        # venv up front — resolved from uv.lock like everything else, which
        # also honors the exact supply-chain pins these extras carry.
        uses: ./.github/actions/retry
        with:
          command: uv sync --locked --python 3.11 --extra all --extra dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra hindsight --extra parallel-web

      - name: Minimize uv cache
        # Optimized for CI: prunes pre-built wheels that are cheap to
        # re-download, keeping the persisted cache small and fast to restore.
        run: uv cache prune --ci

      - name: Run tests (slice ${{ matrix.slice.index }}/${{ inputs.slice_count }})
        # Per-file isolation via scripts/run_tests.sh: each test file runs
        # in its own freshly-spawned `python -m pytest <file>` subprocess
        # with bounded parallelism. No xdist, no shared workers, no
        # module-level state leakage between files.
        #
        # File list is pre-computed by the generate job (--generate-slices)
        # which runs LPT distribution once and passes the file list to each
        # matrix job via --files. Previously each job re-discovered files
        # and re-ran LPT independently — redundant N times.
        uses: ./.github/actions/profile
        with:
          label: tests-slice-${{ matrix.slice.index }}
          command: |
            source .venv/bin/activate
            scripts/run_tests.sh --files '${{ matrix.slice.files }}'
        env:
          # Ensure tests don't accidentally call real APIs
          OPENROUTER_API_KEY: ""
          OPENAI_API_KEY: ""
          NOUS_API_KEY: ""

      - name: Upload per-slice durations
        # Advisory artifact (feeds slice balancing) — a transient artifact-
        # service blip must not fail an otherwise-green test slice.
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: test-durations-slice-${{ matrix.slice.index }}
          path: test_durations.json
          retention-days: 1

  # Merge per-slice duration data into a single cache, so future runs
  # (including PRs) get balanced slicing.
  save-durations:
    needs: test
    if: needs.test.result == 'success' && github.ref == 'refs/heads/main'
    runs-on: arc-runner-set
    timeout-minutes: 10
    steps:
      - name: Download all slice durations
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: test-durations-slice-*
          path: durations
          merge-multiple: true

      - name: Merge into single durations file
        run: |
          python3 -c "
          import json, glob, os
          merged = {}
          for f in glob.glob('durations/*test_durations.json'):
            with open(f) as fh:
              merged.update(json.load(fh))
          with open('test_durations.json', 'w') as fh:
            json.dump(merged, fh, indent=2, sort_keys=True)
          print(f'Merged {len(merged)} file durations')
          "

      - name: Save merged duration cache
        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
        with:
          path: test_durations.json
          key: test-durations-${{ github.run_id }}

  e2e:
    runs-on: arc-runner-set
    timeout-minutes: 15
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      # uv and CPython 3.11 are baked into the nousresearch/nous-gke-runner
      # image (hermes-agent-ci-infra runner/Dockerfile) — no setup-uv, no
      # `uv python install`. Only the wheel cache still needs restoring.
      - name: Restore uv cache
        uses: ./.github/actions/uv-cache

      - name: Install dependencies
        # `uv sync --locked` installs the exact pinned set from uv.lock (and
        # fails if the lock is out of sync with pyproject.toml), giving a
        # reproducible env. It also creates .venv itself, so no separate
        # `uv venv` step is needed.
        #
        # Same extras as the test job's sync above: the hermetic test env
        # forbids mid-run pip installs (HERMES_DISABLE_LAZY_INSTALLS=1 in
        # tests/conftest.py), so lazy-install SDKs exercised by tests must be
        # in the venv up front.
        uses: ./.github/actions/retry
        with:
          command: uv sync --locked --python 3.11 --extra all --extra dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra hindsight --extra parallel-web

      - name: Minimize uv cache
        # Optimized for CI: prunes pre-built wheels that are cheap to
        # re-download, keeping the persisted cache small and fast to restore.
        run: uv cache prune --ci

      - name: Run e2e tests
        uses: ./.github/actions/profile
        with:
          label: tests-e2e
          command: |
            source .venv/bin/activate
            python -m pytest tests/e2e/ -v --tb=short
        env:
          OPENROUTER_API_KEY: ""
          OPENAI_API_KEY: ""
          NOUS_API_KEY: ""