mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-31 19:16:29 +00:00
3fc4e413d2
The profiler now monkey-patches time.sleep alongside subprocess.run, capturing the invisible "gap" time from polling loops (wait_for_container_ready, poll_container, wait_for_log, etc.) that was previously unaccounted for. The JSON report now includes per-test total_sleep_s, sleep_count, and a sleeps[] array with caller location. The summary includes total_sleep_s and total_wall_s (docker + sleep). The CI merge step also aggregates sleep totals. Local profile now shows: 201s docker + 75s sleep = 276s wall (38s runner wall with 32-way parallelism). The biggest sleep consumer is test_dashboard_insecure_env_var_no_longer_bypasses at 11.8s of poll_container sleeps.
271 lines
11 KiB
YAML
271 lines
11 KiB
YAML
name: Docker Build, Test, and Publish
|
|
|
|
on:
|
|
release:
|
|
types: [published]
|
|
workflow_call:
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Concurrency: push/release runs are NEVER cancelled so every merge gets
|
|
# its own image. PR runs reuse a PR-scoped group with
|
|
# cancel-in-progress: true so rapid pushes to the same PR collapse to
|
|
# the latest commit.
|
|
concurrency:
|
|
group: docker-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
env:
|
|
IMAGE_NAME: nousresearch/hermes-agent
|
|
|
|
jobs:
|
|
# Build, test, and optionally push the image for each architecture.
|
|
build:
|
|
if: github.repository == 'NousResearch/hermes-agent'
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- arch: amd64
|
|
runner: ubuntu-latest
|
|
platform: linux/amd64
|
|
cache-from: type=gha,scope=docker-amd64
|
|
cache-to: type=gha,mode=max,scope=docker-amd64
|
|
- arch: arm64
|
|
runner: ubuntu-24.04-arm
|
|
platform: linux/arm64
|
|
cache-from: type=gha,scope=docker-arm64
|
|
cache-to: type=gha,mode=max,scope=docker-arm64
|
|
|
|
runs-on: ${{ matrix.runner }}
|
|
timeout-minutes: 45
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
|
|
|
# Build once, load into the local daemon for testing. Cached
|
|
# per-arch; the push step below reuses every layer from this build.
|
|
- name: Build image (${{ matrix.arch }})
|
|
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
|
with:
|
|
context: .
|
|
file: Dockerfile
|
|
load: true
|
|
platforms: ${{ matrix.platform }}
|
|
tags: ${{ env.IMAGE_NAME }}:test
|
|
build-args: |
|
|
HERMES_GIT_SHA=${{ github.sha }}
|
|
cache-from: ${{ matrix.cache-from }}
|
|
cache-to: ${{ (github.event_name != 'pull_request') && matrix.cache-to || '' }}
|
|
|
|
- name: Log in to Docker Hub
|
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release'
|
|
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
|
|
# Push by digest only (no tag). The merge job assembles the
|
|
# tagged manifest list. `push-by-digest=true` is docker's recommended
|
|
# pattern for multi-runner multi-platform builds.
|
|
- name: Push ${{ matrix.arch }} by digest
|
|
id: push
|
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release'
|
|
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
|
with:
|
|
context: .
|
|
file: Dockerfile
|
|
platforms: ${{ matrix.platform }}
|
|
labels: |
|
|
org.opencontainers.image.revision=${{ github.sha }}
|
|
build-args: |
|
|
HERMES_GIT_SHA=${{ github.sha }}
|
|
outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
|
|
cache-from: ${{ matrix.cache-from }}
|
|
cache-to: ${{ matrix.cache-to }}
|
|
|
|
# Write the digest to a file and upload it as an artifact so the
|
|
# merge job can stitch both per-arch digests into a manifest list.
|
|
- name: Export digest
|
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release'
|
|
run: |
|
|
mkdir -p /tmp/digests
|
|
digest="${{ steps.push.outputs.digest }}"
|
|
touch "/tmp/digests/${digest#sha256:}"
|
|
|
|
- name: Upload digest artifact
|
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release'
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: digest-${{ matrix.arch }}
|
|
path: /tmp/digests/*
|
|
if-no-files-found: error
|
|
retention-days: 1
|
|
|
|
# Run the docker-integration test suite against the freshly-built
|
|
# image already loaded into the local daemon (`:test`).
|
|
#
|
|
# Piggybacking here avoids a second image build: the build step
|
|
# already loaded the image into the daemon under
|
|
# `${IMAGE_NAME}:test`, so we just point ``HERMES_TEST_IMAGE`` at
|
|
# that. The fixture's ``HERMES_TEST_IMAGE`` branch (see
|
|
# tests/docker/conftest.py:62-63) short-circuits the rebuild.
|
|
#
|
|
# Why this job and not a standalone one: the image is 5GB+; passing
|
|
# it between jobs via ``docker save``/``upload-artifact`` is slower
|
|
# than the build itself. Reusing the existing daemon state is the
|
|
# cheapest path to coverage on every PR that touches docker code.
|
|
# ---------------------------------------------------------------------
|
|
- name: Install uv (for docker tests)
|
|
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0
|
|
|
|
- name: Set up Python 3.11 (for docker tests)
|
|
run: uv python install 3.11
|
|
|
|
- name: Install Python dependencies (for docker tests)
|
|
run: |
|
|
# ``dev`` extra pulls in pytest, pytest-asyncio —
|
|
# everything tests/docker/ needs. We deliberately avoid ``all``
|
|
# here because the docker tests only drive the container via
|
|
# subprocess and don't import hermes_agent's optional deps.
|
|
uv sync --locked --python 3.11 --extra dev
|
|
|
|
- name: Run docker integration tests
|
|
env:
|
|
# Skip rebuild; use the image already loaded by the build step.
|
|
HERMES_TEST_IMAGE: ${{ env.IMAGE_NAME }}:test
|
|
# Match the policy in tests.yml :: test job — no accidental
|
|
# real-API calls from inside the harness.
|
|
OPENROUTER_API_KEY: ""
|
|
OPENAI_API_KEY: ""
|
|
NOUS_API_KEY: ""
|
|
# Profile every docker subprocess call so we can diagnose why
|
|
# the suite is slow on CI vs. local. Each per-file subprocess
|
|
# writes its own docker-test-profile-<pid>.json; the merge
|
|
# step below combines them into a single report.
|
|
HERMES_DOCKER_TEST_PROFILE: "1"
|
|
run: |
|
|
scripts/run_tests.sh tests/docker/ --file-timeout 600
|
|
|
|
- name: Merge docker test profiles
|
|
if: always()
|
|
run: |
|
|
python3 -c "
|
|
import json, glob, sys
|
|
files = sorted(glob.glob('docker-test-profile-*.json'))
|
|
if not files:
|
|
print('No per-PID profile files found — profiling may not have activated')
|
|
sys.exit(0)
|
|
merged = {'tests': [], 'summary': {}}
|
|
for f in files:
|
|
with open(f) as fh:
|
|
data = json.load(fh)
|
|
merged['tests'].extend(data.get('tests', []))
|
|
# Recompute summary aggregates from merged tests.
|
|
subcmd_totals = {}
|
|
subcmd_counts = {}
|
|
total_sleep = 0
|
|
total_sleeps = 0
|
|
for t in merged['tests']:
|
|
for sub, info in t.get('by_subcommand', {}).items():
|
|
subcmd_totals[sub] = subcmd_totals.get(sub, 0) + info['total_s']
|
|
subcmd_counts[sub] = subcmd_counts.get(sub, 0) + info['count']
|
|
total_sleep += t.get('total_sleep_s', 0)
|
|
total_sleeps += t.get('sleep_count', 0)
|
|
total_docker = sum(subcmd_totals.values())
|
|
merged['summary'] = {
|
|
'total_tests': len(merged['tests']),
|
|
'total_calls': sum(subcmd_counts.values()),
|
|
'total_sleeps': total_sleeps,
|
|
'total_docker_s': round(total_docker, 3),
|
|
'total_sleep_s': round(total_sleep, 3),
|
|
'total_wall_s': round(total_docker + total_sleep, 3),
|
|
'by_subcommand': {
|
|
sub: {
|
|
'count': subcmd_counts[sub],
|
|
'total_s': round(subcmd_totals[sub], 3),
|
|
'avg_s': round(subcmd_totals[sub] / subcmd_counts[sub], 3) if subcmd_counts[sub] else 0,
|
|
}
|
|
for sub in sorted(subcmd_totals, key=lambda s: subcmd_totals[s], reverse=True)
|
|
},
|
|
}
|
|
with open('docker-test-profile.json', 'w') as fh:
|
|
json.dump(merged, fh, indent=2)
|
|
print(f'Merged {len(files)} per-PID profiles ({len(merged[\"tests\"])} tests, {total_docker:.0f}s docker + {total_sleep:.0f}s sleep = {total_docker + total_sleep:.0f}s wall)')
|
|
"
|
|
|
|
- name: Upload docker test profile
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: docker-test-profile-${{ matrix.arch }}
|
|
path: docker-test-profile.json
|
|
retention-days: 14
|
|
if-no-files-found: warn
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stitch both per-arch digests into a single tagged multi-arch manifest.
|
|
# This is a registry-side operation — no building, no layer re-push —
|
|
# so it runs in ~30 seconds.
|
|
#
|
|
# On main pushes: tags both :main and :latest.
|
|
# On releases: tags :<release_tag_name>.
|
|
# ---------------------------------------------------------------------------
|
|
merge:
|
|
if: github.repository == 'NousResearch/hermes-agent' && (github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release')
|
|
runs-on: ubuntu-latest
|
|
needs: [build]
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Download digests
|
|
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
|
with:
|
|
path: /tmp/digests
|
|
pattern: digest-*
|
|
merge-multiple: true
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
|
|
|
- name: Log in to Docker Hub
|
|
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
|
with:
|
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
|
|
- name: Create manifest list and push
|
|
working-directory: /tmp/digests
|
|
env:
|
|
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
|
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
|
run: |
|
|
set -euo pipefail
|
|
args=()
|
|
for digest_file in *; do
|
|
args+=("${IMAGE_NAME}@sha256:${digest_file}")
|
|
done
|
|
if [ "${{ github.event_name }}" = "release" ]; then
|
|
docker buildx imagetools create \
|
|
-t "${IMAGE_NAME}:${RELEASE_TAG}" \
|
|
"${args[@]}"
|
|
else
|
|
docker buildx imagetools create \
|
|
-t "${IMAGE_NAME}:main" \
|
|
-t "${IMAGE_NAME}:latest" \
|
|
"${args[@]}"
|
|
fi
|
|
|
|
- name: Inspect image
|
|
env:
|
|
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
|
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
|
run: |
|
|
if [ "${{ github.event_name }}" = "release" ]; then
|
|
docker buildx imagetools inspect "${IMAGE_NAME}:${RELEASE_TAG}"
|
|
else
|
|
docker buildx imagetools inspect "${IMAGE_NAME}:main"
|
|
fi
|