diff --git a/.github/actions/profile/action.yml b/.github/actions/profile/action.yml new file mode 100644 index 000000000000..e3e0f9b84e8d --- /dev/null +++ b/.github/actions/profile/action.yml @@ -0,0 +1,88 @@ +name: Profile a command (CPU/RAM/Disk) +description: >- + Run a shell command while sampling CPU, RAM, and disk IO every second. + Produces a resource-profile.json artifact per job so the CI timing + report can show per-job resource usage and identify bottlenecks. + +inputs: + command: + description: Shell command to run (and profile). + required: true + label: + description: Label for this profile (e.g. "tests slice 1/8"). + required: true + working-directory: + description: Directory to run in. + default: '.' + +runs: + using: composite + steps: + - name: Start resource profiler + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + # Start profiler in background. It writes to resource-profile.json + # on SIGTERM (or when the command finishes and we signal it). + python3 scripts/ci/resource_profile.py \ + --output resource-profile.json \ + --label "$PROFILE_LABEL" & + echo $! > "$RUNNER_TEMP/profiler.pid" + env: + PROFILE_LABEL: ${{ inputs.label }} + + - name: Run command + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + _CMD: ${{ inputs.command }} + run: | + bash -c "$_CMD" + + - name: Stop profiler and collect results + id: stop-profiler + if: always() + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + if [ -f "$RUNNER_TEMP/profiler.pid" ]; then + PID=$(cat "$RUNNER_TEMP/profiler.pid") + if kill -0 "$PID" 2>/dev/null; then + kill -TERM "$PID" + # Give it a moment to write the JSON + for i in 1 2 3 4 5; do + if kill -0 "$PID" 2>/dev/null; then + sleep 0.2 + else + break + fi + done + kill -KILL "$PID" 2>/dev/null || true + fi + fi + # hashFiles() only matches inside the workspace, so surface file + # existence as a step output instead for the upload condition. + if [ -s resource-profile.json ]; then + echo "profile_written=true" >> "$GITHUB_OUTPUT" + else + echo "profile_written=false" >> "$GITHUB_OUTPUT" + fi + + - name: Sanitize resource profile label + id: sanitize + if: always() + shell: bash + env: + PROFILE_LABEL: ${{ inputs.label }} + run: | + SAFE=$(printf '%s' "$PROFILE_LABEL" | sed -E 's/[^a-zA-Z0-9]+/-/g') + echo "safe_label=$SAFE" >> "$GITHUB_OUTPUT" + + - name: Upload resource profile + if: always() && steps.stop-profiler.outputs.profile_written == 'true' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: resource-profile-${{ steps.sanitize.outputs.safe_label }} + path: ${{ inputs.working-directory }}/resource-profile.json + retention-days: 14 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33f0fedae995..15be11fd75ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -333,6 +333,18 @@ jobs: restore-keys: | ci-timings-baseline- + - name: Download resource profiles + # Advisory — if no profiles were uploaded (e.g. sub-workflows + # didn't run), this step finds nothing and the report still works. + # NOTE: no merge-multiple — every artifact contains a file named + # resource-profile.json, so merging would clobber all but one. + # Per-artifact subdirs are exactly what load_resource_profiles walks. + continue-on-error: true + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: resource-profile-* + path: resource-profiles + - name: Collect timings and generate report env: GITHUB_TOKEN: ${{ github.token }} @@ -341,7 +353,8 @@ jobs: --baseline ci-timings-baseline.json \ --output ci-timings-report.html \ --json-out ci-timings.json \ - --summary-out ci-timings-summary.md + --summary-out ci-timings-summary.md \ + --profiles-dir resource-profiles - name: Upload HTML report # Advisory report — artifact-service blips must not fail the job. @@ -361,6 +374,7 @@ jobs: python3 scripts/ci/timings_report.py \ --from-json ci-timings.json \ --baseline ci-timings-baseline.json \ + --profiles-dir resource-profiles \ --review-status-out review-status.json \ --review-status-only diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 772ee3b67f7c..f7269f7541b4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -107,16 +107,10 @@ jobs: command: uv sync --locked --python 3.11 --extra dev - name: Run docker integration tests - env: - # Skip rebuild; use the image already loaded by the build step. - HERMES_TEST_IMAGE: ${{ env.IMAGE_NAME }}:test - # Match the policy in tests.yml :: test job — no accidental - # real-API calls from inside the harness. - OPENROUTER_API_KEY: "" - OPENAI_API_KEY: "" - NOUS_API_KEY: "" - run: | - scripts/run_tests.sh tests/docker/ --file-timeout 600 + uses: ./.github/actions/profile + with: + label: docker-tests + command: scripts/run_tests.sh tests/docker/ --file-timeout 600 # --------------------------------------------------------------------------- # Rebuild and push each architecture only after the unprivileged build/test diff --git a/.github/workflows/docs-site-checks.yml b/.github/workflows/docs-site-checks.yml index 816c286909a4..419706711bc5 100644 --- a/.github/workflows/docs-site-checks.yml +++ b/.github/workflows/docs-site-checks.yml @@ -45,5 +45,8 @@ jobs: working-directory: website - name: Build Docusaurus - run: npm run build - working-directory: website + uses: ./.github/actions/profile + with: + label: docs-site-build + working-directory: website + command: npm run build diff --git a/.github/workflows/e2e-desktop.yml b/.github/workflows/e2e-desktop.yml index 011de6c7ad1d..389cdc5caf3e 100644 --- a/.github/workflows/e2e-desktop.yml +++ b/.github/workflows/e2e-desktop.yml @@ -87,17 +87,20 @@ jobs: # `npm run test:e2e` builds dist/ as a pretest hook so the renderer # is always fresh — no separate build step needed. - name: Run Playwright E2E tests - working-directory: apps/desktop - run: | - if [ "${{ github.ref_name }}" = "main" ]; then - echo "On main — generating/updating baseline screenshots" - npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \ - npx playwright test --reporter=list --update-snapshots - else - echo "On PR — comparing against cached baselines" - npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \ - npx playwright test --reporter=list - fi + uses: ./.github/actions/profile + with: + label: e2e-desktop + working-directory: apps/desktop + command: | + if [ "${{ github.ref_name }}" = "main" ]; then + echo "On main — generating/updating baseline screenshots" + npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \ + npx playwright test --reporter=list --update-snapshots + else + echo "On PR — comparing against cached baselines" + npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \ + npx playwright test --reporter=list + fi env: CI: "true" # Ensure no real API keys leak into the test env. diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index b78154571950..027f790ce352 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -66,4 +66,7 @@ jobs: - uses: ./.github/actions/retry with: command: npm ci - - run: npm run --prefix ${{ matrix.package }} ${{ matrix.script }} + - uses: ./.github/actions/profile + with: + label: js-${{ matrix.package }}-${{ matrix.script }} + command: npm run --prefix ${{ matrix.package }} ${{ matrix.script }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7e5000680cb1..fffd59c0ecdf 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -148,8 +148,10 @@ jobs: - name: ruff check . # No --exit-zero, no || true. Exit code propagates to the job, # which propagates to the required-check gate. - run: | - ruff check . + uses: ./.github/actions/profile + with: + label: ruff-blocking + command: ruff check . windows-footguns: # Static guardrails on Windows-unsafe Python primitives — os.kill(pid, 0), diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5214eb86f42e..68f2749884e8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -114,11 +114,14 @@ jobs: # # File list is pre-computed by the generate job (--generate-slices) # which runs LPT distribution once and passes the file list to each - # matrix job via --files. Previously each job re-discovered files and - # re-ran LPT independently — redundant N times. - run: | - source .venv/bin/activate - scripts/run_tests.sh --files '${{ matrix.slice.files }}' + # matrix job via --files. Previously each job re-discovered files + # and re-ran LPT independently — redundant N times. + uses: ./.github/actions/profile + with: + label: tests-slice-${{ matrix.slice.index }} + command: | + source .venv/bin/activate + scripts/run_tests.sh --files '${{ matrix.slice.files }}' env: # Ensure tests don't accidentally call real APIs OPENROUTER_API_KEY: "" @@ -226,9 +229,12 @@ jobs: run: uv cache prune --ci - name: Run e2e tests - run: | - source .venv/bin/activate - python -m pytest tests/e2e/ -v --tb=short + uses: ./.github/actions/profile + with: + label: tests-e2e + command: | + source .venv/bin/activate + python -m pytest tests/e2e/ -v --tb=short env: OPENROUTER_API_KEY: "" OPENAI_API_KEY: "" diff --git a/scripts/ci/resource_profile.py b/scripts/ci/resource_profile.py new file mode 100644 index 000000000000..c343195d31be --- /dev/null +++ b/scripts/ci/resource_profile.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""CPU / RAM / disk-IO profiler for CI jobs. + +Runs as a background daemon: samples /proc every second, accumulates +stats, and on SIGTERM (or timeout) writes a JSON summary to the output +path. Pure stdlib — runs on the bare runner Python with zero deps. + +Usage: + python3 scripts/ci/resource_profile.py \\ + --output resource-profile.json \\ + --label "tests slice 1/8" + +The composite action (.github/actions/profile) starts this as a +background process, runs the real command, then signals it to stop. + +Output JSON shape: + { + "label": "tests slice 1/8", + "duration_s": 42.3, + "cpu": { + "avg_usage_pct": 55.2, + "peak_usage_pct": 89.1, + "samples": 42 + }, + "memory": { # USED memory (MemTotal - MemAvailable) + "avg_mb": 512.0, + "peak_mb": 684.3, + "samples": 42 + }, + "disk": { + "total_mb": 12.4, # sectors read+written, whole devices only + "avg_ops_per_s": 5.2, # completed read+write IOs per second + "peak_ops_per_s": 20.1, + "samples": 42 + } + } + +Caveat: /proc/stat, /proc/meminfo, and /proc/diskstats are NODE-wide. +Inside a Kubernetes pod these numbers include neighbor pods sharing the +node — treat them as indicative, not exact per-job attribution. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import signal +import sys +import time + +_SAMPLE_INTERVAL_S = 1.0 +_CLK_TICK = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100 +_PROC_STAT = "/proc/stat" +_PROC_MEMINFO = "/proc/meminfo" +_PROC_DISKSTATS = "/proc/diskstats" + + +def _read_cpu_usage(prev: dict | None) -> tuple[float, dict]: + """Return (usage_pct since prev sample, current_jiffies_dict). + + Reads /proc/stat line 1 (aggregate CPU). usage_pct = non-idle / total. + """ + try: + with open(_PROC_STAT, encoding="ascii") as f: + first_line = f.readline() + except OSError: + return (0.0, prev or {}) + + parts = first_line.split() + if len(parts) < 5: + return (0.0, prev or {}) + + # user, nice, system, idle, iowait, irq, softirq, steal, ... + vals = [int(x) for x in parts[1:]] + idle = vals[3] + (vals[4] if len(vals) > 4 else 0) + total = sum(vals) + cur = {"total": total, "idle": idle} + + if prev and cur["total"] != prev["total"]: + d_total = cur["total"] - prev["total"] + d_idle = cur["idle"] - prev["idle"] + if d_total > 0: + return (max(0.0, (1.0 - d_idle / d_total) * 100.0), cur) + + return (0.0, cur) + + +def _read_mem_mb() -> float: + """Return *used* memory in MB (MemTotal - MemAvailable) from /proc/meminfo. + + Falls back to MemTotal - MemFree on old kernels without MemAvailable. + Note: in a container this is the host/node view, not the cgroup view — + numbers can include neighbor pods on shared nodes. + """ + try: + with open(_PROC_MEMINFO, encoding="ascii") as f: + text = f.read() + except OSError: + return 0.0 + + total_kb = 0 + available_kb = -1 + free_kb = -1 + for line in text.splitlines(): + if line.startswith("MemTotal:"): + total_kb = int(line.split()[1]) + elif line.startswith("MemAvailable:"): + available_kb = int(line.split()[1]) + elif line.startswith("MemFree:"): + free_kb = int(line.split()[1]) + + if total_kb <= 0: + return 0.0 + unused_kb = available_kb if available_kb >= 0 else max(free_kb, 0) + return max(0, total_kb - unused_kb) / 1024.0 + + +def _read_diskstats() -> dict[str, tuple[int, int]]: + """Return {device: (io_ops_completed, sectors_read_plus_written)}. + + We track whole block devices only (skip partitions — track sda not + sda1, nvme0n1 not nvme0n1p1) so IO isn't double-counted. Each + diskstats line: + major minor name reads_completed reads_merged sectors_read time_reading + writes_completed writes_merged sectors_written ... + """ + try: + with open(_PROC_DISKSTATS, encoding="ascii") as f: + lines = f.readlines() + except OSError: + return {} + + result = {} + for line in lines: + parts = line.split() + if len(parts) < 14: + continue + name = parts[2] + # Skip virtual/removable devices + if name.startswith(("loop", "ram", "sr")): + continue + # Skip partitions: sda1, vda2, mmcblk0p1, nvme0n1p1. For devices + # whose base name ends in a digit (nvme0n1, mmcblk0), partitions + # carry a 'p' suffix; for sdX/vdX a bare trailing digit. + if name.startswith(("nvme", "mmcblk")): + if re.search(r"p\d+$", name): + continue + elif name[-1].isdigit(): + continue + reads_completed = int(parts[3]) + writes_completed = int(parts[7]) + sectors = int(parts[5]) + int(parts[9]) + result[name] = (reads_completed + writes_completed, sectors) + + return result + + +def _read_mem_total_mb() -> float: + """Return MemTotal in MB (0.0 if unreadable).""" + try: + with open(_PROC_MEMINFO, encoding="ascii") as f: + for line in f: + if line.startswith("MemTotal:"): + return int(line.split()[1]) / 1024.0 + except OSError: + pass + return 0.0 + + +def run_profiler(output_path: str, label: str, timeout_s: float = 0) -> None: + """Sample resources until SIGTERM or timeout, then write JSON summary.""" + cpu_samples: list[float] = [] + mem_samples: list[float] = [] + disk_prev = _read_diskstats() + disk_total_sectors = 0 + disk_ops_samples: list[float] = [] + + cpu_prev: dict | None = None + start = time.monotonic() + running = [True] # mutable for signal handler + + def _stop(*_): + running[0] = False + + signal.signal(signal.SIGTERM, _stop) + signal.signal(signal.SIGINT, _stop) + + while running[0]: + cpu_pct, cpu_prev = _read_cpu_usage(cpu_prev) + cpu_samples.append(cpu_pct) + + mem_mb = _read_mem_mb() + mem_samples.append(mem_mb) + + disk_cur = _read_diskstats() + delta_sectors = 0 + delta_ops = 0 + for dev, (ops, sectors) in disk_cur.items(): + prev_ops, prev_sectors = disk_prev.get(dev, (ops, sectors)) + delta_sectors += max(0, sectors - prev_sectors) + delta_ops += max(0, ops - prev_ops) + disk_total_sectors += delta_sectors + disk_ops_samples.append(delta_ops / _SAMPLE_INTERVAL_S) + disk_prev = disk_cur + + elapsed = time.monotonic() - start + if timeout_s > 0 and elapsed >= timeout_s: + break + + time.sleep(_SAMPLE_INTERVAL_S) + + duration_s = time.monotonic() - start + n = len(cpu_samples) or 1 + + # Sectors are 512 bytes + disk_read_written_mb = disk_total_sectors * 512 / (1024 * 1024) + + summary = { + "label": label, + "duration_s": round(duration_s, 1), + "cpu": { + "avg_usage_pct": round(sum(cpu_samples) / n, 1), + "peak_usage_pct": round(max(cpu_samples, default=0.0), 1), + "samples": len(cpu_samples), + }, + "memory": { + "avg_mb": round(sum(mem_samples) / n, 1), + "peak_mb": round(max(mem_samples, default=0.0), 1), + "total_mb": round(_read_mem_total_mb(), 1), + "samples": len(mem_samples), + }, + "disk": { + "total_mb": round(disk_read_written_mb, 1), + "avg_ops_per_s": round(sum(disk_ops_samples) / n, 1), + "peak_ops_per_s": round(max(disk_ops_samples, default=0.0), 1), + "samples": len(disk_ops_samples), + }, + } + + with open(output_path, "w", encoding="utf-8") as f: + json.dump(summary, f, indent=2) + print(f"resource_profile: wrote {output_path} ({n} samples, {duration_s:.1f}s)", file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser(description="CI resource profiler") + parser.add_argument("--output", required=True, help="Output JSON path") + parser.add_argument("--label", default="", help="Label for this profile") + parser.add_argument("--timeout", type=float, default=0, + help="Max seconds to run (0 = until SIGTERM)") + args = parser.parse_args() + run_profiler(args.output, args.label, args.timeout) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/timings_report.py b/scripts/ci/timings_report.py index 8a4d1cf669e5..4924fa77e064 100644 --- a/scripts/ci/timings_report.py +++ b/scripts/ci/timings_report.py @@ -19,6 +19,7 @@ Usage: from __future__ import annotations import argparse +import glob import json import os import sys @@ -417,6 +418,153 @@ def compute_stats(timings: dict, baseline: dict | None = None) -> dict: } +# --------------------------------------------------------------------------- +# Resource profile loading + bottleneck analysis +# --------------------------------------------------------------------------- + +def load_resource_profiles(directory: str) -> dict[str, dict]: + """Load all resource-profile-*/resource-profile.json artifacts. + + Returns {label: profile_dict}. Labels are derived from the artifact + directory name (resource-profile-