mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-29 18:46:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

ci: resource profiler

Browse source
This commit is contained in:
ethernet 2026-07-29 04:31:58 -04:00
parent 67c61c3d29
commit ead990a048
11 changed files with 889 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

88
.github/actions/profile/action.yml vendored Normal file
View file

@ -0,0 +1,88 @@
name: Profile a command (CPU/RAM/Disk)
description: >-
Run a shell command while sampling CPU, RAM, and disk IO every second.
Produces a resource-profile.json artifact per job so the CI timing
report can show per-job resource usage and identify bottlenecks.
inputs:
command:
description: Shell command to run (and profile).
required: true
label:
description: Label for this profile (e.g. "tests slice 1/8").
required: true
working-directory:
description: Directory to run in.
default: '.'
runs:
using: composite
steps:
- name: Start resource profiler
shell: bash
working-directory: ${{ inputs.working-directory }}
run: |
# Start profiler in background. It writes to resource-profile.json
# on SIGTERM (or when the command finishes and we signal it).
python3 scripts/ci/resource_profile.py \
--output resource-profile.json \
--label "$PROFILE_LABEL" &
echo $! > "$RUNNER_TEMP/profiler.pid"
env:
PROFILE_LABEL: ${{ inputs.label }}
- name: Run command
shell: bash
working-directory: ${{ inputs.working-directory }}
env:
_CMD: ${{ inputs.command }}
run: |
bash -c "$_CMD"
- name: Stop profiler and collect results
id: stop-profiler
if: always()
shell: bash
working-directory: ${{ inputs.working-directory }}
run: |
if [ -f "$RUNNER_TEMP/profiler.pid" ]; then
PID=$(cat "$RUNNER_TEMP/profiler.pid")
if kill -0 "$PID" 2>/dev/null; then
kill -TERM "$PID"
# Give it a moment to write the JSON
for i in 1 2 3 4 5; do
if kill -0 "$PID" 2>/dev/null; then
sleep 0.2
else
break
fi
done
kill -KILL "$PID" 2>/dev/null || true
fi
fi
# hashFiles() only matches inside the workspace, so surface file
# existence as a step output instead for the upload condition.
if [ -s resource-profile.json ]; then
echo "profile_written=true" >> "$GITHUB_OUTPUT"
else
echo "profile_written=false" >> "$GITHUB_OUTPUT"
fi
- name: Sanitize resource profile label
id: sanitize
if: always()
shell: bash
env:
PROFILE_LABEL: ${{ inputs.label }}
run: |
SAFE=$(printf '%s' "$PROFILE_LABEL" | sed -E 's/[^a-zA-Z0-9]+/-/g')
echo "safe_label=$SAFE" >> "$GITHUB_OUTPUT"
- name: Upload resource profile
if: always() && steps.stop-profiler.outputs.profile_written == 'true'
continue-on-error: true
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: resource-profile-${{ steps.sanitize.outputs.safe_label }}
path: ${{ inputs.working-directory }}/resource-profile.json
retention-days: 14

16
.github/workflows/ci.yml vendored
View file

@ -333,6 +333,18 @@ jobs:
restore-keys: |
ci-timings-baseline-
- name: Download resource profiles
# Advisory — if no profiles were uploaded (e.g. sub-workflows
# didn't run), this step finds nothing and the report still works.
# NOTE: no merge-multiple — every artifact contains a file named
# resource-profile.json, so merging would clobber all but one.
# Per-artifact subdirs are exactly what load_resource_profiles walks.
continue-on-error: true
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
pattern: resource-profile-*
path: resource-profiles
- name: Collect timings and generate report
env:
GITHUB_TOKEN: ${{ github.token }}
@ -341,7 +353,8 @@ jobs:
--baseline ci-timings-baseline.json \
--output ci-timings-report.html \
--json-out ci-timings.json \
--summary-out ci-timings-summary.md
--summary-out ci-timings-summary.md \
--profiles-dir resource-profiles
- name: Upload HTML report
# Advisory report — artifact-service blips must not fail the job.
@ -361,6 +374,7 @@ jobs:
python3 scripts/ci/timings_report.py \
--from-json ci-timings.json \
--baseline ci-timings-baseline.json \
--profiles-dir resource-profiles \
--review-status-out review-status.json \
--review-status-only

14
.github/workflows/docker.yml vendored
View file

@ -107,16 +107,10 @@ jobs:
command: uv sync --locked --python 3.11 --extra dev
- name: Run docker integration tests
env:
# Skip rebuild; use the image already loaded by the build step.
HERMES_TEST_IMAGE: ${{ env.IMAGE_NAME }}:test
# Match the policy in tests.yml :: test job — no accidental
# real-API calls from inside the harness.
OPENROUTER_API_KEY: ""
OPENAI_API_KEY: ""
NOUS_API_KEY: ""
run: |
scripts/run_tests.sh tests/docker/ --file-timeout 600
uses: ./.github/actions/profile
with:
label: docker-tests
command: scripts/run_tests.sh tests/docker/ --file-timeout 600
# ---------------------------------------------------------------------------
# Rebuild and push each architecture only after the unprivileged build/test

7
.github/workflows/docs-site-checks.yml vendored
View file

@ -45,5 +45,8 @@ jobs:
working-directory: website
- name: Build Docusaurus
run: npm run build
working-directory: website
uses: ./.github/actions/profile
with:
label: docs-site-build
working-directory: website
command: npm run build

25
.github/workflows/e2e-desktop.yml vendored
View file

@ -87,17 +87,20 @@ jobs:
# `npm run test:e2e` builds dist/ as a pretest hook so the renderer
# is always fresh — no separate build step needed.
- name: Run Playwright E2E tests
working-directory: apps/desktop
run: |
if [ "${{ github.ref_name }}" = "main" ]; then
echo "On main — generating/updating baseline screenshots"
npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \
npx playwright test --reporter=list --update-snapshots
else
echo "On PR — comparing against cached baselines"
npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \
npx playwright test --reporter=list
fi
uses: ./.github/actions/profile
with:
label: e2e-desktop
working-directory: apps/desktop
command: |
if [ "${{ github.ref_name }}" = "main" ]; then
echo "On main — generating/updating baseline screenshots"
npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \
npx playwright test --reporter=list --update-snapshots
else
echo "On PR — comparing against cached baselines"
npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \
npx playwright test --reporter=list
fi
env:
CI: "true"
# Ensure no real API keys leak into the test env.

5
.github/workflows/js-tests.yml vendored
View file

@ -66,4 +66,7 @@ jobs:
- uses: ./.github/actions/retry
with:
command: npm ci
- run: npm run --prefix ${{ matrix.package }} ${{ matrix.script }}
- uses: ./.github/actions/profile
with:
label: js-${{ matrix.package }}-${{ matrix.script }}
command: npm run --prefix ${{ matrix.package }} ${{ matrix.script }}

6
.github/workflows/lint.yml vendored
View file

@ -148,8 +148,10 @@ jobs:
- name: ruff check .
# No --exit-zero, no || true. Exit code propagates to the job,
# which propagates to the required-check gate.
run: |
ruff check .
uses: ./.github/actions/profile
with:
label: ruff-blocking
command: ruff check .
windows-footguns:
# Static guardrails on Windows-unsafe Python primitives — os.kill(pid, 0),

22
.github/workflows/tests.yml vendored
View file

@ -114,11 +114,14 @@ jobs:
#
# File list is pre-computed by the generate job (--generate-slices)
# which runs LPT distribution once and passes the file list to each
# matrix job via --files. Previously each job re-discovered files and
# re-ran LPT independently — redundant N times.
run: |
source .venv/bin/activate
scripts/run_tests.sh --files '${{ matrix.slice.files }}'
# matrix job via --files. Previously each job re-discovered files
# and re-ran LPT independently — redundant N times.
uses: ./.github/actions/profile
with:
label: tests-slice-${{ matrix.slice.index }}
command: |
source .venv/bin/activate
scripts/run_tests.sh --files '${{ matrix.slice.files }}'
env:
# Ensure tests don't accidentally call real APIs
OPENROUTER_API_KEY: ""
@ -226,9 +229,12 @@ jobs:
run: uv cache prune --ci
- name: Run e2e tests
run: |
source .venv/bin/activate
python -m pytest tests/e2e/ -v --tb=short
uses: ./.github/actions/profile
with:
label: tests-e2e
command: |
source .venv/bin/activate
python -m pytest tests/e2e/ -v --tb=short
env:
OPENROUTER_API_KEY: ""
OPENAI_API_KEY: ""

258
scripts/ci/resource_profile.py Normal file
View file

@ -0,0 +1,258 @@
#!/usr/bin/env python3
"""CPU / RAM / disk-IO profiler for CI jobs.
Runs as a background daemon: samples /proc every second, accumulates
stats, and on SIGTERM (or timeout) writes a JSON summary to the output
path. Pure stdlib — runs on the bare runner Python with zero deps.
Usage:
python3 scripts/ci/resource_profile.py \\
--output resource-profile.json \\
--label "tests slice 1/8"
The composite action (.github/actions/profile) starts this as a
background process, runs the real command, then signals it to stop.
Output JSON shape:
{
"label": "tests slice 1/8",
"duration_s": 42.3,
"cpu": {
"avg_usage_pct": 55.2,
"peak_usage_pct": 89.1,
"samples": 42
},
"memory": { # USED memory (MemTotal - MemAvailable)
"avg_mb": 512.0,
"peak_mb": 684.3,
"samples": 42
},
"disk": {
"total_mb": 12.4, # sectors read+written, whole devices only
"avg_ops_per_s": 5.2, # completed read+write IOs per second
"peak_ops_per_s": 20.1,
"samples": 42
}
}
Caveat: /proc/stat, /proc/meminfo, and /proc/diskstats are NODE-wide.
Inside a Kubernetes pod these numbers include neighbor pods sharing the
node — treat them as indicative, not exact per-job attribution.
"""
from __future__ import annotations
import argparse
import json
import os
import re
import signal
import sys
import time
_SAMPLE_INTERVAL_S = 1.0
_CLK_TICK = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100
_PROC_STAT = "/proc/stat"
_PROC_MEMINFO = "/proc/meminfo"
_PROC_DISKSTATS = "/proc/diskstats"
def _read_cpu_usage(prev: dict | None) -> tuple[float, dict]:
"""Return (usage_pct since prev sample, current_jiffies_dict).
Reads /proc/stat line 1 (aggregate CPU). usage_pct = non-idle / total.
"""
try:
with open(_PROC_STAT, encoding="ascii") as f:
first_line = f.readline()
except OSError:
return (0.0, prev or {})
parts = first_line.split()
if len(parts) < 5:
return (0.0, prev or {})
# user, nice, system, idle, iowait, irq, softirq, steal, ...
vals = [int(x) for x in parts[1:]]
idle = vals[3] + (vals[4] if len(vals) > 4 else 0)
total = sum(vals)
cur = {"total": total, "idle": idle}
if prev and cur["total"] != prev["total"]:
d_total = cur["total"] - prev["total"]
d_idle = cur["idle"] - prev["idle"]
if d_total > 0:
return (max(0.0, (1.0 - d_idle / d_total) * 100.0), cur)
return (0.0, cur)
def _read_mem_mb() -> float:
"""Return *used* memory in MB (MemTotal - MemAvailable) from /proc/meminfo.
Falls back to MemTotal - MemFree on old kernels without MemAvailable.
Note: in a container this is the host/node view, not the cgroup view —
numbers can include neighbor pods on shared nodes.
"""
try:
with open(_PROC_MEMINFO, encoding="ascii") as f:
text = f.read()
except OSError:
return 0.0
total_kb = 0
available_kb = -1
free_kb = -1
for line in text.splitlines():
if line.startswith("MemTotal:"):
total_kb = int(line.split()[1])
elif line.startswith("MemAvailable:"):
available_kb = int(line.split()[1])
elif line.startswith("MemFree:"):
free_kb = int(line.split()[1])
if total_kb <= 0:
return 0.0
unused_kb = available_kb if available_kb >= 0 else max(free_kb, 0)
return max(0, total_kb - unused_kb) / 1024.0
def _read_diskstats() -> dict[str, tuple[int, int]]:
"""Return {device: (io_ops_completed, sectors_read_plus_written)}.
We track whole block devices only (skip partitions — track sda not
sda1, nvme0n1 not nvme0n1p1) so IO isn't double-counted. Each
diskstats line:
major minor name reads_completed reads_merged sectors_read time_reading
writes_completed writes_merged sectors_written ...
"""
try:
with open(_PROC_DISKSTATS, encoding="ascii") as f:
lines = f.readlines()
except OSError:
return {}
result = {}
for line in lines:
parts = line.split()
if len(parts) < 14:
continue
name = parts[2]
# Skip virtual/removable devices
if name.startswith(("loop", "ram", "sr")):
continue
# Skip partitions: sda1, vda2, mmcblk0p1, nvme0n1p1. For devices
# whose base name ends in a digit (nvme0n1, mmcblk0), partitions
# carry a 'p<N>' suffix; for sdX/vdX a bare trailing digit.
if name.startswith(("nvme", "mmcblk")):
if re.search(r"p\d+$", name):
continue
elif name[-1].isdigit():
continue
reads_completed = int(parts[3])
writes_completed = int(parts[7])
sectors = int(parts[5]) + int(parts[9])
result[name] = (reads_completed + writes_completed, sectors)
return result
def _read_mem_total_mb() -> float:
"""Return MemTotal in MB (0.0 if unreadable)."""
try:
with open(_PROC_MEMINFO, encoding="ascii") as f:
for line in f:
if line.startswith("MemTotal:"):
return int(line.split()[1]) / 1024.0
except OSError:
pass
return 0.0
def run_profiler(output_path: str, label: str, timeout_s: float = 0) -> None:
"""Sample resources until SIGTERM or timeout, then write JSON summary."""
cpu_samples: list[float] = []
mem_samples: list[float] = []
disk_prev = _read_diskstats()
disk_total_sectors = 0
disk_ops_samples: list[float] = []
cpu_prev: dict | None = None
start = time.monotonic()
running = [True] # mutable for signal handler
def _stop(*_):
running[0] = False
signal.signal(signal.SIGTERM, _stop)
signal.signal(signal.SIGINT, _stop)
while running[0]:
cpu_pct, cpu_prev = _read_cpu_usage(cpu_prev)
cpu_samples.append(cpu_pct)
mem_mb = _read_mem_mb()
mem_samples.append(mem_mb)
disk_cur = _read_diskstats()
delta_sectors = 0
delta_ops = 0
for dev, (ops, sectors) in disk_cur.items():
prev_ops, prev_sectors = disk_prev.get(dev, (ops, sectors))
delta_sectors += max(0, sectors - prev_sectors)
delta_ops += max(0, ops - prev_ops)
disk_total_sectors += delta_sectors
disk_ops_samples.append(delta_ops / _SAMPLE_INTERVAL_S)
disk_prev = disk_cur
elapsed = time.monotonic() - start
if timeout_s > 0 and elapsed >= timeout_s:
break
time.sleep(_SAMPLE_INTERVAL_S)
duration_s = time.monotonic() - start
n = len(cpu_samples) or 1
# Sectors are 512 bytes
disk_read_written_mb = disk_total_sectors * 512 / (1024 * 1024)
summary = {
"label": label,
"duration_s": round(duration_s, 1),
"cpu": {
"avg_usage_pct": round(sum(cpu_samples) / n, 1),
"peak_usage_pct": round(max(cpu_samples, default=0.0), 1),
"samples": len(cpu_samples),
},
"memory": {
"avg_mb": round(sum(mem_samples) / n, 1),
"peak_mb": round(max(mem_samples, default=0.0), 1),
"total_mb": round(_read_mem_total_mb(), 1),
"samples": len(mem_samples),
},
"disk": {
"total_mb": round(disk_read_written_mb, 1),
"avg_ops_per_s": round(sum(disk_ops_samples) / n, 1),
"peak_ops_per_s": round(max(disk_ops_samples, default=0.0), 1),
"samples": len(disk_ops_samples),
},
}
with open(output_path, "w", encoding="utf-8") as f:
json.dump(summary, f, indent=2)
print(f"resource_profile: wrote {output_path} ({n} samples, {duration_s:.1f}s)", file=sys.stderr)
def main():
parser = argparse.ArgumentParser(description="CI resource profiler")
parser.add_argument("--output", required=True, help="Output JSON path")
parser.add_argument("--label", default="", help="Label for this profile")
parser.add_argument("--timeout", type=float, default=0,
help="Max seconds to run (0 = until SIGTERM)")
args = parser.parse_args()
run_profiler(args.output, args.label, args.timeout)
if __name__ == "__main__":
main()

232
scripts/ci/timings_report.py
View file

@ -19,6 +19,7 @@ Usage:
from __future__ import annotations
import argparse
import glob
import json
import os
import sys
@ -417,6 +418,153 @@ def compute_stats(timings: dict, baseline: dict | None = None) -> dict:
}
# ---------------------------------------------------------------------------
# Resource profile loading + bottleneck analysis
# ---------------------------------------------------------------------------
def load_resource_profiles(directory: str) -> dict[str, dict]:
"""Load all resource-profile-*/resource-profile.json artifacts.
Returns {label: profile_dict}. Labels are derived from the artifact
directory name (resource-profile-<label> → <label>).
"""
profiles: dict[str, dict] = {}
if not directory or not os.path.isdir(directory):
return profiles
for path in glob.glob(os.path.join(directory, "**", "resource-profile.json"), recursive=True):
try:
with open(path, encoding="utf-8") as f:
profile = json.load(f)
except (json.JSONDecodeError, OSError):
continue
label = profile.get("label") or os.path.basename(os.path.dirname(path))
profiles[label] = profile
return profiles
def classify_bottleneck(timings: dict, profiles: dict[str, dict]) -> str:
"""Return a one-line bottleneck classification.
Examines wall time, compute time, wait time, and resource profiles
to identify the dominant constraint.
Possible verdicts:
- "CPU-bound: <job> at <pct>% CPU for <dur>"
- "Memory-bound: <job> peaked at <mb> MB"
- "Disk IO-bound: <job> at <ops>/s for <dur>"
- "Wait-bound: <wait>s idle waiting for dependencies"
- "Evenly distributed: no single bottleneck"
- "Insufficient data: <reason>"
"""
stats = compute_stats(timings, None)
jobs = [j for j in timings.get("jobs", []) if not is_skipped(j)]
if not jobs:
return "Insufficient data: no jobs in timings"
# --- Wait-bound: if total wait > 50% of wall time ---
wall = stats["wall"]
total_wait = stats["total_wait"]
if wall > 0 and total_wait > 0:
wait_pct = total_wait / wall * 100
if wait_pct > 50:
return (f"Wait-bound: {fmt_dur(total_wait)} idle "
f"({wait_pct:.0f}% of {fmt_dur(wall)} wall) waiting for dependencies")
# --- Resource-bound: check profiles for CPU/mem/disk extremes ---
if profiles:
cpu_hotspot = None
mem_hotspot = None
disk_hotspot = None
max_cpu = 0.0
max_mem_frac = 0.0
max_disk_ops = 0.0
for label, p in profiles.items():
cpu_info = p.get("cpu", {})
mem_info = p.get("memory", {})
disk_info = p.get("disk", {})
cpu_avg = cpu_info.get("avg_usage_pct", 0)
cpu_peak = cpu_info.get("peak_usage_pct", 0)
if cpu_avg > max_cpu:
max_cpu = cpu_avg
cpu_hotspot = (label, cpu_avg, cpu_peak, p.get("duration_s", 0))
# Memory is USED MB. Compare against the machine's total when the
# profile carries it; otherwise fall back to an absolute floor.
mem_peak = mem_info.get("peak_mb", 0)
mem_total = mem_info.get("total_mb", 0)
mem_frac = (mem_peak / mem_total) if mem_total > 0 else (mem_peak / 16000.0)
if mem_frac > max_mem_frac:
max_mem_frac = mem_frac
mem_hotspot = (label, mem_peak, mem_total)
disk_ops = disk_info.get("avg_ops_per_s", 0)
disk_mb = disk_info.get("total_mb", 0)
if disk_ops > max_disk_ops:
max_disk_ops = disk_ops
disk_hotspot = (label, disk_ops, disk_mb, p.get("duration_s", 0))
# Classify: pick the most extreme dimension.
# Each candidate's sort key is normalized to roughly 0-100 so the
# dimensions are comparable:
# CPU — avg usage pct (bound when > 80)
# Disk — avg completed IOs/s / 10 (bound when > 500 ops/s)
# Mem — peak used as pct of total (bound when > 85%)
candidates = []
if cpu_hotspot and cpu_hotspot[1] > 80:
candidates.append((
cpu_hotspot[1], # sort key
f"CPU-bound: {cpu_hotspot[0]} at {cpu_hotspot[1]:.0f}% avg CPU "
f"(peak {cpu_hotspot[2]:.0f}%) for {fmt_dur(cpu_hotspot[3])}"
))
if disk_hotspot and disk_hotspot[1] > 500:
candidates.append((
disk_hotspot[1] / 10,
f"Disk IO-bound: {disk_hotspot[0]} at {disk_hotspot[1]:.0f} ops/s "
f"({disk_hotspot[2]:.0f} MB total) for {fmt_dur(disk_hotspot[3])}"
))
if mem_hotspot and max_mem_frac > 0.85:
total_note = f" of {mem_hotspot[2]:.0f} MB" if mem_hotspot[2] else ""
candidates.append((
max_mem_frac * 100,
f"Memory-bound: {mem_hotspot[0]} peaked at "
f"{mem_hotspot[1]:.0f} MB used{total_note}"
))
if candidates:
candidates.sort(reverse=True)
return candidates[0][1]
# --- No resource profiles, but check wall vs compute ---
compute = stats["compute"]
if wall > 0 and compute > 0:
parallelism = compute / wall
if parallelism < 1.2 and len(jobs) > 2:
# Low parallelism ratio means jobs are serial
slowest = max(jobs, key=lambda j: j.get("duration_s") or 0)
slow_dur = slowest.get("duration_s") or 0
slow_pct = slow_dur / wall * 100 if wall > 0 else 0
if slow_pct > 40:
return (f"Serial bottleneck: {slowest['name']} takes "
f"{fmt_dur(slow_dur)} ({slow_pct:.0f}% of wall)")
# --- Fallback: the single slowest job ---
slowest = max(jobs, key=lambda j: j.get("duration_s") or 0)
slow_dur = slowest.get("duration_s") or 0
if slow_dur > 0 and wall > 0:
slow_pct = slow_dur / wall * 100
if slow_pct > 40 and len(jobs) > 1:
return (f"Dominated by {slowest['name']}: "
f"{fmt_dur(slow_dur)} ({slow_pct:.0f}% of wall)")
return "Evenly distributed: no single bottleneck"
# ---------------------------------------------------------------------------
# HTML generation
# ---------------------------------------------------------------------------
@ -810,7 +958,56 @@ def _regressions(timings: dict, baseline: dict | None) -> str:
)
def generate_html(timings: dict, baseline: dict | None = None) -> str:
def _resource_table(profiles: dict[str, dict]) -> str:
"""Render per-job resource usage as an HTML table."""
if not profiles:
return ""
rows = []
for label in sorted(profiles):
p = profiles[label]
cpu = p.get("cpu", {})
mem = p.get("memory", {})
disk = p.get("disk", {})
rows.append(
f'<tr>'
f'<td class="job-name">{escape(label)}</td>'
f'<td class="num">{fmt_dur(p.get("duration_s"))}</td>'
f'<td class="num">{cpu.get("avg_usage_pct", 0):.0f}%</td>'
f'<td class="num">{cpu.get("peak_usage_pct", 0):.0f}%</td>'
f'<td class="num">{mem.get("avg_mb", 0):.0f}</td>'
f'<td class="num">{mem.get("peak_mb", 0):.0f}</td>'
f'<td class="num">{disk.get("total_mb", 0):.0f}</td>'
f'<td class="num">{disk.get("avg_ops_per_s", 0):.0f}</td>'
f'</tr>'
)
return (
'<table><thead><tr>'
'<th>Job</th><th class="num">Duration</th>'
'<th class="num">CPU avg</th><th class="num">CPU peak</th>'
'<th class="num">Mem avg (MB)</th><th class="num">Mem peak (MB)</th>'
'<th class="num">Disk (MB)</th><th class="num">Disk ops/s</th>'
'</tr></thead><tbody>' + "".join(rows) + '</tbody></table>'
)
def _bottleneck_box(timings: dict, profiles: dict[str, dict]) -> str:
"""Render the bottleneck analysis as a callout box."""
verdict = classify_bottleneck(timings, profiles)
return (
f'<div style="background:#161b22;border:1px solid #30363d;'
f'border-radius:8px;padding:16px;margin-bottom:24px">'
f'<div style="font-size:12px;color:#8b949e;text-transform:uppercase;'
f'letter-spacing:0.5px;margin-bottom:4px">Bottleneck Analysis</div>'
f'<div style="font-size:16px;font-weight:500">{escape(verdict)}</div>'
f'</div>'
)
def generate_html(timings: dict, baseline: dict | None = None,
profiles: dict[str, dict] | None = None) -> str:
stats = compute_stats(timings, baseline)
sha_short = (timings.get("head_sha") or "")[:7]
@ -837,6 +1034,12 @@ def generate_html(timings: dict, baseline: dict | None = None) -> str:
html += '<h2>Global Stats</h2>\n'
html += _stats_cards(stats)
html += _bottleneck_box(timings, profiles or {})
if profiles:
html += '<h2>Resource Usage</h2>\n'
html += _resource_table(profiles)
if baseline:
html += '<h2>Top Regressions & Improvements</h2>\n'
html += _regressions(timings, baseline)
@ -858,12 +1061,18 @@ def generate_html(timings: dict, baseline: dict | None = None) -> str:
# Markdown summary for $GITHUB_STEP_SUMMARY
# ---------------------------------------------------------------------------
def generate_summary(timings: dict, baseline: dict | None = None) -> str:
def generate_summary(timings: dict, baseline: dict | None = None,
profiles: dict[str, dict] | None = None) -> str:
stats = compute_stats(timings, baseline)
bl_map = {j["name"]: j for j in (baseline or {}).get("jobs", [])}
lines = ["## CI Timing Summary\n"]
# Bottleneck analysis
bottleneck = classify_bottleneck(timings, profiles or {})
lines.append(f"**Bottleneck:** {bottleneck}")
lines.append("")
# Global stats table
lines.append("| Metric | Current | Baseline | Delta |")
lines.append("|--------|---------|----------|-------|")
@ -905,7 +1114,8 @@ _TIMINGS_WARN_PCT = 0.25
def generate_review_status(
timings: dict, baseline: dict | None, report_url: str | None = None
timings: dict, baseline: dict | None, report_url: str | None = None,
profiles: dict[str, dict] | None = None
) -> list[dict]:
"""Produce a review_status JSON array for the CI timings review section.
@ -916,6 +1126,7 @@ def generate_review_status(
fragment.
"""
stats = compute_stats(timings, baseline)
bottleneck = classify_bottleneck(timings, profiles or {})
if baseline is None:
severity = "debug"
@ -942,6 +1153,8 @@ def generate_review_status(
wall_str += f" {stats['unchanged']} unchanged."
summary = wall_str
summary += f" Bottleneck: {bottleneck}"
# Per-job delta detail (top 5 by absolute change)
detail_lines: list[str] = []
if baseline:
@ -1001,8 +1214,13 @@ def main():
help="If set, write a review-status JSON for the unified PR comment.")
parser.add_argument("--review-status-only", action="store_true",
help="Write review status from existing timings without regenerating the report.")
parser.add_argument("--profiles-dir", default="",
help="Directory of downloaded resource-profile-* artifacts.")
args = parser.parse_args()
# Load resource profiles (available in both API and --from-json modes)
profiles = load_resource_profiles(args.profiles_dir) if args.profiles_dir else {}
# Collect or load timings
if args.from_json:
with open(args.from_json, encoding="utf-8") as f:
@ -1051,20 +1269,20 @@ def main():
if not args.review_status_out:
parser.error("--review-status-only requires --review-status-out")
report_url = os.environ.get("CI_TIMINGS_REPORT_URL", "")
statuses = generate_review_status(timings, baseline, report_url)
statuses = generate_review_status(timings, baseline, report_url, profiles)
with open(args.review_status_out, "w", encoding="utf-8") as f:
f.write(f"review_status={json.dumps(statuses)}\n")
print(f"Wrote review status to {args.review_status_out}")
return
# Generate HTML
html = generate_html(timings, baseline)
html = generate_html(timings, baseline, profiles)
with open(args.output, "w", encoding="utf-8") as f:
f.write(html)
print(f"Generated HTML report: {args.output}")
# Write summary
summary = generate_summary(timings, baseline)
summary = generate_summary(timings, baseline, profiles)
with open(args.summary_out, "a", encoding="utf-8") as f:
f.write(summary)
print(f"Wrote summary to {args.summary_out}")
@ -1074,7 +1292,7 @@ def main():
# format) so the ci-timings job can expose it as a workflow_call output.
if args.review_status_out:
report_url = os.environ.get("CI_TIMINGS_REPORT_URL", "")
statuses = generate_review_status(timings, baseline, report_url)
statuses = generate_review_status(timings, baseline, report_url, profiles)
json_str = json.dumps(statuses)
with open(args.review_status_out, "a", encoding="utf-8") as f:
f.write(f"review_status={json_str}\n")

258
tests/ci/test_resource_profiles.py Normal file
View file

@ -0,0 +1,258 @@
"""Tests for resource profile loading + bottleneck classification in timings_report.py."""
from __future__ import annotations
import importlib.util
import json
import os
import tempfile
from datetime import datetime, timezone
from pathlib import Path
_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "timings_report.py"
_spec = importlib.util.spec_from_file_location("timings_report", _PATH)
if _spec is None or _spec.loader is None:
raise ImportError("Failed to load timings_report.py")
_mod = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(_mod)
_T0 = datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
def _ts(seconds: float) -> str:
dt = _T0.timestamp() + seconds
return datetime.fromtimestamp(dt, tz=timezone.utc).isoformat().replace("+00:00", "Z")
def _job(name: str, dur_s: float, start_s: float = 0.0, conclusion: str = "success") -> dict:
return {
"name": name,
"duration_s": dur_s,
"conclusion": conclusion,
"started_at": _ts(start_s),
"completed_at": _ts(start_s + dur_s),
"wait_s": 0.0,
}
def _timings(jobs: list[dict]) -> dict:
return {"run_id": "123", "head_sha": "abc", "created_at": "", "jobs": jobs}
def _profile(label: str, cpu_avg: float = 50.0, cpu_peak: float = 80.0,
mem_peak: float = 500.0, mem_total: float = 16000.0,
disk_ops: float = 10.0, disk_mb: float = 5.0,
duration_s: float = 60.0) -> dict:
return {
"label": label,
"duration_s": duration_s,
"cpu": {"avg_usage_pct": cpu_avg, "peak_usage_pct": cpu_peak, "samples": 60},
"memory": {"avg_mb": mem_peak * 0.8, "peak_mb": mem_peak,
"total_mb": mem_total, "samples": 60},
"disk": {"total_mb": disk_mb, "avg_ops_per_s": disk_ops,
"peak_ops_per_s": disk_ops * 2, "samples": 60},
}
# ── load_resource_profiles ──────────────────────────────────────────────
def test_load_profiles_from_directory():
with tempfile.TemporaryDirectory() as d:
# Simulate artifact download structure: resource-profiles/<name>/resource-profile.json
p1 = os.path.join(d, "resource-profile-tests-slice-1")
os.makedirs(p1)
with open(os.path.join(p1, "resource-profile.json"), "w") as f:
json.dump(_profile("tests-slice-1"), f)
p2 = os.path.join(d, "resource-profile-ruff-blocking")
os.makedirs(p2)
with open(os.path.join(p2, "resource-profile.json"), "w") as f:
json.dump(_profile("ruff-blocking"), f)
profiles = _mod.load_resource_profiles(d)
assert len(profiles) == 2
assert "tests-slice-1" in profiles
assert "ruff-blocking" in profiles
assert profiles["tests-slice-1"]["cpu"]["avg_usage_pct"] == 50.0
def test_load_profiles_empty_dir():
with tempfile.TemporaryDirectory() as d:
profiles = _mod.load_resource_profiles(d)
assert profiles == {}
def test_load_profiles_nonexistent_dir():
profiles = _mod.load_resource_profiles("/nonexistent/path/xyz")
assert profiles == {}
def test_load_profiles_malformed_json_skipped():
with tempfile.TemporaryDirectory() as d:
p1 = os.path.join(d, "resource-profile-bad")
os.makedirs(p1)
with open(os.path.join(p1, "resource-profile.json"), "w") as f:
f.write("not json {{{")
profiles = _mod.load_resource_profiles(d)
assert profiles == {}
# ── classify_bottleneck ─────────────────────────────────────────────────
def test_bottleneck_no_profiles_evenly_distributed():
"""Multiple short jobs, no profiles → evenly distributed."""
t = _timings([
_job("a", 30.0),
_job("b", 30.0, start_s=30.0),
_job("c", 30.0, start_s=60.0),
])
verdict = _mod.classify_bottleneck(t, {})
assert "evenly distributed" in verdict.lower()
def test_bottleneck_cpu_bound():
"""A job with >80% avg CPU → CPU-bound."""
t = _timings([_job("compile", 120.0)])
profiles = {"compile": _profile("compile", cpu_avg=95.0, cpu_peak=99.0, duration_s=120.0)}
verdict = _mod.classify_bottleneck(t, profiles)
assert "cpu" in verdict.lower()
assert "compile" in verdict
assert "95" in verdict
def test_bottleneck_disk_io_bound():
"""A job with >500 completed disk IOs/s → disk IO-bound."""
t = _timings([_job("io-heavy", 60.0)])
profiles = {"io-heavy": _profile("io-heavy", cpu_avg=30.0, disk_ops=900.0, disk_mb=500.0, duration_s=60.0)}
verdict = _mod.classify_bottleneck(t, profiles)
assert "disk" in verdict.lower()
assert "io-heavy" in verdict
def test_bottleneck_memory_bound():
"""A job using >85% of total memory → memory-bound."""
t = _timings([_job("big-mem", 60.0)])
profiles = {"big-mem": _profile("big-mem", cpu_avg=30.0, mem_peak=14500.0,
mem_total=16000.0, duration_s=60.0)}
verdict = _mod.classify_bottleneck(t, profiles)
assert "memory" in verdict.lower()
assert "big-mem" in verdict
assert "14500" in verdict
def test_bottleneck_memory_fallback_floor_without_total():
"""Profiles without total_mb fall back to a 16 GB floor."""
t = _timings([_job("big-mem", 60.0)])
p = _profile("big-mem", cpu_avg=30.0, mem_peak=15000.0, duration_s=60.0)
del p["memory"]["total_mb"]
verdict = _mod.classify_bottleneck(t, {"big-mem": p})
assert "memory" in verdict.lower()
def test_bottleneck_moderate_used_memory_not_memory_bound():
"""Regression: moderate absolute usage on a big node must NOT flag as
memory-bound (the old absolute >6000 MB threshold misfired on any
box with lots of RAM)."""
t = _timings([_job("normal", 60.0)])
profiles = {"normal": _profile("normal", cpu_avg=95.0, cpu_peak=99.0,
mem_peak=8000.0, mem_total=32000.0,
duration_s=60.0)}
verdict = _mod.classify_bottleneck(t, profiles)
assert "memory" not in verdict.lower()
assert "cpu" in verdict.lower()
def test_bottleneck_wait_bound():
"""Total wait > 50% of wall time → wait-bound."""
# Job A runs 0-10s, Job B waits 100s then runs 10s (wait=100s, wall=110s)
t = _timings([
_job("fast", 10.0, start_s=0.0),
_job("delayed", 10.0, start_s=100.0),
])
# Set wait_s on the delayed job
t["jobs"][1]["wait_s"] = 100.0
verdict = _mod.classify_bottleneck(t, {})
assert "wait" in verdict.lower()
def test_bottleneck_serial_no_profiles():
"""No profiles, low parallelism, one job dominates → serial bottleneck."""
t = _timings([
_job("slow", 300.0, start_s=0.0),
_job("fast1", 10.0, start_s=0.0),
_job("fast2", 10.0, start_s=0.0),
])
verdict = _mod.classify_bottleneck(t, {})
assert "slow" in verdict.lower()
def test_bottleneck_no_jobs():
"""Empty timings → insufficient data."""
t = _timings([])
verdict = _mod.classify_bottleneck(t, {})
assert "insufficient" in verdict.lower()
def test_bottleneck_cpu_wins_over_disk():
"""When both CPU and disk are high, CPU should win (higher sort key)."""
t = _timings([_job("heavy", 120.0)])
profiles = {"heavy": _profile("heavy", cpu_avg=95.0, disk_ops=600.0, duration_s=120.0)}
verdict = _mod.classify_bottleneck(t, profiles)
assert "cpu" in verdict.lower()
# ── generate_summary includes bottleneck ─────────────────────────────────
def test_summary_includes_bottleneck():
t = _timings([_job("tests", 60.0)])
summary = _mod.generate_summary(t, None)
assert "Bottleneck:" in summary
def test_summary_includes_bottleneck_with_profiles():
t = _timings([_job("compile", 120.0)])
profiles = {"compile": _profile("compile", cpu_avg=95.0, duration_s=120.0)}
summary = _mod.generate_summary(t, None, profiles)
assert "Bottleneck:" in summary
assert "CPU" in summary
# ── generate_review_status includes bottleneck ───────────────────────────
def test_review_status_includes_bottleneck():
t = _timings([_job("tests", 60.0)])
statuses = _mod.generate_review_status(t, None)
result = statuses[0]["results"][0]
assert "Bottleneck:" in result["summary"]
def test_review_status_includes_bottleneck_with_profiles():
t = _timings([_job("compile", 120.0)])
profiles = {"compile": _profile("compile", cpu_avg=95.0, duration_s=120.0)}
statuses = _mod.generate_review_status(t, None, profiles=profiles)
result = statuses[0]["results"][0]
assert "Bottleneck:" in result["summary"]
assert "CPU" in result["summary"]
# ── generate_html includes resource sections ─────────────────────────────
def test_html_includes_bottleneck_box():
t = _timings([_job("tests", 60.0)])
html = _mod.generate_html(t, None)
assert "Bottleneck Analysis" in html
def test_html_includes_resource_table_with_profiles():
t = _timings([_job("compile", 120.0)])
profiles = {"compile": _profile("compile", cpu_avg=95.0, duration_s=120.0)}
html = _mod.generate_html(t, None, profiles)
assert "Resource Usage" in html
assert "compile" in html
assert "95%" in html
def test_html_no_resource_table_without_profiles():
t = _timings([_job("tests", 60.0)])
html = _mod.generate_html(t, None)
assert "Resource Usage" not in html