mirrors/hermes-agent
1
0
Fork 0
mirror of https://github.com/NousResearch/hermes-agent.git synced 2026-07-31 19:16:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

ci: resource profiler

Browse source
This commit is contained in:
ethernet 2026-07-29 04:31:58 -04:00
parent 67c61c3d29
commit ead990a048
11 changed files with 889 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

258
scripts/ci/resource_profile.py Normal file
View file

@ -0,0 +1,258 @@
#!/usr/bin/env python3
"""CPU / RAM / disk-IO profiler for CI jobs.
Runs as a background daemon: samples /proc every second, accumulates
stats, and on SIGTERM (or timeout) writes a JSON summary to the output
path. Pure stdlib — runs on the bare runner Python with zero deps.
Usage:
python3 scripts/ci/resource_profile.py \\
--output resource-profile.json \\
--label "tests slice 1/8"
The composite action (.github/actions/profile) starts this as a
background process, runs the real command, then signals it to stop.
Output JSON shape:
{
"label": "tests slice 1/8",
"duration_s": 42.3,
"cpu": {
"avg_usage_pct": 55.2,
"peak_usage_pct": 89.1,
"samples": 42
},
"memory": { # USED memory (MemTotal - MemAvailable)
"avg_mb": 512.0,
"peak_mb": 684.3,
"samples": 42
},
"disk": {
"total_mb": 12.4, # sectors read+written, whole devices only
"avg_ops_per_s": 5.2, # completed read+write IOs per second
"peak_ops_per_s": 20.1,
"samples": 42
}
}
Caveat: /proc/stat, /proc/meminfo, and /proc/diskstats are NODE-wide.
Inside a Kubernetes pod these numbers include neighbor pods sharing the
node — treat them as indicative, not exact per-job attribution.
"""
from __future__ import annotations
import argparse
import json
import os
import re
import signal
import sys
import time
_SAMPLE_INTERVAL_S = 1.0
_CLK_TICK = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100
_PROC_STAT = "/proc/stat"
_PROC_MEMINFO = "/proc/meminfo"
_PROC_DISKSTATS = "/proc/diskstats"
def _read_cpu_usage(prev: dict | None) -> tuple[float, dict]:
"""Return (usage_pct since prev sample, current_jiffies_dict).
Reads /proc/stat line 1 (aggregate CPU). usage_pct = non-idle / total.
"""
try:
with open(_PROC_STAT, encoding="ascii") as f:
first_line = f.readline()
except OSError:
return (0.0, prev or {})
parts = first_line.split()
if len(parts) < 5:
return (0.0, prev or {})
# user, nice, system, idle, iowait, irq, softirq, steal, ...
vals = [int(x) for x in parts[1:]]
idle = vals[3] + (vals[4] if len(vals) > 4 else 0)
total = sum(vals)
cur = {"total": total, "idle": idle}
if prev and cur["total"] != prev["total"]:
d_total = cur["total"] - prev["total"]
d_idle = cur["idle"] - prev["idle"]
if d_total > 0:
return (max(0.0, (1.0 - d_idle / d_total) * 100.0), cur)
return (0.0, cur)
def _read_mem_mb() -> float:
"""Return *used* memory in MB (MemTotal - MemAvailable) from /proc/meminfo.
Falls back to MemTotal - MemFree on old kernels without MemAvailable.
Note: in a container this is the host/node view, not the cgroup view —
numbers can include neighbor pods on shared nodes.
"""
try:
with open(_PROC_MEMINFO, encoding="ascii") as f:
text = f.read()
except OSError:
return 0.0
total_kb = 0
available_kb = -1
free_kb = -1
for line in text.splitlines():
if line.startswith("MemTotal:"):
total_kb = int(line.split()[1])
elif line.startswith("MemAvailable:"):
available_kb = int(line.split()[1])
elif line.startswith("MemFree:"):
free_kb = int(line.split()[1])
if total_kb <= 0:
return 0.0
unused_kb = available_kb if available_kb >= 0 else max(free_kb, 0)
return max(0, total_kb - unused_kb) / 1024.0
def _read_diskstats() -> dict[str, tuple[int, int]]:
"""Return {device: (io_ops_completed, sectors_read_plus_written)}.
We track whole block devices only (skip partitions — track sda not
sda1, nvme0n1 not nvme0n1p1) so IO isn't double-counted. Each
diskstats line:
major minor name reads_completed reads_merged sectors_read time_reading
writes_completed writes_merged sectors_written ...
"""
try:
with open(_PROC_DISKSTATS, encoding="ascii") as f:
lines = f.readlines()
except OSError:
return {}
result = {}
for line in lines:
parts = line.split()
if len(parts) < 14:
continue
name = parts[2]
# Skip virtual/removable devices
if name.startswith(("loop", "ram", "sr")):
continue
# Skip partitions: sda1, vda2, mmcblk0p1, nvme0n1p1. For devices
# whose base name ends in a digit (nvme0n1, mmcblk0), partitions
# carry a 'p<N>' suffix; for sdX/vdX a bare trailing digit.
if name.startswith(("nvme", "mmcblk")):
if re.search(r"p\d+$", name):
continue
elif name[-1].isdigit():
continue
reads_completed = int(parts[3])
writes_completed = int(parts[7])
sectors = int(parts[5]) + int(parts[9])
result[name] = (reads_completed + writes_completed, sectors)
return result
def _read_mem_total_mb() -> float:
"""Return MemTotal in MB (0.0 if unreadable)."""
try:
with open(_PROC_MEMINFO, encoding="ascii") as f:
for line in f:
if line.startswith("MemTotal:"):
return int(line.split()[1]) / 1024.0
except OSError:
pass
return 0.0
def run_profiler(output_path: str, label: str, timeout_s: float = 0) -> None:
"""Sample resources until SIGTERM or timeout, then write JSON summary."""
cpu_samples: list[float] = []
mem_samples: list[float] = []
disk_prev = _read_diskstats()
disk_total_sectors = 0
disk_ops_samples: list[float] = []
cpu_prev: dict | None = None
start = time.monotonic()
running = [True] # mutable for signal handler
def _stop(*_):
running[0] = False
signal.signal(signal.SIGTERM, _stop)
signal.signal(signal.SIGINT, _stop)
while running[0]:
cpu_pct, cpu_prev = _read_cpu_usage(cpu_prev)
cpu_samples.append(cpu_pct)
mem_mb = _read_mem_mb()
mem_samples.append(mem_mb)
disk_cur = _read_diskstats()
delta_sectors = 0
delta_ops = 0
for dev, (ops, sectors) in disk_cur.items():
prev_ops, prev_sectors = disk_prev.get(dev, (ops, sectors))
delta_sectors += max(0, sectors - prev_sectors)
delta_ops += max(0, ops - prev_ops)
disk_total_sectors += delta_sectors
disk_ops_samples.append(delta_ops / _SAMPLE_INTERVAL_S)
disk_prev = disk_cur
elapsed = time.monotonic() - start
if timeout_s > 0 and elapsed >= timeout_s:
break
time.sleep(_SAMPLE_INTERVAL_S)
duration_s = time.monotonic() - start
n = len(cpu_samples) or 1
# Sectors are 512 bytes
disk_read_written_mb = disk_total_sectors * 512 / (1024 * 1024)
summary = {
"label": label,
"duration_s": round(duration_s, 1),
"cpu": {
"avg_usage_pct": round(sum(cpu_samples) / n, 1),
"peak_usage_pct": round(max(cpu_samples, default=0.0), 1),
"samples": len(cpu_samples),
},
"memory": {
"avg_mb": round(sum(mem_samples) / n, 1),
"peak_mb": round(max(mem_samples, default=0.0), 1),
"total_mb": round(_read_mem_total_mb(), 1),
"samples": len(mem_samples),
},
"disk": {
"total_mb": round(disk_read_written_mb, 1),
"avg_ops_per_s": round(sum(disk_ops_samples) / n, 1),
"peak_ops_per_s": round(max(disk_ops_samples, default=0.0), 1),
"samples": len(disk_ops_samples),
},
}
with open(output_path, "w", encoding="utf-8") as f:
json.dump(summary, f, indent=2)
print(f"resource_profile: wrote {output_path} ({n} samples, {duration_s:.1f}s)", file=sys.stderr)
def main():
parser = argparse.ArgumentParser(description="CI resource profiler")
parser.add_argument("--output", required=True, help="Output JSON path")
parser.add_argument("--label", default="", help="Label for this profile")
parser.add_argument("--timeout", type=float, default=0,
help="Max seconds to run (0 = until SIGTERM)")
args = parser.parse_args()
run_profiler(args.output, args.label, args.timeout)
if __name__ == "__main__":
main()

232
scripts/ci/timings_report.py
View file

@ -19,6 +19,7 @@ Usage:
from __future__ import annotations
import argparse
import glob
import json
import os
import sys
@ -417,6 +418,153 @@ def compute_stats(timings: dict, baseline: dict | None = None) -> dict:
}
# ---------------------------------------------------------------------------
# Resource profile loading + bottleneck analysis
# ---------------------------------------------------------------------------
def load_resource_profiles(directory: str) -> dict[str, dict]:
"""Load all resource-profile-*/resource-profile.json artifacts.
Returns {label: profile_dict}. Labels are derived from the artifact
directory name (resource-profile-<label> → <label>).
"""
profiles: dict[str, dict] = {}
if not directory or not os.path.isdir(directory):
return profiles
for path in glob.glob(os.path.join(directory, "**", "resource-profile.json"), recursive=True):
try:
with open(path, encoding="utf-8") as f:
profile = json.load(f)
except (json.JSONDecodeError, OSError):
continue
label = profile.get("label") or os.path.basename(os.path.dirname(path))
profiles[label] = profile
return profiles
def classify_bottleneck(timings: dict, profiles: dict[str, dict]) -> str:
"""Return a one-line bottleneck classification.
Examines wall time, compute time, wait time, and resource profiles
to identify the dominant constraint.
Possible verdicts:
- "CPU-bound: <job> at <pct>% CPU for <dur>"
- "Memory-bound: <job> peaked at <mb> MB"
- "Disk IO-bound: <job> at <ops>/s for <dur>"
- "Wait-bound: <wait>s idle waiting for dependencies"
- "Evenly distributed: no single bottleneck"
- "Insufficient data: <reason>"
"""
stats = compute_stats(timings, None)
jobs = [j for j in timings.get("jobs", []) if not is_skipped(j)]
if not jobs:
return "Insufficient data: no jobs in timings"
# --- Wait-bound: if total wait > 50% of wall time ---
wall = stats["wall"]
total_wait = stats["total_wait"]
if wall > 0 and total_wait > 0:
wait_pct = total_wait / wall * 100
if wait_pct > 50:
return (f"Wait-bound: {fmt_dur(total_wait)} idle "
f"({wait_pct:.0f}% of {fmt_dur(wall)} wall) waiting for dependencies")
# --- Resource-bound: check profiles for CPU/mem/disk extremes ---
if profiles:
cpu_hotspot = None
mem_hotspot = None
disk_hotspot = None
max_cpu = 0.0
max_mem_frac = 0.0
max_disk_ops = 0.0
for label, p in profiles.items():
cpu_info = p.get("cpu", {})
mem_info = p.get("memory", {})
disk_info = p.get("disk", {})
cpu_avg = cpu_info.get("avg_usage_pct", 0)
cpu_peak = cpu_info.get("peak_usage_pct", 0)
if cpu_avg > max_cpu:
max_cpu = cpu_avg
cpu_hotspot = (label, cpu_avg, cpu_peak, p.get("duration_s", 0))
# Memory is USED MB. Compare against the machine's total when the
# profile carries it; otherwise fall back to an absolute floor.
mem_peak = mem_info.get("peak_mb", 0)
mem_total = mem_info.get("total_mb", 0)
mem_frac = (mem_peak / mem_total) if mem_total > 0 else (mem_peak / 16000.0)
if mem_frac > max_mem_frac:
max_mem_frac = mem_frac
mem_hotspot = (label, mem_peak, mem_total)
disk_ops = disk_info.get("avg_ops_per_s", 0)
disk_mb = disk_info.get("total_mb", 0)
if disk_ops > max_disk_ops:
max_disk_ops = disk_ops
disk_hotspot = (label, disk_ops, disk_mb, p.get("duration_s", 0))
# Classify: pick the most extreme dimension.
# Each candidate's sort key is normalized to roughly 0-100 so the
# dimensions are comparable:
# CPU — avg usage pct (bound when > 80)
# Disk — avg completed IOs/s / 10 (bound when > 500 ops/s)
# Mem — peak used as pct of total (bound when > 85%)
candidates = []
if cpu_hotspot and cpu_hotspot[1] > 80:
candidates.append((
cpu_hotspot[1], # sort key
f"CPU-bound: {cpu_hotspot[0]} at {cpu_hotspot[1]:.0f}% avg CPU "
f"(peak {cpu_hotspot[2]:.0f}%) for {fmt_dur(cpu_hotspot[3])}"
))
if disk_hotspot and disk_hotspot[1] > 500:
candidates.append((
disk_hotspot[1] / 10,
f"Disk IO-bound: {disk_hotspot[0]} at {disk_hotspot[1]:.0f} ops/s "
f"({disk_hotspot[2]:.0f} MB total) for {fmt_dur(disk_hotspot[3])}"
))
if mem_hotspot and max_mem_frac > 0.85:
total_note = f" of {mem_hotspot[2]:.0f} MB" if mem_hotspot[2] else ""
candidates.append((
max_mem_frac * 100,
f"Memory-bound: {mem_hotspot[0]} peaked at "
f"{mem_hotspot[1]:.0f} MB used{total_note}"
))
if candidates:
candidates.sort(reverse=True)
return candidates[0][1]
# --- No resource profiles, but check wall vs compute ---
compute = stats["compute"]
if wall > 0 and compute > 0:
parallelism = compute / wall
if parallelism < 1.2 and len(jobs) > 2:
# Low parallelism ratio means jobs are serial
slowest = max(jobs, key=lambda j: j.get("duration_s") or 0)
slow_dur = slowest.get("duration_s") or 0
slow_pct = slow_dur / wall * 100 if wall > 0 else 0
if slow_pct > 40:
return (f"Serial bottleneck: {slowest['name']} takes "
f"{fmt_dur(slow_dur)} ({slow_pct:.0f}% of wall)")
# --- Fallback: the single slowest job ---
slowest = max(jobs, key=lambda j: j.get("duration_s") or 0)
slow_dur = slowest.get("duration_s") or 0
if slow_dur > 0 and wall > 0:
slow_pct = slow_dur / wall * 100
if slow_pct > 40 and len(jobs) > 1:
return (f"Dominated by {slowest['name']}: "
f"{fmt_dur(slow_dur)} ({slow_pct:.0f}% of wall)")
return "Evenly distributed: no single bottleneck"
# ---------------------------------------------------------------------------
# HTML generation
# ---------------------------------------------------------------------------
@ -810,7 +958,56 @@ def _regressions(timings: dict, baseline: dict | None) -> str:
)
def generate_html(timings: dict, baseline: dict | None = None) -> str:
def _resource_table(profiles: dict[str, dict]) -> str:
"""Render per-job resource usage as an HTML table."""
if not profiles:
return ""
rows = []
for label in sorted(profiles):
p = profiles[label]
cpu = p.get("cpu", {})
mem = p.get("memory", {})
disk = p.get("disk", {})
rows.append(
f'<tr>'
f'<td class="job-name">{escape(label)}</td>'
f'<td class="num">{fmt_dur(p.get("duration_s"))}</td>'
f'<td class="num">{cpu.get("avg_usage_pct", 0):.0f}%</td>'
f'<td class="num">{cpu.get("peak_usage_pct", 0):.0f}%</td>'
f'<td class="num">{mem.get("avg_mb", 0):.0f}</td>'
f'<td class="num">{mem.get("peak_mb", 0):.0f}</td>'
f'<td class="num">{disk.get("total_mb", 0):.0f}</td>'
f'<td class="num">{disk.get("avg_ops_per_s", 0):.0f}</td>'
f'</tr>'
)
return (
'<table><thead><tr>'
'<th>Job</th><th class="num">Duration</th>'
'<th class="num">CPU avg</th><th class="num">CPU peak</th>'
'<th class="num">Mem avg (MB)</th><th class="num">Mem peak (MB)</th>'
'<th class="num">Disk (MB)</th><th class="num">Disk ops/s</th>'
'</tr></thead><tbody>' + "".join(rows) + '</tbody></table>'
)
def _bottleneck_box(timings: dict, profiles: dict[str, dict]) -> str:
"""Render the bottleneck analysis as a callout box."""
verdict = classify_bottleneck(timings, profiles)
return (
f'<div style="background:#161b22;border:1px solid #30363d;'
f'border-radius:8px;padding:16px;margin-bottom:24px">'
f'<div style="font-size:12px;color:#8b949e;text-transform:uppercase;'
f'letter-spacing:0.5px;margin-bottom:4px">Bottleneck Analysis</div>'
f'<div style="font-size:16px;font-weight:500">{escape(verdict)}</div>'
f'</div>'
)
def generate_html(timings: dict, baseline: dict | None = None,
profiles: dict[str, dict] | None = None) -> str:
stats = compute_stats(timings, baseline)
sha_short = (timings.get("head_sha") or "")[:7]
@ -837,6 +1034,12 @@ def generate_html(timings: dict, baseline: dict | None = None) -> str:
html += '<h2>Global Stats</h2>\n'
html += _stats_cards(stats)
html += _bottleneck_box(timings, profiles or {})
if profiles:
html += '<h2>Resource Usage</h2>\n'
html += _resource_table(profiles)
if baseline:
html += '<h2>Top Regressions & Improvements</h2>\n'
html += _regressions(timings, baseline)
@ -858,12 +1061,18 @@ def generate_html(timings: dict, baseline: dict | None = None) -> str:
# Markdown summary for $GITHUB_STEP_SUMMARY
# ---------------------------------------------------------------------------
def generate_summary(timings: dict, baseline: dict | None = None) -> str:
def generate_summary(timings: dict, baseline: dict | None = None,
profiles: dict[str, dict] | None = None) -> str:
stats = compute_stats(timings, baseline)
bl_map = {j["name"]: j for j in (baseline or {}).get("jobs", [])}
lines = ["## CI Timing Summary\n"]
# Bottleneck analysis
bottleneck = classify_bottleneck(timings, profiles or {})
lines.append(f"**Bottleneck:** {bottleneck}")
lines.append("")
# Global stats table
lines.append("| Metric | Current | Baseline | Delta |")
lines.append("|--------|---------|----------|-------|")
@ -905,7 +1114,8 @@ _TIMINGS_WARN_PCT = 0.25
def generate_review_status(
timings: dict, baseline: dict | None, report_url: str | None = None
timings: dict, baseline: dict | None, report_url: str | None = None,
profiles: dict[str, dict] | None = None
) -> list[dict]:
"""Produce a review_status JSON array for the CI timings review section.
@ -916,6 +1126,7 @@ def generate_review_status(
fragment.
"""
stats = compute_stats(timings, baseline)
bottleneck = classify_bottleneck(timings, profiles or {})
if baseline is None:
severity = "debug"
@ -942,6 +1153,8 @@ def generate_review_status(
wall_str += f" {stats['unchanged']} unchanged."
summary = wall_str
summary += f" Bottleneck: {bottleneck}"
# Per-job delta detail (top 5 by absolute change)
detail_lines: list[str] = []
if baseline:
@ -1001,8 +1214,13 @@ def main():
help="If set, write a review-status JSON for the unified PR comment.")
parser.add_argument("--review-status-only", action="store_true",
help="Write review status from existing timings without regenerating the report.")
parser.add_argument("--profiles-dir", default="",
help="Directory of downloaded resource-profile-* artifacts.")
args = parser.parse_args()
# Load resource profiles (available in both API and --from-json modes)
profiles = load_resource_profiles(args.profiles_dir) if args.profiles_dir else {}
# Collect or load timings
if args.from_json:
with open(args.from_json, encoding="utf-8") as f:
@ -1051,20 +1269,20 @@ def main():
if not args.review_status_out:
parser.error("--review-status-only requires --review-status-out")
report_url = os.environ.get("CI_TIMINGS_REPORT_URL", "")
statuses = generate_review_status(timings, baseline, report_url)
statuses = generate_review_status(timings, baseline, report_url, profiles)
with open(args.review_status_out, "w", encoding="utf-8") as f:
f.write(f"review_status={json.dumps(statuses)}\n")
print(f"Wrote review status to {args.review_status_out}")
return
# Generate HTML
html = generate_html(timings, baseline)
html = generate_html(timings, baseline, profiles)
with open(args.output, "w", encoding="utf-8") as f:
f.write(html)
print(f"Generated HTML report: {args.output}")
# Write summary
summary = generate_summary(timings, baseline)
summary = generate_summary(timings, baseline, profiles)
with open(args.summary_out, "a", encoding="utf-8") as f:
f.write(summary)
print(f"Wrote summary to {args.summary_out}")
@ -1074,7 +1292,7 @@ def main():
# format) so the ci-timings job can expose it as a workflow_call output.
if args.review_status_out:
report_url = os.environ.get("CI_TIMINGS_REPORT_URL", "")
statuses = generate_review_status(timings, baseline, report_url)
statuses = generate_review_status(timings, baseline, report_url, profiles)
json_str = json.dumps(statuses)
with open(args.review_status_out, "a", encoding="utf-8") as f:
f.write(f"review_status={json_str}\n")