From 7cc95e51eff952548398fffa220a9b678c0482c5 Mon Sep 17 00:00:00 2001 From: ethernet Date: Thu, 30 Jul 2026 14:51:08 -0400 Subject: [PATCH] fix(tests): cgroup-aware worker count in parallel test runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit os.cpu_count() reports the HOST cores. In an ARC runner pod (limit 8 CPU on a 22-core node) the runner spawned -j 44 workers on 8 usable CPUs — ~5x oversubscription. Every 'timing flake' family on the self-hosted runners (docker rm teardown TimeoutExpired x111, compression fork, termux probe, pty reaper, session hygiene) is CPU starvation from that oversubscription, not real test bugs. Read cgroup v2 cpu.max (v1 cfs_quota fallback) and clamp to host count. Verified: --cpus=8 container reports 8, bare host unchanged. HERMES_TEST_WORKERS override still wins. --- scripts/run_tests_parallel.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/scripts/run_tests_parallel.py b/scripts/run_tests_parallel.py index 1b34c74b4d6..2042247d8bb 100755 --- a/scripts/run_tests_parallel.py +++ b/scripts/run_tests_parallel.py @@ -673,6 +673,35 @@ def _make_stdio_glyph_safe() -> None: pass +def _effective_cpu_count() -> int: + """CPU count respecting cgroup quotas (containers/K8s pods). + + ``os.cpu_count()`` reports the HOST core count. Inside a CPU-limited + cgroup (e.g. an ARC runner pod with a 8-CPU limit on a 22-core node) + that oversubscribes the pod ~3x and per-file test subprocesses hit + their timeouts from CPU starvation. Read the cgroup v2 ``cpu.max`` + (or v1 quota/period) when present. + """ + host = os.cpu_count() or 4 + try: + with open("/sys/fs/cgroup/cpu.max", encoding="ascii") as f: + quota_s, period_s = f.read().split() + if quota_s != "max": + return max(1, min(host, int(int(quota_s) / int(period_s)))) + except (OSError, ValueError): + pass + try: + with open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", encoding="ascii") as f: + quota = int(f.read()) + with open("/sys/fs/cgroup/cpu/cpu.cfs_period_us", encoding="ascii") as f: + period = int(f.read()) + if quota > 0: + return max(1, min(host, quota // period)) + except (OSError, ValueError): + pass + return host + + def main() -> int: _make_stdio_glyph_safe() parser = argparse.ArgumentParser( @@ -683,8 +712,8 @@ def main() -> int: "-j", "--jobs", type=int, - default=int(os.environ.get("HERMES_TEST_WORKERS") or (os.cpu_count() or 4) * 2), - help="Parallel worker count (default: $HERMES_TEST_WORKERS or cpu_count*2)", + default=int(os.environ.get("HERMES_TEST_WORKERS") or _effective_cpu_count() * 2), + help="Parallel worker count (default: $HERMES_TEST_WORKERS or cgroup-aware cpu_count*2)", ) parser.add_argument( "--paths",