From 707ae6e6239b699c8c7a65aa00c8643b68186f12 Mon Sep 17 00:00:00 2001 From: ethernet Date: Fri, 26 Jun 2026 19:29:36 -0400 Subject: [PATCH] change(tests): don't count with pytest collect it's way too slow. just grep files lol --- scripts/run_tests_parallel.py | 76 ++++++++++------------------------- 1 file changed, 21 insertions(+), 55 deletions(-) diff --git a/scripts/run_tests_parallel.py b/scripts/run_tests_parallel.py index 0b9a227a145..4edbd7bcd03 100755 --- a/scripts/run_tests_parallel.py +++ b/scripts/run_tests_parallel.py @@ -80,61 +80,27 @@ _DEFAULT_FILE_TIMEOUT_SECONDS = 140.0 # set by observing the slowest file at com _DURATIONS_FILE = "test_durations.json" -def _count_tests( +def _approximately_count_tests( files: List[Path], repo_root: Path ) -> dict[Path, int]: - """Run ``pytest --co -q`` once to count individual tests per file. + """ + Make a decent estimate at individual tests per file. + Running ``pytest --co -q`` is WAY too slow because it actually imports everything. Returns a mapping ``{file_path: test_count}``. Files with zero collected tests are omitted from the dict (not an error — e.g. the file only defines fixtures / conftest helpers). - This is a single subprocess call (~2-5s for ~1k files) that gives - us the total test count for the discovery announcement and - per-file counts for the progress lines. - - ``--ignore`` flags for directories in ``_SKIP_PARTS`` are added - automatically so that pytest's own collection machinery (conftest - walking, directory traversal) doesn't pull in tests we intend to - skip — matching what the per-file runs will actually execute. """ - # Build --ignore flags for skipped dirs so the --co collection - # mirrors what we'll actually run (not what pytest might find via - # conftest walking or directory traversal). - ignore_args: List[str] = [] - for root in [repo_root / p for p in _DEFAULT_ROOTS]: - for part in _SKIP_PARTS: - d = root / part - if d.is_dir(): - ignore_args.extend(["--ignore", str(d)]) - cmd = [ - sys.executable, "-m", "pytest", - "--co", "-q", - *ignore_args, - *[str(f) for f in files], - ] - try: - result = subprocess.run( - cmd, - cwd=repo_root, - capture_output=True, - text=True, - timeout=120, - ) - except (subprocess.TimeoutExpired, OSError): - return {} + results = {} - counts: dict[Path, int] = {} - for line in result.stdout.splitlines(): - # Lines look like: tests/acp/test_auth.py::TestClass::test_name - if "::" not in line: - continue - file_part = line.split("::", 1)[0] - key = repo_root / file_part - counts[key] = counts.get(key, 0) + 1 + for path in files: + with open(path, "r", encoding="utf-8") as f: + contents = f.read() + results[path] = contents.count("def test_") - return counts + return results def _discover_files(roots: List[Path]) -> List[Path]: @@ -389,7 +355,7 @@ def _format_file(file: Path, repo_root: Path) -> str: def _print_progress( tests_done: int, - total_tests: int, + approx_total_tests: int, file: Path, rc: int, dur: float, @@ -411,7 +377,7 @@ def _print_progress( time and the queue-inclusive elapsed time. """ status = "✓" if rc == 0 else "✗" - pct = (tests_done / total_tests * 100) if total_tests else 0 + pct = min((tests_done / approx_total_tests * 100), 100) if approx_total_tests else 0 # Digit width for left-side counter padding (derived from total file count). fw = len(str(tests_passed + tests_failed)) # Build per-file test count string. @@ -446,7 +412,7 @@ def _print_progress( else: time_str = f"{dur:.1f}s" msg = ( - f"[{pct:5.1f}% | {tests_done:>5}/{total_tests}" + f"[{pct:5.1f}% | {tests_done:>5}/~{approx_total_tests}" f" | ✓{tests_passed:>{fw}} | ✗{tests_failed:>{fw}}] " f"{status} {_format_file(file, repo_root)} ({test_str}{time_str})" ) @@ -695,9 +661,9 @@ def main() -> int: print(f"No test files discovered under {[str(r) for r in roots]}", file=sys.stderr) return 1 - # Count individual tests per file via a single pytest --co pass. - test_counts = _count_tests(files, repo_root) - total_tests = sum(test_counts.values()) + # Count individual tests per file + test_counts = _approximately_count_tests(files, repo_root) + approx_total_tests = sum(test_counts.values()) # Apply slicing if requested — distribute files across CI jobs by # estimated duration so no one job gets all the slow files. @@ -706,10 +672,10 @@ def main() -> int: files = _slice_files(files, slice_index, slice_count, durations, repo_root) # Recount after slicing. test_counts = {f: test_counts[f] for f in files if f in test_counts} - total_tests = sum(test_counts.values()) + approx_total_tests = sum(test_counts.values()) print( - f"Discovered {len(files)} test files ({total_tests} tests) under " + f"Discovered {len(files)} test files (~{approx_total_tests} tests) under " f"{[str(r.relative_to(repo_root)) if r.is_relative_to(repo_root) else str(r) for r in roots]}; " f"running with -j {args.jobs}", flush=True, @@ -740,7 +706,7 @@ def main() -> int: fail_count += 1 failures.append((file, f"runner crashed: {exc!r}", {})) _print_progress( - tests_done, total_tests, file, 1, + tests_done, approx_total_tests, file, 1, time.monotonic() - started_at, repo_root, tests_passed, tests_failed, test_counts, @@ -760,7 +726,7 @@ def main() -> int: fail_count += 1 failures.append((fpath, output, summary)) _print_progress( - tests_done, total_tests, fpath, rc, + tests_done, approx_total_tests, fpath, rc, time.monotonic() - started_at, repo_root, tests_passed, tests_failed, test_counts, @@ -787,7 +753,7 @@ def main() -> int: elapsed = time.monotonic() - started print() - pct = (tests_done / total_tests * 100) if total_tests else 0 + pct = min(100, (tests_done / approx_total_tests * 100)) if approx_total_tests else 0 print(f"=== Summary: {len(files)} files, {tests_passed} tests passed, {tests_failed} failed ({pct:.0f}% complete) in {elapsed:.1f}s ({args.jobs} workers) ===") # Save durations for future --slice runs. Each slice writes its own