perf(ci): sparse+blobless checkout for the test-slice generate job

The "Generate slices" job spent ~90% of its wall time in actions/checkout
(4-27s across the last 10 runs, of a 10-32s job) pulling a ~212MB working
tree. Everything in the test matrix waits on it.

It doesn't need those files. `--generate-slices` returns before
`_approximately_count_tests`, so it only ever uses test file *paths* plus
the cached durations — it never opens a test file.

The blocker was discovery: `_discover_files` rglobs the filesystem, which
finds nothing under a sparse checkout. So add `--discover-from-git`, which
lists paths via `git ls-files`. Sparse checkout only clears the worktree
(entries are marked skip-worktree), so the index still carries every path
and enumerates exactly the same set. Skip-part filtering and the
root-override rule are duplicated to match `_discover_files` semantics.

Measured on a real clone of this repo:

    full depth=1 clone:       7s   212M
    blobless+sparse clone:    3s   3.4M   (still sees all 2510 test paths)

Both discovery paths produce byte-identical slice JSON over the full 2472
test files, so slice assignment is unchanged.

Tests assert the properties that make this safe: the two discovery paths
agree on a full checkout, the git path still works when the files are
absent from disk, and the skip-part override behaves the same either way.
This commit is contained in:
ethernet 2026-07-31 13:26:27 -04:00
parent f702bba63c
commit 4e3768ea36
3 changed files with 176 additions and 2 deletions

View file

@ -170,6 +170,56 @@ def _discover_files(roots: List[Path]) -> List[Path]:
return sorted(out)
def _discover_files_from_git(roots: List[Path], repo_root: Path) -> List[Path]:
"""Like :func:`_discover_files`, but list paths from the git index.
``_discover_files`` walks the filesystem, which requires a full
working tree. The CI ``--generate-slices`` job only needs test file
*names* (it never opens them), so it checks out sparsely a blobless,
sparse clone is ~3MB against ~212MB for the full tree. Under a sparse
checkout ``rglob`` finds nothing, because the files aren't on disk.
The git *index* still carries every path (sparse checkout only clears
the worktree, marking entries skip-worktree), so ``git ls-files``
enumerates the same set the filesystem walk would have. Skip-part
filtering and root-override semantics match ``_discover_files``
exactly, so both discovery paths produce identical slicing input.
"""
seen: set[Path] = set()
out: List[Path] = []
for root in roots:
rel = root.relative_to(repo_root) if root.is_absolute() else root
# Same opt-in rule as _discover_files: naming a skipped dir as a
# root overrides the skip for that subtree.
root_skip_overrides = {part for part in rel.parts if part in _SKIP_PARTS}
effective_skips = _SKIP_PARTS - root_skip_overrides
try:
listed = subprocess.run(
["git", "ls-files", "-z", "--", str(rel)],
cwd=repo_root,
capture_output=True,
text=True,
check=True,
).stdout
except (subprocess.CalledProcessError, FileNotFoundError) as e:
print(f"error: git ls-files failed for {rel}: {e}", file=sys.stderr)
raise SystemExit(2) from e
for entry in listed.split("\0"):
if not entry:
continue
path = Path(entry)
if not path.name.startswith("test_") or path.suffix != ".py":
continue
if any(part in effective_skips for part in path.parts):
continue
abs_path = repo_root / path
if abs_path in seen:
continue
seen.add(abs_path)
out.append(abs_path)
return sorted(out)
def _kill_tree(proc: "subprocess.Popen", pgid: int | None = None) -> None:
"""Kill the pytest subprocess and every descendant it spawned.
@ -773,6 +823,15 @@ def main() -> int:
"so the CI generate job can feed it directly into a matrix."
),
)
parser.add_argument(
"--discover-from-git",
action="store_true",
help=(
"List test files from the git index (git ls-files) instead of "
"walking the filesystem. Lets --generate-slices run against a "
"blobless sparse checkout, where the test files aren't on disk."
),
)
parser.add_argument(
"--files",
metavar="LIST",
@ -811,6 +870,7 @@ def main() -> int:
OUR_FLAGS = {
"-j", "--jobs", "--paths", "--include-integration",
"--file-timeout", "--file-retries", "--slice", "--generate-slices", "--files",
"--discover-from-git",
}
# pytest short flags that consume the NEXT token as their value.
PYTEST_VALUE_FLAGS = {"-k", "-m", "-p", "-o", "-c", "-r", "-W"}
@ -929,7 +989,11 @@ def main() -> int:
global _SKIP_PARTS # noqa: PLW0603 — config knob
_SKIP_PARTS = set()
files = _discover_files(roots)
files = (
_discover_files_from_git(roots, repo_root)
if args.discover_from_git
else _discover_files(roots)
)
if not files:
print("No test files to run", file=sys.stderr)