Coverage for packages/workshop/src/livery/workshop/_scale.py: 89%

Measured by coverage.py 7.16.2 on CPython 3.14 while the suites of tests, packages/workshop, packages/extensions/ty, packages/extensions/ruff, packages/extensions/python, packages/extensions/pytest, packages/extensions/pyrefly, packages/extensions/nanobind, packages/extensions/mypy, packages/extensions/mkdocstrings, packages/extensions/lfs, packages/extensions/docs, packages/extensions/cpp, packages/extensions/coverage, packages/extensions/conan, packages/extensions/cmake, packages/extensions/claude, packages/extensions/clang-tidy, packages/extensions/clang-format, packages/extensions/changelog, packages/extensions/basedpyright ran.

1"""Build a workspace of many members and time the workshop's verbs over it.
2
3The budgets of a sync and a gate are set from a measured baseline,
4and judged against the CI store. `fm ci.scale` builds the workspace
5that baseline is measured on, times `fm sync`, `fm drift.check`
6and `fm check` there, and prints the timings. Inside CI it also
7records them as one ``scale`` job on the run's file of the
8``metrics`` series ([livery.workshop._metrics][]), so `fm ci.timings`
9shows them beside the gate's own rows.
10
11The workspace is a copy of this checkout's committed tree at HEAD,
12made a repository of its own with no remote, plus the generated
13members. The workshop under test is the copy's own member, so the
14timings measure the code being edited, and nothing is fetched from
15an index or a forge. The copy keeps this repository's own members
16and their source, so every checker reads them, but not their test
17suites: those test facts about this repository, which the generated
18members change.
19
20Each verb is timed in three states:
21
22- ``cold``: no environment in the copy and no gate record, the gate
23 run whole. The machine's uv cache and tool store stay warm.
24- ``warm``: again at once, nothing changed.
25- ``one changed``: after an edit to the source of one python member
26 no other member depends on.
27
28The generated python members form a binary tree of runtime
29dependencies: member ``i`` depends on member ``(i - 1) // 2``,
30declared as a ``[[depends]]`` edge and as a requirement in its
31``pyproject.toml``, so the graph the gate walks has depth as well as
32width.
33
34This is development tooling of this repository, not part of the
35workshop a project uses.
36"""
37
38from __future__ import annotations
39
40import os
41import re
42import shutil
43import sys
44import tarfile
45import tempfile
46import time
47from dataclasses import dataclass
48from datetime import UTC, datetime
49from pathlib import Path
50from typing import Annotated, Any
51
52import livery.footman as footman
53from livery import git
54from livery.footman import doc, fail
55from livery.workshop._ci_tasks import ci
56
57#: The members a fixture holds by default, per template.
58DEFAULT_COUNTS: dict[str, int] = {
59 "python-library": 240,
60 "cpp-package": 40,
61 "nanobind-module": 20,
62}
63
64#: The short tag a generated member's name carries, per template.
65_TAGS = {
66 "python-library": "p",
67 "cpp-package": "c",
68 "nanobind-module": "n",
69}
70
71#: The verbs timed, in the order each state runs them.
72VERBS: tuple[tuple[str, ...], ...] = (
73 ("sync",),
74 ("drift.check",),
75 ("check",),
76)
77
78#: The states each verb is timed in, in order.
79STATES = ("cold", "warm", "one changed")
80
81
82@dataclass(frozen=True)
83class Timing:
84 """One timed run of a verb in the fixture.
85
86 Attributes:
87 name: The metric's name, the verb then the state.
88 ms: The wall time, in milliseconds.
89 code: The verb's exit code.
90 """
91
92 name: str
93 ms: float
94 code: int
95
96
97def member_names(counts: dict[str, int]) -> list[tuple[str, str]]:
98 """The generated members, as ``(directory name, template)`` pairs.
99
100 Names sort by template tag, then by a zero-padded index, so the
101 python members' tree order is their name order.
102 """
103 out: list[tuple[str, str]] = []
104 for template, count in counts.items():
105 tag = _TAGS.get(template)
106 if tag is None:
107 fail(
108 f"no scale tag for template {template!r}: use one of {', '.join(_TAGS)}"
109 )
110 out.extend((f"scale-{tag}{index:03d}", template) for index in range(count))
111 return out
112
113
114def parent_of(index: int) -> int | None:
115 """The python member *index* depends on, or ``None`` for the root."""
116 return None if index == 0 else (index - 1) // 2
117
118
119def _git(root: Path, *args: str) -> str:
120 """Run git under *root*; stdout, or fail with git's own words."""
121 try:
122 return git.run(root, *args)
123 except git.GitError as error:
124 fail(str(error))
125
126
127def copy_tree(source: Path, destination: Path) -> None:
128 """Copy *source*'s committed tree at HEAD into *destination* as a new repository.
129
130 The copy has one commit and no remote. Its commits are unsigned:
131 they are scratch, and a signer that waits for a person would
132 stop an unattended run.
133 """
134 destination.mkdir(parents=True, exist_ok=False)
135 archive = destination.parent / f"{destination.name}.tar"
136 _git(source, "archive", "--format=tar", f"--output={archive}", "HEAD")
137 with tarfile.open(archive) as bundle:
138 bundle.extractall(destination, filter="data")
139 archive.unlink()
140 _git(destination, "init", "--quiet", "--initial-branch=main")
141 for key, value in (
142 ("user.name", "scale fixture"),
143 ("user.email", "scale@fixture.invalid"),
144 ("commit.gpgsign", "false"),
145 ("tag.gpgsign", "false"),
146 ):
147 _git(destination, "config", key, value)
148 _commit(destination, "chore: the tree at HEAD")
149
150
151def _commit(root: Path, subject: str) -> None:
152 _git(root, "add", "--all")
153 _git(root, "commit", "--quiet", "--no-verify", "-m", subject)
154
155
156def strip_own_suites(root: Path) -> list[str]:
157 """Remove the copy's own members' test modules and floors; the members.
158
159 The fixture measures the verbs over the generated members. The
160 suites of the members the copy started with test facts about the
161 repository they come from, which the generated members change,
162 and their time is not the fixture's. Each such member loses its
163 ``test_*.py`` modules and keeps its helpers, since the rendered
164 checker configuration names its ``tests`` directory; a directory
165 left with no module gets one, named with the prefix the test
166 layout rule wants of a helper: the package's path under
167 ``packages/``, a group directory's name included. Its
168 ``[coverage]`` floor gives way to ``required = false``, so nothing
169 judges its coverage. The root ``tests`` directory is the project
170 render's and stays as it is.
171 """
172 from livery.workshop._packages import discover_packages
173
174 stripped: list[str] = []
175 for package in discover_packages(root):
176 tests = package.directory / "tests"
177 if tests.is_dir():177 ↛ 186line 177 didn't jump to line 186 because the condition on line 177 was always true
178 for module in tests.rglob("test_*.py"):
179 module.unlink()
180 if not any(tests.rglob("*.py")):
181 name = package.member.replace("-", "_").replace("/", "_")
182 (tests / f"{name}_copy.py").write_text(
183 '"""The scale fixture\'s copy keeps no suite of this member."""\n',
184 encoding="utf-8",
185 )
186 contract = package.directory / "workshop.toml"
187 text = contract.read_text("utf-8")
188 # A [coverage] floor is a number; a [[depends]] floor is a quoted version.
189 contract.write_text(
190 re.sub(r"(?m)^floor = [\d.]+$", "required = false", text),
191 encoding="utf-8",
192 )
193 stripped.append(package.path)
194 _commit(root, "chore: the copy's own suites removed")
195 return stripped
196
197
198def generate(root: Path, counts: dict[str, int]) -> list[tuple[str, str]]:
199 """Add the generated members to the workspace at *root*; the members.
200
201 The root's list takes in what the members' templates need first,
202 written as its canonical list, since a creation installs nothing
203 and judges the list: this checkout's environment, which runs the
204 creation, installs each of them. Each member is rendered and added
205 to the roster, then the
206 project is rendered once and the result committed. Locking and
207 installing are left to the first `fm sync`.
208 """
209 from livery.workshop._extensions import list_in_root
210 from livery.workshop._packages import declare_edge, discover_packages
211 from livery.workshop._seeds import chain_needs, template_chain
212 from livery.workshop._templates import apply_project, render_member
213
214 members = member_names(counts)
215 python = [name for name, template in members if template == "python-library"]
216 list_in_root(
217 root,
218 (
219 needed
220 for template in dict.fromkeys(template for _name, template in members)
221 for needed in chain_needs(template_chain(template))
222 ),
223 )
224 for name, template in members:
225 render_member(root, name, template)
226 by_path = {package.path: package for package in discover_packages(root)}
227 for index, name in enumerate(python):
228 parent = parent_of(index)
229 if parent is not None:
230 declare_edge(
231 by_path[f"packages/{name}"],
232 by_path[f"packages/{python[parent]}"],
233 kind="runtime",
234 floor="0.0.0",
235 )
236 apply_project(root)
237 _commit(root, f"chore: {len(members)} generated members")
238 return members
239
240
241def leaf(members: list[tuple[str, str]]) -> str:
242 """The python member the edit touches: the last, which no member depends on."""
243 python = [name for name, template in members if template == "python-library"]
244 if not python:
245 fail("the fixture has no python member to edit")
246 return python[-1]
247
248
249def edit_member(root: Path, name: str) -> Path:
250 """Append one covered, formatted statement to *name*'s module; the file edited."""
251 found = sorted((root / "packages" / name / "src").rglob("__init__.py"))
252 if not found:
253 fail(f"packages/{name}: no package module to edit")
254 module = found[-1]
255 with module.open("a", encoding="utf-8") as handle:
256 # Two blank lines are formatted after any top-level statement.
257 handle.write("\n\nSCALE_EDIT = 1\n")
258 return module
259
260
261def _environment(root: Path) -> dict[str, str]:
262 """The children's environment: this one, minus the venv it names.
263
264 The variables the extensions of the fixture's set declare join it,
265 each one naming ``{place}`` placed in the fixture's directory,
266 beside the workspace: a tool's home there is the fixture's own, so
267 what its native members build never reaches the machine's. The
268 fixture forms its children's environment, so an ``always`` one
269 beats the value this one carries, and each child keeps it.
270
271 Raises:
272 Failed: when two of the extensions declare one variable.
273 """
274 from livery.workshop._variables import placed, workspace_variables
275
276 env = {k: v for k, v in os.environ.items() if k != "VIRTUAL_ENV"}
277 env.update(placed(workspace_variables(root), str(root.parent), env, forming=True))
278 return env
279
280
281def run_verb(root: Path, verb: tuple[str, ...], *, name: str) -> Timing:
282 """Run ``fm <verb>`` in *root* and time it; the timing, printed as it ends.
283
284 The child is this interpreter's footman, which enters the
285 workspace's own environment. Its output is printed when it
286 exits non-zero, and the timing records the exit code either way.
287 """
288 start = time.perf_counter()
289 result = footman.run(
290 [sys.executable, "-m", "livery.footman", "--yes", *verb],
291 cwd=root,
292 env=_environment(root),
293 nofail=True,
294 timeout=7200.0,
295 )
296 ms = round((time.perf_counter() - start) * 1000.0, 1)
297 print(f" {name}: {ms / 1000:.1f}s, exit {result.code}")
298 if result.code != 0:298 ↛ 300line 298 didn't jump to line 300 because the condition on line 298 was always true
299 print(f"{result.stdout}{result.stderr}")
300 return Timing(name=name, ms=ms, code=result.code)
301
302
303def measure(root: Path, members: list[tuple[str, str]]) -> list[Timing]:
304 """Time every verb in every state in the fixture at *root*; the timings."""
305 timings: list[Timing] = []
306 shutil.rmtree(root / ".venv", ignore_errors=True)
307 for state in STATES:
308 if state == "one changed":
309 edited = edit_member(root, leaf(members))
310 print(f" edited {edited.relative_to(root).as_posix()}")
311 for verb in VERBS:
312 spelled = " ".join(verb)
313 args = (*verb, "--full") if verb == ("check",) and state == "cold" else verb
314 timings.append(run_verb(root, args, name=f"{spelled} {state}"))
315 return timings
316
317
318def row(
319 timings: list[Timing], members: list[tuple[str, str]], *, job: str = "scale"
320) -> dict[str, Any]:
321 """The ``scale`` job's row: the timings as tasks, exits and members."""
322 counts: dict[str, int] = {}
323 for _name, template in members:
324 counts[template] = counts.get(template, 0) + 1
325 return {
326 "job": job,
327 "total_ms": round(sum(timing.ms for timing in timings), 1),
328 "tasks": {timing.name: timing.ms for timing in timings},
329 "codes": {timing.name: timing.code for timing in timings},
330 "members": dict(sorted(counts.items())),
331 }
332
333
334def record(root: Path, job_row: dict[str, Any], *, sha: str) -> str:
335 """Put *job_row* on the run's file of the metrics series; what happened.
336
337 Outside CI nothing is written and the line says so.
338 """
339 from livery.git import remote_snapshot
340 from livery.workshop._ci_context import run_context, state_store
341 from livery.workshop._metrics import SERIES, run_file
342
343 run = run_context()
344 if run is None:344 ↛ 346line 344 didn't jump to line 346 because the condition on line 344 was always true
345 return " not a CI run: the timings are recorded by CI only"
346 entry: dict[str, Any] = {
347 "forge": run.forge,
348 "run": run.run_id,
349 "event": run.event,
350 "sha": run.head_sha or sha,
351 "checkout": sha,
352 "ref": run.ref,
353 "collected_at": datetime.now(UTC).isoformat(timespec="seconds"),
354 "jobs": {str(job_row["job"]): job_row},
355 }
356 with remote_snapshot(state_store(root), fetch=("metrics",)):
357 why = SERIES.put(
358 state_store(root),
359 {run_file(run.run_id): entry},
360 message=f"metrics: scale fixture of run {run.run_id} at {sha[:12]}",
361 )
362 return (
363 f" {SERIES.ref}: {why}"
364 if why
365 else f" {SERIES.ref}: run {run.run_id} recorded"
366 )
367
368
369def _counts(python: int, cpp: int, nanobind: int) -> dict[str, int]:
370 return {
371 template: count
372 for template, count in (
373 ("python-library", python),
374 ("cpp-package", cpp),
375 ("nanobind-module", nanobind),
376 )
377 if count > 0
378 }
379
380
381@ci.task(name="scale")
382@git.unsigned
383def ci_scale(
384 python: Annotated[int, doc("members born from python-library")] = DEFAULT_COUNTS[
385 "python-library"
386 ],
387 cpp: Annotated[int, doc("members born from cpp-package")] = DEFAULT_COUNTS[
388 "cpp-package"
389 ],
390 nanobind: Annotated[int, doc("members born from nanobind-module")] = DEFAULT_COUNTS[
391 "nanobind-module"
392 ],
393 keep: Annotated[
394 str, doc("build the fixture in this new directory and keep it")
395 ] = "",
396) -> None:
397 """Time sync, drift.check and check over a generated workspace.
398
399 Copies the committed tree at HEAD, adds the generated members,
400 and times each verb cold, warm with nothing changed, and after
401 one member's source changed. Prints every timing. Inside CI the
402 timings are also put on the metrics series as the run's
403 ``scale`` job, which `fm ci.timings` shows. The fixture is
404 removed afterwards unless *keep* names where to build it.
405 """
406 from livery.workshop._templates import _root # pyright: ignore[reportPrivateUsage]
407
408 source = _root()
409 counts = _counts(python, cpp, nanobind)
410 if keep:410 ↛ 413line 410 didn't jump to line 413 because the condition on line 410 was always true
411 timings = run_fixture(source, Path(keep).resolve(), counts)
412 else:
413 with tempfile.TemporaryDirectory(
414 prefix="fm-scale-", ignore_cleanup_errors=True
415 ) as scratch:
416 timings = run_fixture(source, Path(scratch) / "workspace", counts)
417 failed = [timing.name for timing in timings if timing.code != 0]
418 if failed:418 ↛ exitline 418 didn't return from function 'ci_scale' because the condition on line 418 was always true
419 fail(f"the fixture's verbs exited non-zero: {', '.join(failed)}")
420
421
422def run_fixture(source: Path, root: Path, counts: dict[str, int]) -> list[Timing]:
423 """Build the fixture at *root* from *source*, time it, record it; the timings."""
424 copy_tree(source, root)
425 stripped = strip_own_suites(root)
426 print(f" {len(stripped)} member(s) of the copy without their suites")
427 started = time.perf_counter()
428 members = generate(root, counts)
429 print(
430 f" generated {len(members)} member(s) in"
431 f" {time.perf_counter() - started:.1f}s at {root}"
432 )
433 timings = measure(root, members)
434 sha = _git(source, "rev-parse", "HEAD").strip()
435 print(record(source, row(timings, members), sha=sha))
436 return timings