Coverage for packages/workshop/src/livery/workshop/_metrics.py: 94%

Measured by coverage.py 7.16.2 on CPython 3.14 while the suites of tests, packages/workshop, packages/extensions/ty, packages/extensions/ruff, packages/extensions/python, packages/extensions/pytest, packages/extensions/pyrefly, packages/extensions/nanobind, packages/extensions/mypy, packages/extensions/mkdocstrings, packages/extensions/lfs, packages/extensions/docs, packages/extensions/cpp, packages/extensions/coverage, packages/extensions/conan, packages/extensions/cmake, packages/extensions/claude, packages/extensions/clang-tidy, packages/extensions/clang-format, packages/extensions/changelog, packages/extensions/basedpyright ran.

1"""Timing rows on the CI state store, and the reader that renders them.
2
3One row per job per run, one file per run on the ``metrics`` series
4of the state store ([livery.git][]), named by zero-padded
5run id so that name order is time order. A row is end to end, not
6gate-only: the queue wait, the job's wall, and every step's wall as
7the forge times them, beside the per-task durations and lane waits
8the leg's own trace recorded, and the test time per package summed
9from the trace's pytest slices.
10
11Three writers, all inside CI. Each check leg ends with `put_leg`: it
12reads the trace the profiled gate wrote and puts the task half of
13its row on the leg's per-run ref, in one write with every file the
14leg's other entries left in its spool (`LEG_SPOOL`); a leg that ran
15as one shard of several carries the record it left of what it
16partitioned in the row, under ``shard``; a write that fails is a red
17leg. The run's gate job first folds each platform's shards into one
18ref ([livery.workshop._fold][]), then runs `collect`: it reads the
19per-run refs, asks the forge for the run's jobs, joins the two
20halves, folds in the sections the job's judges left (`RUN_ROW`),
21puts the run's file on the series under its window, and writes each
22check leg's seconds per item onto its platform leg's series of
23the [livery.workshop.DURATIONS][] family; it fails open loudly, a
24printed reason and never a red job. The run's last green job runs
25`drop_halves`: the per-run refs go once every job that writes a half
26has completed green, so a re-run of a red gate job still finds what
27its legs put, and a point with no gate job, where every job puts a
28half and collects, keeps each job's half until the last one has
29folded it in. Local runs read: `render` is what `fm ci.timings`
30prints, the latest value, p50, and p90 per metric over the window, a
31platform that ran as shards as one section with a line per shard, the
32movers against a base window, and each section's trend.
33"""
34
35from __future__ import annotations(empty)
36
37import json(empty)
38import math(empty)
39import re(empty)
40import statistics(empty)
41from collections import defaultdict(empty)
42from collections.abc import Mapping(empty)
43from datetime import UTC, datetime, timedelta(empty)
44from pathlib import Path(empty)
45from typing import TYPE_CHECKING, Any(empty)
46
47from livery import git(empty)
48from livery.workshop._ci_context import RunContext, state_store(empty)
49from livery.workshop._shard import ((empty)
50 DURATIONS,
51 Shard,
52 marker_fields,
53 platform_leg,
54 read_shard,
55)
56from livery.workshop._speed import forge_shard_row, leg_of, package_of(empty)
57
58if TYPE_CHECKING:(empty)
59 from livery.forge import Job, Repository, Run
60
61#: The rows' schema. A reader skips a row of another version and
62#: names it; a new field is the same version, a changed meaning is
63#: the next.
64SCHEMA = 1(empty)
65
66#: The series the gate job writes: one file per run, the newest 300
67#: kept. Three hundred runs at a few kilobytes each keeps the tree
68#: under a megabyte and a quarter's trend in reach.
69SERIES = git.Series("metrics", window=300, schema=SCHEMA)(empty)
70
71#: The per-run family: one series per run and check leg, keyed
72#: ``(run, leg)``, holding the leg's half of its timing row, and beside
73#: it every file the leg's entries left for its one write, until the
74#: run's last green job drops the refs, those files included. A half
75#: older than six hours outlived a run that was cancelled or died
76#: before its last job could drop, and the janitor drops it: no run
77#: lasts that long, and the forge has no run-by-id lookup to ask.
78RUNS: git.Keyed = git.Keyed((empty)
79 "run", ("run", "leg"), schema=SCHEMA, stale_after=timedelta(hours=6)
80)
81
82#: The leg's own file on its per-run ref: the task half of its row.
83ROW_FILE = "row.json"(empty)
84
85#: Where a check leg's entries leave the files its one write puts on
86#: its per-run ref beside the row, one JSON object per file, each put
87#: under its own name.
88LEG_SPOOL = ".workshop/leg"(empty)
89
90#: Where the gate job's entries leave the sections the run's file
91#: carries beside the timings, one file per section, named after it:
92#: a number per package, which `fm ci.timings` renders as a trend.
93RUN_ROW = ".workshop/run-row"(empty)
94
95#: The slowest tests a leg's row names, by their call phase.
96SLOWEST = 20(empty)
97
98#: The gate's steps among a row's tasks: the task ``check`` and the
99#: step's title, ``<phase>.<step>``. A task under ``check/`` spelled
100#: otherwise is a step's own command and is not read.
101GATE_STEPS = re.compile(r"^check/[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$")(empty)
102
103#: The test phase's steps, the ones the shards split, start with this.
104#: Every other gate step is unsplit: it runs whole on the first shard.
105TEST_PHASE = "check/test."(empty)
106
107
108def run_ref(run: RunContext, leg: str) -> str:(empty)
109 """The per-run ref one *leg* of *run* writes."""
110 return RUNS.series(run.run_id, leg).ref
111
112
113def leave_leg_file(root: Path, name: str, data: Mapping[str, Any]) -> Path:(empty)
114 """Leave *data* in the leg's spool as *name*, for the leg's one write.
115
116 The leg's last entry, ``ci.leg``, puts every file the spool holds
117 on the leg's per-run ref, a series of [livery.workshop.RUNS][],
118 beside its timing row and under its own name, where the gate job
119 reads it. That write refuses a file named ``row.json``, the timing
120 row's own name. A file left twice under one name keeps the last
121 *data*.
122
123 Returns:
124 The path of the file left under *root*.
125 """
126 spool = root / LEG_SPOOL
127 spool.mkdir(parents=True, exist_ok=True)
128 path = spool / name
129 path.write_text(json.dumps(data, sort_keys=True), encoding="utf-8")
130 return path
131
132
133def leg_files(root: Path) -> tuple[dict[str, dict[str, Any]], str]:(empty)
134 """The files the leg's entries left in its spool, by name; or why one cannot ride.
135
136 A file that is not a JSON object, or one named as the leg's own
137 row, refuses by name: the write would carry what no reader takes.
138 """
139 found: dict[str, dict[str, Any]] = {}
140 spool = root / LEG_SPOOL
141 if not spool.is_dir():
142 return found, ""
143 for path in sorted(spool.glob("*.json")):
144 if path.name == ROW_FILE:
145 return (
146 {},
147 f"{LEG_SPOOL}/{ROW_FILE} is the leg's own row; name the file otherwise",
148 )
149 try:
150 loaded = json.loads(path.read_text("utf-8"))
151 except (OSError, ValueError) as error:
152 return {}, f"{LEG_SPOOL}/{path.name} does not parse ({error})"
153 if not isinstance(loaded, dict):
154 return {}, f"{LEG_SPOOL}/{path.name} is not a JSON object"
155 found[path.name] = loaded
156 return found, ""
157
158
159def leave_section((empty)
160 root: Path, name: str, packages: Mapping[str, float], *, unit: str = ""
161) -> str:
162 """Leave a section of the run's row, a number per package, for the gate job.
163
164 The gate job's ``ci.metrics.collect`` entry, before its verdict,
165 folds every section left at *root* into the run's file on the
166 metrics series, and ``fm ci.timings`` renders each as a trend, the
167 numbers followed by *unit*. A section left twice under one *name*
168 keeps the last numbers.
169
170 Returns:
171 The line to print, naming the section and how many packages
172 it holds.
173 """
174 directory = root / RUN_ROW
175 directory.mkdir(parents=True, exist_ok=True)
176 (directory / f"{name}.json").write_text(
177 json.dumps({"unit": unit, "packages": dict(sorted(packages.items()))}),
178 encoding="utf-8",
179 )
180 return f"{name} row: {len(packages)} package(s) for the run's record"
181
182
183def sections(root: Path) -> dict[str, dict[str, Any]]:(empty)
184 """The sections the gate job's entries left, by name; a malformed file is left out.
185
186 A section is its unit and a number per package.
187 """
188 found: dict[str, dict[str, Any]] = {}
189 directory = root / RUN_ROW
190 if not directory.is_dir():
191 return found
192 for path in sorted(directory.glob("*.json")):
193 try:
194 loaded = json.loads(path.read_text("utf-8"))
195 except (OSError, ValueError):
196 continue
197 section = _section(loaded)
198 if section is not None:
199 found[path.stem] = section
200 return found
201
202
203def _section(raw: Any) -> dict[str, Any] | None:(empty)
204 """*raw* as a section, its unit and a number per package; None when it is none."""
205 if not isinstance(raw, dict):205 ↛ 206line 205 didn't jump to line 206 because the condition on line 205 was never true
206 return None
207 packages = raw.get("packages")
208 if not isinstance(packages, dict):
209 return None
210 numbers = {
211 str(name): float(value)
212 for name, value in packages.items()
213 if isinstance(value, int | float) and not isinstance(value, bool)
214 }
215 return {"unit": str(raw.get("unit", "") or ""), "packages": numbers}
216
217
218def _ms(event: dict[str, Any]) -> float:(empty)
219 return round(float(event.get("dur", 0.0)) / 1000.0, 1)
220
221
222def _track(event: dict[str, Any]) -> int:(empty)
223 return int(event.get("tid", 0))
224
225
226def _start(event: dict[str, Any]) -> float:(empty)
227 return float(event.get("ts", 0.0))
228
229
230def _end(event: dict[str, Any]) -> float:(empty)
231 return _start(event) + float(event.get("dur", 0.0))
232
233
234def _pair_id(event: dict[str, Any]) -> Any:(empty)
235 """What pairs an async begin with its end: the event's id, scoped or not."""
236 scoped = event.get("id2")
237 if isinstance(scoped, dict):237 ↛ 239line 237 didn't jump to line 239 because the condition on line 237 was always true
238 return scoped.get("local", scoped.get("global"))
239 return event.get("id")
240
241
242def _steps_by_task((empty)
243 spans: list[tuple[str, int, float, float]],
244 ran: list[tuple[str, int, float, float, float]],
245) -> dict[str, float]:
246 """What each task's own steps took, by `<task>/<step title>`.
247
248 A step carries no task of its own: the trace nests it under the
249 task it ran in, on that task's track and inside its span, which is
250 how the timeline draws it. So a step belongs to the innermost task
251 containing it on its own track, and a step on another track is not
252 that task's however their times overlap. A title a task ran twice
253 is one key and their sum, since the question a key answers is what
254 the task spent on that command.
255
256 A step no task contains has no address and is left out; the trace
257 writes one only inside a task, so this is a trace that is not the
258 shape it claims.
259 """
260 by_key: dict[str, float] = {}
261 for name, track, start, end, ms in ran:
262 holding = [
263 span
264 for span in spans
265 if span[1] == track and span[2] <= start and end <= span[3]
266 ]
267 if not holding:
268 continue
269 task = min(holding, key=lambda span: span[3] - span[2])[0]
270 key = f"{task}/{name}"
271 by_key[key] = round(by_key.get(key, 0.0) + ms, 1)
272 return by_key
273
274
275def _package_of(nodeid: str) -> str:(empty)
276 """The member a test node is of: ``forge``, ``extensions/widgets``, or empty."""
277 return package_of(nodeid).removeprefix("packages/")
278
279
280def leg_row(trace: Path, *, job: str) -> tuple[dict[str, Any] | None, str]:(empty)
281 """The task half of *job*'s row from its trace; ``(row, "")`` or ``(None, reason)``.
282
283 The trace is the Chrome trace the profiled gate wrote: every
284 task's slice (its request address and duration), the lane waits,
285 each step inside its task, and each pytest phase of every test as
286 a slice of its own. A step is a slice nested in its task, or an
287 async pair (a begin and an end under one id) where it overlaps a
288 sibling on the task's track, which is how footman draws the steps
289 of one ``parallel()`` block; both read as the step's whole wall.
290 The row's ``items`` is the seconds each item took, the test file
291 of each test's node id, every phase of every test in it summed, to
292 a tenth. A missing or unreadable trace is a reason, never a row.
293 """
294 if not trace.is_file():
295 return None, f"no trace at {trace}: the gate did not run under --profile"
296 try:
297 loaded = json.loads(trace.read_text("utf-8"))
298 except (OSError, ValueError) as error:
299 return None, f"the trace at {trace} does not parse: {error}"
300 events = loaded.get("traceEvents") if isinstance(loaded, dict) else loaded
301 if not isinstance(events, list):
302 return None, f"the trace at {trace} carries no event list"
303 tasks: dict[str, float] = {}
304 spans: list[tuple[str, int, float, float]] = []
305 ran: list[tuple[str, int, float, float, float]] = []
306 waits: dict[str, float] = defaultdict(float)
307 packages: dict[str, dict[str, float]] = defaultdict(
308 lambda: {"tests_ms": 0.0, "tests": 0}
309 )
310 # Every phase of every test, summed per item, the test file of its
311 # node id: what a shard partitions the next run's items by.
312 items: dict[str, float] = defaultdict(float)
313 calls: list[tuple[float, str]] = []
314 begins: list[float] = []
315 ends: list[float] = []
316 # A step that overlaps a sibling on its track is an async pair
317 # rather than a slice, since the slices on one track must nest.
318 # Its begin waits here for its end; the two are one span. A begin
319 # the trace never ends is no span.
320 begun: dict[tuple[Any, int, Any], dict[str, Any]] = {}
321 for event in events:
322 if not isinstance(event, dict):322 ↛ 323line 322 didn't jump to line 323 because the condition on line 322 was never true
323 continue
324 phase = event.get("ph")
325 cat = str(event.get("cat", ""))
326 if phase in ("b", "e"):
327 if cat != "step":
328 continue
329 key = (event.get("pid"), _track(event), _pair_id(event))
330 if phase == "b":
331 begun[key] = event
332 elif (opened := begun.pop(key, None)) is not None:332 ↛ 337line 332 didn't jump to line 337 because the condition on line 332 was always true
333 start, end = _start(opened), _start(event)
334 ms = round((end - start) / 1000.0, 1)
335 name = str(opened.get("name", ""))
336 ran.append((name, _track(event), start, end, ms))
337 continue
338 if phase != "X":
339 continue
340 name = str(event.get("name", ""))
341 if event.get("pid") == 1:
342 begins.append(float(event.get("ts", 0.0)))
343 ends.append(float(event.get("ts", 0.0)) + float(event.get("dur", 0.0)))
344 if cat == "task":
345 tasks[name] = _ms(event)
346 spans.append((name, _track(event), _start(event), _end(event)))
347 elif cat == "step":
348 ran.append((name, _track(event), _start(event), _end(event), _ms(event)))
349 elif cat == "lane":
350 waits[name.removeprefix("lane: ")] += _ms(event)
351 elif cat.startswith("test."):
352 items[name.split("::", 1)[0]] += _ms(event)
353 package = _package_of(name)
354 if package:
355 packages[package]["tests_ms"] = round(
356 packages[package]["tests_ms"] + _ms(event), 1
357 )
358 if cat == "test.call":
359 packages[package]["tests"] += 1
360 calls.append((_ms(event), name))
361 if not tasks:
362 return None, f"the trace at {trace} records no task: nothing to row"
363 steps = _steps_by_task(spans, ran)
364 total = round((max(ends) - min(begins)) / 1000.0, 1) if begins else 0.0
365 # The slowest tests by their call phase, for the speed judge's
366 # warning to name; the per-package sums count every phase.
367 calls.sort(reverse=True)
368 return {
369 "job": job,
370 "total_ms": total,
371 "tasks": dict(sorted(tasks.items())),
372 "task_steps": dict(sorted(steps.items())),
373 "waits_ms": {name: round(ms, 1) for name, ms in sorted(waits.items())},
374 "packages": dict(sorted(packages.items())),
375 "items": {item: round(ms / 1000.0, 1) for item, ms in sorted(items.items())},
376 "slowest": [
377 {"test": name, "s": round(ms / 1000.0, 2)} for ms, name in calls[:SLOWEST]
378 ],
379 }, ""
380
381
382def put_leg((empty)
383 root: Path, run: RunContext, *, job: str, label: str, trace: Path
384) -> tuple[list[str], str]:
385 """Put *job*'s task half and the leg's spool on its per-run ref, in one write.
386
387 Returns the lines to print and why the write failed, ``""`` when
388 it landed or there was nothing to write. The row is read from
389 *trace* ([livery.workshop._metrics.leg_row][]), with the scope the
390 gate ran; a trace that cannot be read leaves the row out, says
391 why, and the spool's files go alone. A leg that ran as one shard
392 of several left what it partitioned in `SHARD_FILE`
393 ([livery.workshop.read_shard][]), and the row carries that record
394 under ``shard`` for the gate job's fold; a leg that is one shard
395 of one leaves no such file and its row carries no ``shard``. Every
396 file the leg's entries left in `LEG_SPOOL` rides beside the row
397 under its own name ([livery.workshop._metrics.leg_files][]), so
398 the leg writes its ref once. A leg without a label, a shard file
399 that is not a record, and a spool file no reader takes, fail the
400 write by name.
401 """
402 lines: list[str] = []
403 if not label:403 ↛ 404line 403 didn't jump to line 404 because the condition on line 403 was never true
404 return lines, "the leg has no label, so its row has no key"
405 files: dict[str, Mapping[str, Any]] = {}
406 row, why = leg_row(trace, job=job)
407 if row is None:
408 lines.append(f" {job}: {why}; the timing row stays unwritten")
409 else:
410 from dataclasses import asdict
411
412 from livery.workshop._verified import leg_scope
413
414 # The scope the gate ran, from the marker it left beside the
415 # trace: the stamp after the verdict reads it back per leg.
416 scope = leg_scope(trace.parent if trace.is_absolute() else root)
417 row["scope"] = {**asdict(scope), "packages": list(scope.packages)}
418 # What the shard partitioned, for the gate job's proof that the
419 # platform's shards are one leg's work.
420 marker, why = read_shard(root)
421 if why:
422 return lines, why
423 if marker is not None:
424 row["shard"] = marker_fields(marker)
425 files[ROW_FILE] = row
426 spooled, why = leg_files(root)
427 if why:
428 return lines, why
429 files.update(spooled)
430 if not files:
431 return lines, ""
432 why = RUNS.series(run.run_id, label).put(
433 state_store(root), files, message=f"metrics: {job} of run {run.run_id}"
434 )
435 if why:
436 return lines, why
437 lines.append(f" {label}: {', '.join(sorted(files))} on run {run.run_id}'s ref")
438 return lines, ""
439
440
441def _moment(stamp: str) -> datetime | None:(empty)
442 if not stamp:442 ↛ 443line 442 didn't jump to line 443 because the condition on line 442 was never true
443 return None
444 try:
445 return datetime.fromisoformat(stamp.replace("Z", "+00:00"))
446 except ValueError:
447 return None
448
449
450def _between(start: str, end: str) -> float | None:(empty)
451 """Milliseconds from *start* to *end*, or ``None`` when either is unknown."""
452 a, b = _moment(start), _moment(end)
453 if a is None or b is None:453 ↛ 454line 453 didn't jump to line 454 because the condition on line 453 was never true
454 return None
455 return round((b - a).total_seconds() * 1000.0, 1)
456
457
458def run_file(run_id: str) -> str:(empty)
459 """The series file for *run_id*: zero-padded when numeric, so names sort by time."""
460 return f"{int(run_id):020d}.json" if run_id.isdigit() else f"{run_id}.json"
461
462
463def _times(job: Job, *, created: str, now: str) -> dict[str, Any]:(empty)
464 """A job's queue, wall and step times from the forge's stamps.
465
466 A job still running when the gate job collects (the gate job is
467 always one) is measured to *now*, its running step too, and marked
468 incomplete, so the row reads as a real duration with its end named
469 rather than as a time the forge never set. A job the forge has not
470 started carries no queue or wall time.
471 """
472 started = job.started_at
473 return {
474 "complete": bool(job.completed_at),
475 "queued_ms": _between(created, started) if started else None,
476 "wall_ms": _between(started, job.completed_at or now) if started else None,
477 "steps": [
478 {
479 "name": step.name,
480 "ms": (
481 _between(step.started_at, step.completed_at or now)
482 if step.started_at
483 else None
484 ),
485 }
486 for step in job.steps
487 ],
488 }
489
490
491def _forge_row(job: Job, *, created: str, now: str) -> dict[str, Any]:(empty)
492 """A job's times as the forge reports them, for a job that left no trace.
493
494 The docs build, the merge point's jobs, and a gate job that ran
495 without a profile have no leg row; their queue and wall times come
496 from the forge alone, measured as [livery.workshop._metrics._times][]
497 measures every job's.
498 """
499 return {
500 "conclusion": job.conclusion,
501 "status": job.status,
502 **_times(job, created=created, now=now),
503 }
504
505
506def _forge_jobs((empty)
507 repo: Repository, run: RunContext, *, sha: str
508) -> tuple[Run | None, dict[str, Job], list[str]]:
509 """The forge's run for *run* and its jobs by name; the lines about the lookup.
510
511 The run is looked up under the head the runner's event names,
512 because a pull request's checkout is a merge commit the forge
513 never files a run under; *sha* is the checkout, the fallback. A
514 forge that numbers the run differently from the environment (a
515 pull request's merge sha, say) still yields the newest run for
516 the sha, named as a substitution.
517 """
518 from livery.forge import ForgeError
519
520 lines: list[str] = []
521 head = run.head_sha or sha
522 try:
523 runs = repo.checks.runs(head_sha=head)
524 forge_run = next((r for r in runs if str(r.id) == run.run_id), None)
525 if forge_run is None and runs:
526 forge_run = runs[0]
527 lines.append(
528 f" run {run.run_id}: not among the forge's runs for {head[:12]};"
529 f" using the newest, run {forge_run.id}"
530 )
531 jobs = (
532 {job.name: job for job in repo.checks.jobs(forge_run.id)}
533 if forge_run
534 else {}
535 )
536 except ForgeError as error:
537 # A job without a forge credential (one that only builds) can
538 # push its half and read nothing back: the record stays a
539 # record, never a verdict, so the question's refusal is a line.
540 lines.append(
541 f" run {run.run_id}: the forge could not be asked for its jobs"
542 f" ({error}); the rows carry their trace halves alone"
543 )
544 return None, {}, lines
545 return forge_run, jobs, lines
546
547
548def collect(root: Path, repo: Repository, run: RunContext, *, sha: str) -> list[str]:(empty)
549 """Join the legs' halves with the forge's times into the run's file; what happened.
550
551 Every line names what was done or why not. Nothing is written
552 when no leg left a row. The per-run refs stay: `drop_halves`
553 takes them once every half is in, so a re-run of a red gate job
554 still unions what its legs put, and a point whose every job
555 collects folds each earlier job's half into its own write. A job
556 the forge does not know keeps its trace half and is named; a row
557 of another schema is skipped and named. The run is looked up
558 under the head the runner's event names (`_forge_jobs`); *sha*
559 is the checkout, the fallback and the row's own record. Every
560 job the forge lists is recorded, the legs with their traces, the
561 others with their forge times alone, and the run's own wall from
562 its start to this collection. A platform that ran as shards has
563 one row, folded under the platform's display name before the
564 collect ([livery.workshop._fold][]), a name the forge never ran:
565 the row takes the forge times of the first shard's job and
566 concludes as the shards' jobs did, and the forge's rows of the
567 shards stand beside it with their times alone, for
568 ``fm ci.timings``. After the run's file lands, each check leg's
569 seconds per item go onto its platform leg's series of the
570 [livery.workshop.DURATIONS][] family, one row per run, which the
571 next run's shards partition by; a leg without them writes none,
572 and a refused write is a line.
573 """
574 lines: list[str] = []
575 keys = RUNS.listed(state_store(root), run.run_id)
576 if keys is None:576 ↛ 577line 576 didn't jump to line 577 because the condition on line 576 was never true
577 return [
578 f" {RUNS.prefix}{run.run_id}/*: the remote could not be listed;"
579 " nothing collected"
580 ]
581 halves: dict[str, dict[str, Any]] = {}
582 for key in keys:
583 series = RUNS.series(*key)
584 found, why = series.row(state_store(root), ROW_FILE)
585 if why:
586 lines.append(f" {series.ref}: {why}; skipped")
587 continue
588 if found is None:588 ↛ 589line 588 didn't jump to line 589 because the condition on line 588 was never true
589 lines.append(f" {series.ref}: empty; skipped")
590 continue
591 halves[str(found.data.get("job", key[-1]))] = found.data
592 if not halves:
593 lines.append(f" run {run.run_id}: no leg left a row; nothing collected")
594 return lines
595 head = run.head_sha or sha
596 forge_run, jobs, looked = _forge_jobs(repo, run, sha=sha)
597 lines.extend(looked)
598 if forge_run is None and not looked:
599 lines.append(
600 f" run {run.run_id}: the forge lists no such run for {head[:12]};"
601 " the rows carry their trace halves alone"
602 )
603 entry: dict[str, Any] = {
604 "forge": run.forge,
605 "run": run.run_id,
606 "workflow": forge_run.workflow if forge_run else "",
607 "event": run.event,
608 "sha": head,
609 "checkout": sha,
610 "ref": run.ref,
611 "created_at": forge_run.created_at if forge_run else "",
612 "started_at": forge_run.started_at if forge_run else "",
613 "collected_at": datetime.now(UTC).isoformat(timespec="seconds"),
614 "jobs": {},
615 }
616 for name, half in sorted(halves.items()):
617 job = jobs.get(name)
618 row: dict[str, Any] = {
619 "total_ms": half.get("total_ms"),
620 "tasks": half.get("tasks", {}),
621 "task_steps": half.get("task_steps", {}),
622 "waits_ms": half.get("waits_ms", {}),
623 "packages": half.get("packages", {}),
624 "slowest": half.get("slowest", []),
625 "scope": half.get("scope", {"scope": "unknown", "packages": []}),
626 }
627 shard = half.get("shard")
628 if isinstance(shard, dict):
629 row["shard"] = shard
630 conclusion: str = job.conclusion if job is not None else ""
631 if job is None and isinstance(shard, dict):
632 job, conclusion, line = _folded_job(name, shard, jobs)
633 if line:
634 lines.append(line)
635 if job is None:
636 lines.append(f" {name}: unknown to the forge; its trace half rides alone")
637 else:
638 row["conclusion"] = conclusion
639 row.update(
640 _times(job, created=entry["created_at"], now=entry["collected_at"])
641 )
642 entry["jobs"][name] = row
643 for name, job in sorted(jobs.items()):
644 if name not in entry["jobs"]:
645 entry["jobs"][name] = _forge_row(
646 job, created=entry["created_at"], now=entry["collected_at"]
647 )
648 entry["run_wall_ms"] = (
649 _between(entry["started_at"], entry["collected_at"])
650 if entry["started_at"]
651 else None
652 )
653 left = sections(root)
654 if left:
655 entry["sections"] = left
656 for name, section in left.items():
657 lines.append(
658 f" {name}: {len(section['packages'])} package(s) recorded on the run"
659 )
660 why = SERIES.put(
661 state_store(root),
662 {run_file(run.run_id): entry},
663 message=f"metrics: run {run.run_id} at {sha[:12]}",
664 )
665 if why:665 ↛ 666line 665 didn't jump to line 666 because the condition on line 665 was never true
666 lines.append(f" {SERIES.ref}: {why}; the per-run refs stay for a re-run")
667 return lines
668 lines.append(
669 f" {SERIES.ref}: run {run.run_id} recorded, {len(entry['jobs'])} job(s)"
670 )
671 lines.extend(_put_durations(root, run, halves))
672 return lines
673
674
675def _green(job: Job) -> bool:(empty)
676 """Whether *job* has completed and did not fail."""
677 return job.status == "completed" and job.conclusion in ("success", "skipped")
678
679
680def drop_halves((empty)
681 root: Path,
682 repo: Repository,
683 run: RunContext,
684 *,
685 sha: str,
686 job: str,
687 writers: tuple[str, ...],
688) -> list[str]:
689 """Drop the run's per-run refs once every half is in and nothing is red.
690
691 Runs as the last entry of a job that collected, after its own
692 verdict, so its own job is green when it runs and is left out of
693 the question. *writers* names the jobs that put a half, as the
694 schedule spells them (``check``); a forge job answers to a writer
695 by its display name's prefix (``check (ubuntu-latest, 3.14)``).
696 The refs stay, and the lines say so, while a writer other than
697 *job* has not completed green or any completed job is red: a
698 re-run of a red job finds what the legs put, and the last of a
699 point's collecting jobs is the one that drops. A job the forge
700 does not list, or a run it does not know, keeps the refs too; the
701 janitor's six-hour sweep is the net under every kept ref.
702 """
703 lines: list[str] = []
704 keys = RUNS.listed(state_store(root), run.run_id)
705 if keys is None:705 ↛ 706line 705 didn't jump to line 706 because the condition on line 705 was never true
706 return [f" {RUNS.prefix}{run.run_id}/*: the remote could not be listed; kept"]
707 if not keys:
708 return [f" run {run.run_id}: no per-run ref to drop"]
709 forge_run, jobs, looked = _forge_jobs(repo, run, sha=sha)
710 lines.extend(looked)
711 if forge_run is None:
712 lines.append(f" run {run.run_id}: the forge's jobs are unknown; the refs stay")
713 return lines
714 waiting = sorted(
715 f"{name}: {found.conclusion or found.status}"
716 for name, found in jobs.items()
717 if name != job
718 and any(name == writer or name.startswith(f"{writer} (") for writer in writers)
719 and not _green(found)
720 )
721 red = sorted(
722 f"{name}: {found.conclusion}"
723 for name, found in jobs.items()
724 if found.status == "completed" and not _green(found)
725 )
726 if waiting or red:
727 lines.append(
728 " the per-run refs stay: " + ", ".join(dict.fromkeys([*red, *waiting]))
729 )
730 return lines
731 for key in sorted(keys):
732 ref = RUNS.series(*key).ref
733 gone = git.drop(state_store(root), ref)
734 lines.append(f" {ref}: {gone or 'dropped'}")
735 return lines
736
737
738def _folded_job((empty)
739 name: str, shard: Mapping[str, Any], jobs: Mapping[str, Job]
740) -> tuple[Job | None, str, str]:
741 """The forge job a folded row takes its times from, its conclusion, and the line.
742
743 The folded row of a platform's shards is named by the platform's
744 display name (``check (windows-latest, 3.14)``), which the forge
745 never ran; its ``shard`` lists the shards' jobs. The row takes the
746 first listed job the forge knows, and concludes as the shards'
747 jobs did: ``success`` when every known one did, else the first
748 other conclusion. ``(None, "", "")`` when the forge knows none of
749 them.
750 """
751 listed: Any = shard.get("shards")
752 names = (
753 [str(item.get("job", "")) for item in listed if isinstance(item, dict)]
754 if isinstance(listed, list)
755 else []
756 )
757 known = [jobs[job] for job in names if job in jobs]
758 if not known:
759 return None, "", ""
760 first = known[0]
761 conclusion = next(
762 (job.conclusion for job in known if job.conclusion != "success"),
763 first.conclusion,
764 )
765 return (
766 first,
767 conclusion,
768 f" {name}: the forge times of {first.name}, the first of {len(names)}"
769 " shard(s)",
770 )
771
772
773def _number(value: Any) -> float | None:(empty)
774 if isinstance(value, bool) or not isinstance(value, int | float):774 ↛ 775line 774 didn't jump to line 775 because the condition on line 774 was never true
775 return None
776 return float(value)
777
778
779def gate_steps(row: Mapping[str, Any]) -> dict[str, float]:(empty)
780 """The gate's steps of a leg's *row* by task key, in milliseconds.
781
782 The row's tasks and the steps its tasks ran (``task_steps``) whose
783 key `GATE_STEPS` matches; a key in both is read from the tasks.
784 """
785 found: dict[str, float] = {}
786 for field in ("tasks", "task_steps"):
787 values = row.get(field)
788 if not isinstance(values, Mapping):
789 continue
790 for key, value in values.items():
791 ms = _number(value)
792 if GATE_STEPS.match(str(key)) and ms is not None:
793 found.setdefault(str(key), ms)
794 return found
795
796
797def tests_ms(packages: Any) -> float:(empty)
798 """The test time summed over *packages*, a row's ``packages``, in milliseconds."""
799 if not isinstance(packages, Mapping):799 ↛ 800line 799 didn't jump to line 800 because the condition on line 799 was never true
800 return 0.0
801 return sum(
802 _number(package.get("tests_ms")) or 0.0
803 for package in packages.values()
804 if isinstance(package, Mapping)
805 )
806
807
808def first_shard_head(row: Mapping[str, Any]) -> float | None:(empty)
809 """The item seconds the first shard's unsplit steps stand for on *row*.
810
811 The unsplit steps are the gate's steps outside the test phase
812 (`TEST_PHASE`); only a platform's first shard runs them, and the
813 folded row of a platform's shards carries the first shard's steps.
814 On the sharded runs the first shard's wall is about the summed wall
815 W of those steps plus its slice's test wall, not the longer of the
816 two: the steps hold the shard's cores while they run, and its tests
817 wait. The partition packs item seconds, and a second of a shard's
818 test wall holds p of them, p being the parallelism its tests reach,
819 so the head is W times p. p is the median, over the shards after
820 the first that the folded row lists under ``shard``, of each one's
821 summed test time over its wall; with no such shard, the row's own
822 summed test time over its test steps' wall. When neither reads, the
823 head is the longest unsplit step's wall, the least the first
824 shard's tests wait.
825
826 Returns:
827 The head, to a tenth of a second; ``None`` when the row names
828 no unsplit step.
829 """
830 steps = gate_steps(row)
831 unsplit = [ms for key, ms in steps.items() if not key.startswith(TEST_PHASE)]
832 if not unsplit:
833 return None
834 speed = _test_speed(row, steps)
835 if speed is None:
836 return round(max(unsplit) / 1000.0, 1)
837 return round(sum(unsplit) / 1000.0 * speed, 1)
838
839
840def _test_speed(row: Mapping[str, Any], steps: Mapping[str, float]) -> float | None:(empty)
841 """The parallelism *row*'s tests reached, as `first_shard_head` reads it."""
842 shard = row.get("shard")
843 listed: Any = shard.get("shards") if isinstance(shard, Mapping) else None
844 own: Any = row.get("packages")
845 speeds: list[float] = []
846 for item in listed if isinstance(listed, list) else ():
847 if not isinstance(item, Mapping):
848 continue
849 try:
850 first = Shard.parse(str(item.get("shard", ""))).first
851 except ValueError:
852 continue
853 if first:
854 own = item.get("packages")
855 continue
856 wall = _number(item.get("total_ms")) or 0.0
857 tests = tests_ms(item.get("packages"))
858 if wall > 0 and tests > 0:
859 speeds.append(tests / wall)
860 if speeds:
861 return statistics.median(speeds)
862 wall = sum(ms for key, ms in steps.items() if key.startswith(TEST_PHASE))
863 tests = tests_ms(own)
864 return tests / wall if wall > 0 and tests > 0 else None
865
866
867def _put_durations((empty)
868 root: Path, run: RunContext, halves: Mapping[str, Mapping[str, Any]]
869) -> list[str]:
870 """Write each leg's seconds per item onto its platform leg's series; the lines.
871
872 One row per leg whose half carries ``items``, named as the run's
873 file ([livery.workshop._metrics.run_file][]), on the series of
874 [livery.workshop.DURATIONS][] keyed by the platform leg the leg's
875 label names ([livery.workshop.platform_leg][]), with the first
876 shard's head (`first_shard_head`) under ``head`` when the half
877 names an unsplit step. A half without ``items`` writes nothing.
878 The row of one shard of several writes nothing either, since the
879 fold's row carries the platform's
880 items; a shard's row reaches the collect only when the fold did
881 not run. A refused write is a line, never a red job.
882 """
883 lines: list[str] = []
884 for name, half in sorted(halves.items()):
885 items = half.get("items")
886 if not isinstance(items, dict) or not items:
887 continue
888 shard = half.get("shard")
889 if isinstance(shard, dict) and "shards" not in shard:
890 lines.append(
891 f" {name}: one shard's row; the platform's folded row carries"
892 " the durations"
893 )
894 continue
895 leg = platform_leg(leg_of(name, half))
896 series = DURATIONS.series(leg)
897 written: dict[str, Any] = {"run": run.run_id, "items": items}
898 head = first_shard_head(half)
899 if head is not None:
900 written["head"] = head
901 why = series.put(
902 state_store(root),
903 {run_file(run.run_id): written},
904 message=f"durations: {leg} from run {run.run_id}",
905 )
906 if why:906 ↛ 907line 906 didn't jump to line 907 because the condition on line 906 was never true
907 lines.append(f" {series.ref}: not written ({why})")
908 else:
909 lines.append(
910 f" {series.ref}: {len(items)} item(s) recorded for run {run.run_id}"
911 + (f", a head of {head:.1f} s" if head is not None else "")
912 )
913 return lines
914
915
916def _percentile(values: list[float], fraction: float) -> float:(empty)
917 """The nearest-rank percentile of *values*: the value at rank ceil(p n)."""
918 ordered = sorted(values)
919 rank = max(1, math.ceil(fraction * len(ordered)))
920 return ordered[rank - 1]
921
922
923def _metrics(entry: dict[str, Any]) -> dict[str, dict[str, float]]:(empty)
924 """Every numeric metric of a run's file, by job then metric name.
925
926 The run's own wall, start to collection, renders as the ``run``
927 group, so the floor the legs never touch has a row of its own. The
928 forge's own row of a shard
929 ([livery.workshop._speed.forge_shard_row][]) is left out: the
930 platform's folded row carries the shards' work, and `render` prints
931 each shard's times under it, so neither a section nor a mover is a
932 shard's.
933 """
934 out: dict[str, dict[str, float]] = {}
935 wall = entry.get("run_wall_ms")
936 if isinstance(wall, int | float):
937 out["run"] = {"wall_ms": float(wall)}
938 for name, row in sorted(entry.get("jobs", {}).items()):
939 if forge_shard_row(name, row):
940 continue
941 metrics: dict[str, float] = {}
942 for key in ("queued_ms", "wall_ms", "total_ms"):
943 value = row.get(key)
944 if isinstance(value, int | float):
945 metrics[key] = float(value)
946 for task, ms in row.get("tasks", {}).items():
947 metrics[f"task {task}"] = float(ms)
948 for step in row.get("steps", []):
949 if isinstance(step.get("ms"), int | float):
950 metrics[f"step {step['name']}"] = float(step["ms"])
951 # The job's steps above are the workflow's; these are the ones a
952 # task ran, addressed under it, so a checker that moved is named
953 # rather than left inside its task's total.
954 for key, ms in row.get("task_steps", {}).items():954 ↛ 955line 954 didn't jump to line 955 because the loop on line 954 never started
955 metrics[f"step {key}"] = float(ms)
956 for package, data in row.get("packages", {}).items():
957 metrics[f"tests {package}"] = float(data.get("tests_ms", 0.0))
958 out[name] = metrics
959 return out
960
961
962def render((empty)
963 root: Path, *, since: int = 1, base: int = 20, movers: int = 10
964) -> list[str]:
965 """The timings, rendered: per job and metric the latest, p50, p90, and the movers.
966
967 The window is every run the series holds. The latest *since* runs
968 are the recent side; the *base* runs before them are the base the
969 movers compare against, by the change of the recent median over
970 the base median. A platform that ran as shards is one section, its
971 folded row, and under its metrics one line per shard with the
972 shard's wall and queue times from the newest run, read from the
973 forge's own rows of the shards; those rows are no sections of their
974 own, and the movers read the platform legs alone. Nothing yet, an
975 unreadable series, and a row of another schema each say so.
976 """
977 found = SERIES.rows(state_store(root))
978 if found.failed:
979 return [f" {found.reason}"]
980 if not found.rows and not found.skipped:
981 return [" no timing rows yet: the gate job writes one per run"]
982 lines = [f" {line}" for line in found.skipped]
983 # Oldest first: the trend reads the newest run at the end.
984 entries = [row.data for row in reversed(found.rows)]
985 if not entries:985 ↛ 986line 985 didn't jump to line 986 because the condition on line 985 was never true
986 lines.append(" no readable timing rows")
987 return lines
988 newest = entries[-1]
989 lines.append(
990 f" {len(entries)} run(s), latest {newest.get('run')}"
991 f" at {str(newest.get('sha', ''))[:12]}"
992 )
993 recent, older = entries[-since:], entries[:-since]
994 base_entries = older[-base:] if base else older
995 series: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
996 for entry in entries:
997 for job, metrics in _metrics(entry).items():
998 for metric, value in metrics.items():
999 series[job][metric].append(value)
1000 latest = _metrics(entries[-1])
1001 for job in sorted(series):
1002 lines.append(f" {job}")
1003 if newest.get("jobs", {}).get(job, {}).get("complete") is False:
1004 lines.append(
1005 " still running when the gate job collected: measured to the"
1006 " collection"
1007 )
1008 lines.append(f" {'metric':<44} {'latest':>9} {'p50':>9} {'p90':>9}")
1009 for metric, values in sorted(series[job].items()):
1010 now = latest.get(job, {}).get(metric)
1011 lines.append(
1012 f" {metric[:44]:<44} {_seconds(now):>9}"
1013 f" {_percentile(values, 0.5) / 1000:8.1f}s"
1014 f" {_percentile(values, 0.9) / 1000:8.1f}s"
1015 )
1016 lines.extend(_shard_lines(newest, job))
1017 if base_entries and recent:
1018 deltas: list[tuple[float, str, str, float, float]] = []
1019 recent_series: dict[str, dict[str, list[float]]] = defaultdict(
1020 lambda: defaultdict(list)
1021 )
1022 base_series: dict[str, dict[str, list[float]]] = defaultdict(
1023 lambda: defaultdict(list)
1024 )
1025 for entry in recent:
1026 for job, metrics in _metrics(entry).items():
1027 for metric, value in metrics.items():
1028 recent_series[job][metric].append(value)
1029 for entry in base_entries:
1030 for job, metrics in _metrics(entry).items():
1031 for metric, value in metrics.items():
1032 base_series[job][metric].append(value)
1033 for job, per_metric in recent_series.items():
1034 for metric, recent_values in per_metric.items():
1035 before = base_series.get(job, {}).get(metric)
1036 if not before:1036 ↛ 1037line 1036 didn't jump to line 1037 because the condition on line 1036 was never true
1037 continue
1038 now, then = _percentile(recent_values, 0.5), _percentile(before, 0.5)
1039 deltas.append((now - then, job, metric, now, then))
1040 deltas.sort(key=lambda d: -abs(d[0]))
1041 lines.append(
1042 f" movers: the last {len(recent)} run(s) against the"
1043 f" {len(base_entries)} before"
1044 )
1045 for delta, job, metric, now, then in deltas[:movers]:
1046 sign = "+" if delta >= 0 else "-"
1047 lines.append(
1048 f" {sign}{abs(delta) / 1000:.1f}s {job}: {metric}"
1049 f" ({then / 1000:.1f}s -> {now / 1000:.1f}s)"
1050 )
1051 lines.extend(_render_sections(entries))
1052 return lines
1053
1054
1055def _seconds(ms: Any) -> str:(empty)
1056 """*ms* as seconds to a tenth, ``-`` when it is no number."""
1057 return f"{ms / 1000:.1f}s" if isinstance(ms, int | float) else "-"
1058
1059
1060def _shard_lines(entry: dict[str, Any], job: str) -> list[str]:(empty)
1061 """One line per shard of *job* in *entry*: the shard's wall and queue times.
1062
1063 A platform's folded row lists its shards under ``shard.shards``,
1064 each naming the forge's job the shard ran as; the forge's own row
1065 of that job in the same run's file carries the shard's times. A row
1066 without shards gives no line; a shard whose forge row the file
1067 lacks, or whose time the forge did not set, shows ``-``.
1068 """
1069 jobs: Any = entry.get("jobs", {})
1070 row: Any = jobs.get(job) if isinstance(jobs, dict) else None
1071 shard: Any = row.get("shard") if isinstance(row, dict) else None
1072 listed: Any = shard.get("shards") if isinstance(shard, dict) else None
1073 if not isinstance(listed, list):
1074 return []
1075 lines: list[str] = []
1076 for item in listed:
1077 if not isinstance(item, dict):1077 ↛ 1078line 1077 didn't jump to line 1078 because the condition on line 1077 was never true
1078 continue
1079 forge: Any = jobs.get(str(item.get("job", "")))
1080 wall = forge.get("wall_ms") if isinstance(forge, dict) else None
1081 queued = forge.get("queued_ms") if isinstance(forge, dict) else None
1082 lines.append(
1083 f" shard {item.get('shard', '')}: wall {_seconds(wall)},"
1084 f" queued {_seconds(queued)}"
1085 )
1086 return lines
1087
1088
1089def _sections_of(entry: dict[str, Any]) -> dict[str, dict[str, Any]]:(empty)
1090 """The sections *entry* carries, by name, each its unit and its numbers per package.
1091
1092 A row without ``sections`` carries none.
1093 """
1094 found: dict[str, dict[str, Any]] = {}
1095 raw = entry.get("sections")
1096 if isinstance(raw, dict):
1097 for name, value in raw.items():
1098 section = _section(value)
1099 if section is not None:1099 ↛ 1097line 1099 didn't jump to line 1097 because the condition on line 1099 was always true
1100 found[str(name)] = section
1101 return found
1102
1103
1104def _render_sections(entries: list[dict[str, Any]]) -> list[str]:(empty)
1105 """Each section's rows: per package the latest, p50, and p90 number."""
1106 series: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
1107 latest: dict[str, dict[str, float]] = defaultdict(dict)
1108 units: dict[str, str] = {}
1109 for entry in entries:
1110 for name, section in _sections_of(entry).items():
1111 units[name] = section["unit"]
1112 for package, value in sorted(section["packages"].items()):
1113 series[name][package].append(value)
1114 latest[name][package] = value
1115 lines: list[str] = []
1116 for name in sorted(series):
1117 unit = units[name]
1118 lines += [
1119 f" {name}",
1120 f" {'package':<44} {'latest':>9} {'p50':>9} {'p90':>9}",
1121 ]
1122 for package, values in sorted(series[name].items()):
1123 lines.append(
1124 f" {package[:44]:<44} {latest[name][package]:8.1f}{unit}"
1125 f" {_percentile(values, 0.5):8.1f}{unit}"
1126 f" {_percentile(values, 0.9):8.1f}{unit}"
1127 )
1128 return lines