Coverage for packages/workshop/src/livery/workshop/_ci_tasks.py: 80%

Measured by coverage.py 7.16.2 on CPython 3.14 while the suites of tests, packages/workshop, packages/extensions/ty, packages/extensions/ruff, packages/extensions/python, packages/extensions/pytest, packages/extensions/pyrefly, packages/extensions/nanobind, packages/extensions/mypy, packages/extensions/mkdocstrings, packages/extensions/lfs, packages/extensions/docs, packages/extensions/cpp, packages/extensions/coverage, packages/extensions/conan, packages/extensions/cmake, packages/extensions/claude, packages/extensions/clang-tidy, packages/extensions/clang-format, packages/extensions/changelog, packages/extensions/basedpyright ran.

1"""``fm status`` and the ``ci`` group.
2
3``status`` says where the current branch's pull request stands and
4exits that state's code (0 while nothing is wrong); ``--watch``
5follows instead of reading once. The ``ci`` group acts on the head
6commit's runs: ``ci.rerun`` re-runs the failed jobs, ``ci.cancel``
7cancels what is still moving (the relief for a wedged queue),
8``ci.status`` says where the head commit's runs stand and exits that
9state's code, so a script can wait on main after a merge, and
10``ci.logs`` prints the job logs, the one read that stays here so
11logs reach an agent through fm; both read the newest run of a point
12instead under ``--point``, which is how the nightly's verdict reaches
13a person. ``ci.dispatch`` starts a point that has a dispatch entry
14(the gate, the nightly) and follows it to its verdict. ``ci.run``
15runs one job of a point
16(livery.workshop._points), the one verb the emitted shells call;
17``ci.verdict`` is the gate job's judgement of the jobs it needs.
18``ci.timings`` prints the timing rows the gate writes on
19that store (livery.workshop._metrics), ``ci.balance`` reads the
20same rows to predict each platform's wall at one to several shards
21and propose the counts (livery.workshop._balance), and
22``ci.footprint`` sums the same rows into runner-minutes per runner
23label, priced where GitHub would bill them
24(livery.workshop._footprint); the hidden
25``ci.metrics.collect`` is the gate job's writer ``ci.run`` schedules,
26the hidden ``ci.fold`` folds each platform's shards into one leg
27before the judges read, and the hidden ``ci.leg`` is each check leg's
28one write of its row.
29"""
30
31from __future__ import annotations
32
33from pathlib import Path
34from typing import Annotated
35
36import livery.footman as footman
37from livery import git
38from livery.footman import doc, fail, group, pre_tasks, task
39from livery.forge import (
40 ForgeError,
41 Job,
42 Repository,
43 Run,
44 Unsupported,
45)
46from livery.workshop._ci_context import state_store
47from livery.workshop._extensions import workspace_root
48from livery.workshop._points import TRACE
49from livery.workshop._verdict import (
50 EXIT_CI_FAILED,
51 EXIT_PENDING,
52 EXIT_TIMEOUT,
53 EXIT_UNREACHABLE,
54 FollowTimedOut,
55 ForgeUnreachable,
56 JobWatch,
57 Pending,
58 classify,
59 follow,
60 follow_pull_request,
61)
62
63ci = group("ci", help="The head commit's CI runs")
64
65
66@pre_tasks
67def remember_mounted_tasks(inv: footman.Invocation) -> None:
68 """Keep this run's merged task tree for the declared points' task check.
69
70 A ``[[ci.point]]`` names a task some extension mounts; the check reads
71 the tree discovery merged for this invocation, which the
72 module-level registry no longer holds by the time a task runs.
73 """
74 from livery.workshop import _points(empty)
75
76 _points.MOUNTED = inv.tasks(empty)
77
78
79def _resolved() -> tuple[Repository, Path]:
80 root = workspace_root()
81 if root is None:81 ↛ 82line 81 didn't jump to line 82 because the condition on line 81 was never true
82 fail("no workspace: no workshop.toml above the working directory")
83 from livery.workshop._forge_lane import this_repository
84
85 return this_repository(root), root
86
87
88def status_flow(repo: Repository, root: Path) -> int:
89 """Print where the branch stands; return that state's exit code."""
90 verdict = classify(repo, git.current_branch(root), root)
91 print(f" {verdict.state}: {verdict.detail}")
92 return verdict.exit_code
93
94
95@task
96def status(
97 watch: Annotated[bool, doc("follow until it lands or needs a person")] = False,
98 workflow: Annotated[bool, doc("the reserved workflows instead")] = False,
99 interval: Annotated[int, doc("watch poll seconds")] = 15,
100 timeout: Annotated[int, doc("watch deadline seconds")] = 1800,
101) -> None:
102 """Say where the branch's pull request stands; exit that state's code.
103
104 Exit 0 covers merged, in flight, and no pull request; each blocker
105 state keeps its own stable code (see livery.workshop._verdict), and
106 the printed sentence always says what to do next. ``--watch``
107 follows instead of reading once: the same classification, polled
108 until the branch lands or a person is needed. ``--workflow``
109 reads the reserved workflows instead, one line each with its
110 state, author, and, for a mid-publish release, the members whose
111 receipt tag is already cut.
112 """
113 repo, root = _resolved()
114 if workflow:114 ↛ 115line 114 didn't jump to line 115 because the condition on line 114 was never true
115 from livery.workshop._workflow_state import workflow_states
116 from livery.workshop._workflow_tasks import render_workflows
117
118 render_workflows(workflow_states(repo, root))
119 return
120 if watch:120 ↛ 121line 120 didn't jump to line 121 because the condition on line 120 was never true
121 follow_pull_request(
122 repo, git.current_branch(root), root, interval=interval, timeout=timeout
123 )
124 return
125 code = status_flow(repo, root)
126 if code:126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true
127 raise SystemExit(code)
128
129
130def _head_sha(repo: Repository, root: Path) -> str:
131 """The commit whose runs matter: the PR's head, else the local HEAD.
132
133 After a push the two agree; the pull request's answer also covers
134 a checkout that has moved on locally since the push.
135 """
136 pr = repo.pr.find_by_head(git.current_branch(root))
137 return pr.head_sha if pr is not None else git.head_sha(root)
138
139
140def rerun_flow(repo: Repository, root: Path, *, failed_only: bool = True) -> None:
141 """Re-run the failed runs that hold this branch's verdict.
142
143 The verdict is each workflow's newest run on a commit this branch
144 contains. The head commit's runs are the common case, but a
145 workflow triggered on an earlier commit (a merged release squash,
146 a path-filtered job) keeps its verdict until it re-runs, so
147 scoping to the head sha alone would miss it. With *failed_only*
148 the failed jobs alone rerun, except where the contract splits a
149 check leg into shards ([livery.workshop._points.sharded_legs][]):
150 a rerun of one shard partitions afresh and the gate job's fold
151 would read one platform from two attempts, so the whole run reruns
152 and one line says why. Each rerun is read
153 back: a forge that quietly ignores the request is named, never
154 reported as re-running. Off main, a run on a commit main already
155 contains is main's verdict (a scheduled nightly, a merge-point
156 run): it is named and left for a re-run from main.
157 """
158 sha = _head_sha(repo, root)
159 on_main = git.current_branch(root) == "main"
160 newest: dict[str, Run] = {}
161 # The newest hundred, asked of the forge so a repository with a
162 # long run history stays cheap. A run on a sha this clone does not
163 # contain belongs to another branch and is skipped.
164 for run in repo.checks.runs(limit=100):
165 if run.workflow in newest:
166 continue
167 if run.head_sha == sha or git.is_ancestor(root, run.head_sha, "HEAD"):
168 newest[run.workflow] = run
169 failed = [
170 run
171 for run in newest.values()
172 if run.status == "completed" and run.conclusion != "success"
173 ]
174 from livery.workshop._points import BUILTIN, workflow_of
175
176 release_workflow = workflow_of("release")
177 mains = [
178 run
179 for run in failed
180 if not on_main
181 and run.head_sha != sha
182 and git.is_ancestor(root, run.head_sha, "origin/main")
183 and (run.workflow or "").rsplit("/", 1)[-1] != release_workflow
184 ]
185 for run in mains:
186 print(
187 f" {run.workflow} (run {run.id}) failed on {run.head_sha[:10]}, which"
188 " main already has: it is main's verdict, so re-run it from main"
189 )
190 failed = [run for run in failed if run not in mains]
191 if not failed and mains:191 ↛ 192line 191 didn't jump to line 192 because the condition on line 191 was never true
192 return
193 if not failed:
194 running = [run for run in newest.values() if run.status != "completed"]
195 for run in running:
196 red = [
197 job.name
198 for job in repo.checks.jobs(run.id)
199 if job.status == "completed" and job.conclusion not in _GREEN
200 ]
201 so_far = f" ({', '.join(red)} red so far)" if red else ""
202 print(
203 f" {run.workflow} (run {run.id}) is still in progress{so_far}: a"
204 f" re-run needs its verdict first, `{footman.prog()} ci.status --wait`"
205 )
206 if not running:
207 print(f" nothing failed on this branch as of {sha[:10]}")
208 return
209 verdict_jobs = {entry.job for entry in BUILTIN if entry.task == "ci.verdict"}
210 if failed_only:
211 # A rerun of the failed jobs alone keeps the per-run refs the
212 # first attempt's passing shards left, and the shard that reruns
213 # partitions afresh from the record as it then is; the gate
214 # job's fold reads one platform from two attempts and refuses.
215 from livery.workshop._points import sharded_legs
216
217 sharded = sharded_legs(root)
218 if sharded:
219 print(
220 f" rerun: every job, since {len(sharded)} check leg(s) run as"
221 " shards and a rerun of one shard partitions afresh"
222 )
223 failed_only = False
224 for run in failed:
225 workflow = (run.workflow or "").rsplit("/", 1)[-1]
226 # A release wave is the train's: its recovery re-dispatches
227 # at the squash with the right driver, which a rerun of the
228 # same run cannot do. On main it is this branch's verdict.
229 if workflow == release_workflow and not on_main:
230 print(
231 f" {workflow} (run {run.id}) is a release wave, left to the"
232 f" train: `{footman.prog()} workflow.release <set>` re-dispatches it"
233 )
234 continue
235 whole = not failed_only
236 if failed_only:
237 # The verdict job judges the legs' rows, which the first
238 # attempt's collect dropped; re-run alone it can only fail.
239 red = [
240 job.name
241 for job in repo.checks.jobs(run.id)
242 if job.status == "completed" and job.conclusion not in _GREEN
243 ]
244 if red and all(job in verdict_jobs for job in red):
245 whole = True
246 print(
247 f" {workflow} (run {run.id}): only the verdict job failed,"
248 " and it judges the legs' rows; the whole run is asked to run again"
249 )
250 repo.checks.rerun(run.id, failed_only=not whole)
251 after = next(
252 (
253 candidate
254 for candidate in repo.checks.runs(head_sha=run.head_sha)
255 if candidate.id == run.id
256 ),
257 None,
258 )
259 if after is not None and after.status == "completed":
260 print(
261 f" {run.workflow} (run {run.id}): the forge did not"
262 f" restart it; still {after.conclusion}"
263 )
264 else:
265 print(f" re-running {run.workflow} (run {run.id})")
266
267
268@ci.task(name="rerun")
269def ci_rerun(
270 failed_only: Annotated[bool, doc("re-run only the failed jobs")] = True,
271) -> None:
272 """Re-run the head commit's failed runs.
273
274 The failed jobs alone by default; the whole run under
275 ``--no-failed-only``, and whenever the contract splits a check leg
276 into shards, since a rerun of one shard partitions the items
277 afresh and the gate job's fold refuses shards of two attempts.
278 """
279 repo, root = _resolved()
280 rerun_flow(repo, root, failed_only=failed_only)
281
282
283def cancel_flow(repo: Repository, root: Path, *, force: bool = False) -> None:
284 """Cancel the head commit's unfinished runs.
285
286 The relief for a wedged queue; ``force`` reaches the runs a plain
287 cancel cannot, where the forge grants the capability.
288 """
289 sha = _head_sha(repo, root)
290 cancelled = 0
291 for run in repo.checks.runs(head_sha=sha):
292 if run.status != "completed":292 ↛ 291line 292 didn't jump to line 291 because the condition on line 292 was always true
293 repo.checks.cancel_run(run.id, force=force)
294 print(f" cancelled {run.workflow} (run {run.id})")
295 cancelled += 1
296 if not cancelled:
297 print(f" nothing running for {sha[:10]}")
298
299
300@ci.task(name="cancel")
301def ci_cancel(
302 force: Annotated[bool, doc("force-cancel a wedged run")] = False,
303) -> None:
304 """Cancel the head commit's unfinished runs."""
305 repo, root = _resolved()
306 cancel_flow(repo, root, force=force)
307
308
309#: The run conclusions that are not red.
310_GREEN = ("success", "skipped", "neutral")
311
312
313def cancel_superseded_runs(repo: Repository, sha: str) -> list[str]:
314 """Cancel the runs still moving for *sha*, a commit a newer push has superseded.
315
316 A push that moves a branch's head leaves the old head's runs
317 running with nothing left to read them, and on a shared runner
318 they delay the new head's. The cancel is best effort: a run the
319 forge refuses to cancel (already terminal, cancelling) is named
320 and left. Returns one line per run touched.
321 """
322 lines: list[str] = []
323 try:
324 runs = repo.checks.runs(head_sha=sha)
325 except ForgeError as error:
326 return [f" superseded runs for {sha[:12]}: not read ({error})"]
327 for run in runs:
328 if run.status == "completed":
329 continue
330 try:
331 repo.checks.cancel_run(run.id)
332 except ForgeError as error:
333 lines.append(
334 f" superseded run {run.id} for {sha[:12]}: not cancelled ({error})"
335 )
336 continue
337 lines.append(f" superseded run {run.id} for {sha[:12]}: cancelled")
338 return lines
339
340
341#: How many of a workflow's newest runs a follow or a dispatch reads. A
342#: run that started since the last read is among them, however long the
343#: history behind them is.
344RECENT_RUNS = 20
345
346
347def point_runs(
348 repo: Repository,
349 point: str,
350 root: Path | None = None,
351 *,
352 limit: int = RECENT_RUNS,
353) -> tuple[Run, ...]:
354 """The newest *limit* runs of *point*'s workflow, whatever commit they checked.
355
356 Newest first. Asked of the workflow's own listing for each of the
357 point's events, so a forge with a long history answers with one
358 page. The workflow's file name is compared here as well (GitHub
359 lists the path, Gitea the file; the basename is the comparison). A
360 forge that names no workflow on a run (GitLab) answers by event
361 alone.
362 """
363 from livery.workshop._points import events_of, workflow_of
364
365 root = root if root is not None else workspace_root()
366 workflow = workflow_of(point, root)
367 found: dict[int, Run] = {}
368 for event in events_of(root, point):
369 for run in repo.checks.runs(event=event, workflow=workflow, limit=limit):
370 if not run.workflow or run.workflow.rsplit("/", 1)[-1] == workflow:370 ↛ 369line 370 didn't jump to line 369 because the condition on line 370 was always true
371 found[run.id] = run
372 newest = sorted(found.values(), key=lambda run: run.id, reverse=True)
373 return tuple(newest[:limit] if limit else newest)
374
375
376def _run_line(run: Run) -> str:
377 """One run: workflow, conclusion or status, event, commit, page."""
378 return (
379 f" {run.workflow:14} {run.conclusion or run.status:12}"
380 f" {run.event:17} {run.head_sha[:10]} {run.url}"
381 )
382
383
384def follow_run(
385 repo: Repository,
386 point: str,
387 run: Run,
388 *,
389 interval: float = 15,
390 timeout: float = 1800,
391) -> int:
392 """Follow *run* of *point* to its verdict; return that state's exit.
393
394 Progress prints on state changes and every minute. Green is 0;
395 red is 13 with the red jobs named; a wait that runs out is 14; a
396 forge unreachable past the transient budget is 15.
397 """
398
399 def read() -> Run | Pending:
400 current = next((r for r in point_runs(repo, point) if r.id == run.id), run)
401 if current.status == "completed":
402 return current
403 return Pending(
404 f" waiting: {point} run {current.id} is {current.status}",
405 key=current.status,
406 )
407
408 try:
409 current = follow(
410 repo,
411 read,
412 subject=f"{point} run {run.id} {run.url}",
413 interval=interval,
414 timeout=timeout,
415 )
416 except ForgeUnreachable as exc:
417 print(exc.line)
418 return EXIT_UNREACHABLE
419 except FollowTimedOut as exc:
420 print(f" still {exc.last.state} after {timeout:.0f}s; {run.url}")
421 return EXIT_TIMEOUT
422 word, code = runs_state((current,))
423 print(f" {word}: {point} run {current.id} {current.conclusion} {current.url}")
424 if code:
425 for job in repo.checks.jobs(current.id):
426 if job.conclusion not in _GREEN:426 ↛ 425line 426 didn't jump to line 425 because the condition on line 426 was always true
427 print(f" {job.name}: {job.conclusion or job.status}")
428 return code
429
430
431def _short_ref(ref: str) -> str:
432 """*ref* as a forge names a run's ref: ``refs/heads/main`` is ``main``."""
433 for prefix in ("refs/heads/", "refs/tags/"):
434 if ref.startswith(prefix):434 ↛ 435line 434 didn't jump to line 435 because the condition on line 434 was never true
435 return ref[len(prefix) :]
436 return ref
437
438
439def remote_head(root: Path, ref: str) -> str:
440 """The commit branch *ref* names on origin; empty for a tag or a missing branch.
441
442 The commit a dispatched run checks out, read just before the
443 dispatch. A tag names one commit for good, so only a branch is
444 read. An unreachable origin refuses, naming git's words.
445 """
446 name = ref if ref.startswith("refs/") else f"refs/heads/{ref}"
447 if not name.startswith("refs/heads/"):
448 return ""
449 try:
450 return git.ls_remote(root, name).get(name, "")
451 except git.GitError as error:
452 fail(
453 f"cannot read {ref} on origin, the commit that tells the dispatched"
454 f" run from another: {error}"
455 )
456
457
458def _named(run: Run) -> str:
459 """One run as a refusal names it: id, ref, commit, page."""
460 return f"run {run.id} on {run.ref or '?'} at {run.head_sha[:10]} {run.url}"
461
462
463def own_runs(
464 runs: tuple[Run, ...],
465 *,
466 seen: set[int],
467 started: int | None,
468 ref: str,
469 head: str,
470) -> list[Run]:
471 """The runs among *runs* that may be the one a dispatch on *ref* started.
472
473 *started* is the id the forge's answer named, and settles it.
474 Without one, a run qualifies when it is a dispatched run, is not
475 in *seen* (the listing read just before the dispatch), checked
476 out *ref*, and, when *head* names a commit, checked out that
477 commit. A run of another ref never qualifies, however new.
478 """
479 if started is not None:
480 return [run for run in runs if run.id == started]
481 return [
482 run
483 for run in runs
484 if run.id not in seen
485 and run.event == "workflow_dispatch"
486 and _short_ref(run.ref) == _short_ref(ref)
487 and (not head or run.head_sha == head)
488 ]
489
490
491def dispatch_flow(
492 repo: Repository,
493 *,
494 point: str,
495 ref: str,
496 head: str = "",
497 follow: bool = True,
498 interval: float = 15,
499 timeout: float = 1800,
500 register_timeout: float = 60,
501 only: str = "",
502) -> int:
503 """Start *point* on *ref* by hand; follow it to its verdict; return the exit.
504
505 Refuses a point without a dispatch entry, naming what starts it
506 instead. *only* names the scheduled tasks the run takes,
507 comma-separated, for a point that takes the input: a rehearsal
508 names the loop alone. A name the point does not run, or *only* on
509 a point without the input, refuses before anything is dispatched.
510
511 The dispatch is confirmed by its own run: the one the forge's
512 answer names, or else the one dispatched run of *ref* that was
513 not listed before the dispatch and, when *head* is given, checked
514 out that commit, the one *ref* named just before. A run of another
515 ref is never followed. Two or more such runs refuse, naming each,
516 since the listing cannot tell which one this dispatch started.
517 None within *register_timeout* is reported with the verb that
518 reads the point later and the dispatched runs of other refs or
519 commits that did appear, exit 14. Without *follow* the run is
520 named and the exit is 18 while it moves.
521 """
522 from livery.workshop._points import (
523 ONLY,
524 asked_tasks,
525 dispatchable,
526 point_by_name,
527 workflow_of,
528 )
529
530 root = workspace_root()
531 names = point_by_name(root)
532 if point not in names:
533 fail(f"{point!r} is not a point; the points are {', '.join(names)}")
534 starts = dispatchable(root)
535 if point not in starts:
536 if point == "release":
537 fail(
538 "the release point has no dispatch entry of its own: the merge"
539 f" point dispatches the wave, and `{footman_prog()}"
540 " workflow.release.dispatch` does by hand"
541 )
542 fail(
543 f"the {point} point has no dispatch entry: it runs on a push to"
544 f" main; `{footman_prog()} ci.dispatch` starts"
545 f" {', '.join(starts)}"
546 )
547 inputs: dict[str, str] = {}
548 if only:
549 if ONLY not in {item.name for item in names[point].inputs}:
550 takes = [
551 name
552 for name, found in names.items()
553 if ONLY in {i.name for i in found.inputs}
554 ]
555 fail(
556 f"the {point} point takes no --only; {', '.join(takes) or 'no point'}"
557 " does"
558 )
559 if root is None:559 ↛ 560line 559 didn't jump to line 560 because the condition on line 559 was never true
560 fail("--only reads the workspace's schedule: run it inside a workspace")
561 asked = asked_tasks(root, point, only)
562 inputs[ONLY] = ",".join(asked)
563 workflow = workflow_of(point, root)
564 seen = {run.id for run in point_runs(repo, point)}
565 try:
566 started = repo.checks.dispatch(workflow, ref=ref, inputs=inputs or None)
567 except Unsupported as exc:
568 fail(f"this forge cannot dispatch a workflow: {exc}")
569 except ForgeError as exc:
570 fail(f"the forge refused the {point} dispatch on {ref}: {exc}")
571 running = f", running {inputs[ONLY]} alone" if inputs.get(ONLY) else ""
572 at = f" at {head[:10]}" if head else ""
573 print(f" dispatched the {point} point on {ref}{at} ({workflow}){running}")
574 last_runs: tuple[Run, ...] = ()
575
576 def registered() -> Run | Pending:
577 nonlocal last_runs
578 last_runs = point_runs(repo, point)
579 own = own_runs(last_runs, seen=seen, started=started, ref=ref, head=head)
580 if len(own) > 1:
581 fail(
582 f"{len(own)} {point} runs started on {ref}{at} after this"
583 " dispatch, and the forge's answer named none of them, so it"
584 f" cannot tell which one is its own: {'; '.join(map(_named, own))}."
585 " Read them on the forge, or dispatch again once no other"
586 f" {point} dispatch on {ref} is starting"
587 )
588 return own[0] if own else Pending()
589
590 # `follow` is this verb's flag; the helper comes in under its own name.
591 from livery.workshop._verdict import follow as follow_until
592
593 try:
594 run = follow_until(
595 repo,
596 registered,
597 subject=f"the {point} run of {ref}{at}",
598 interval=interval,
599 timeout=register_timeout,
600 quiet=None,
601 )
602 except ForgeUnreachable as exc:
603 fail(exc.line.strip())
604 except FollowTimedOut:
605 print(
606 f" the dispatch was accepted but no {point} run of {ref}{at}"
607 f" appeared within {register_timeout:.0f}s; `{footman_prog()}"
608 f" ci.status --point={point}` reads one that appears later"
609 )
610 others = [
611 run
612 for run in last_runs
613 if run.id not in seen and run.event == "workflow_dispatch"
614 ]
615 if others:
616 print(f" not its own: {'; '.join(map(_named, others))}")
617 return EXIT_TIMEOUT
618 print(f" {point}: run {run.id} {run.url}")
619 if not follow:619 ↛ 621line 619 didn't jump to line 621 because the condition on line 619 was always true
620 return runs_state((run,))[1]
621 return follow_run(repo, point, run, interval=interval, timeout=timeout)
622
623
624def footman_prog() -> str:
625 """The runner's name, for a remedy in a message."""
626 import livery.footman as footman
627
628 return footman.prog()
629
630
631@ci.task(name="dispatch")
632def ci_dispatch(
633 point: Annotated[
634 str, doc("the point to start: gate, nightly, or one a package contributes")
635 ] = "nightly",
636 ref: Annotated[str, doc("the branch or tag the run checks out")] = "main",
637 follow: Annotated[bool, doc("wait for the run's verdict")] = True,
638 interval: Annotated[int, doc("poll seconds")] = 15,
639 timeout: Annotated[int, doc("deadline seconds for the verdict")] = 1800,
640 only: Annotated[
641 str,
642 doc(
643 "the scheduled tasks the run takes, comma-separated, for the"
644 " nightly: ci.e2e rehearses the loop alone; empty runs everything"
645 ),
646 ] = "",
647) -> None:
648 """Start a point on the forge by hand and follow it to its verdict.
649
650 The gate and the nightly carry a dispatch entry. A dispatched
651 gate pays the full gate on *ref*, since the check verb narrows on
652 a pull request alone, and its stamp writes a full row for the
653 tree it proved; the nightly's run waits for nobody, and this verb
654 starts the same run now, so a failure only the nightly meets (a
655 floor Python, a release's verification) is reproduced on demand.
656 The run followed is the dispatch's own: the one the forge names,
657 or the one dispatched run of *ref*, at the commit origin names for
658 it, that appeared after the dispatch. Two such runs refuse, naming
659 both. Exits green 0, red 13 with the red jobs named, a run that
660 never registered or a wait that runs out 14, a forge unreachable
661 for five polls in a row 15, and, with ``--no-follow``, 18 while
662 the run moves. The merge point runs on a push alone, and the
663 release wave is the merge point's to dispatch.
664 """
665 repo, root = _resolved()
666 code = dispatch_flow(
667 repo,
668 point=point,
669 ref=ref,
670 head=remote_head(root, ref),
671 follow=follow,
672 interval=interval,
673 timeout=timeout,
674 only=only,
675 )
676 if code:
677 raise SystemExit(code)
678
679
680def runs_state(runs: tuple[Run, ...]) -> tuple[str, int]:
681 """The one word for *runs* together and its exit: green 0, pending 18, red 13.
682
683 No run at all is pending: the forge may not have registered the
684 push yet, and a script waiting on it keeps waiting. A run still
685 moving is pending whatever the others concluded. Red is any run
686 that completed with a conclusion that is not green.
687 """
688 if not runs or any(run.status != "completed" for run in runs):
689 return "pending", EXIT_PENDING
690 if any(run.conclusion not in _GREEN for run in runs):
691 return "red", EXIT_CI_FAILED
692 return "green", 0
693
694
695def runs_status_flow(
696 repo: Repository,
697 root: Path,
698 *,
699 wait: bool = False,
700 interval: float = 15,
701 timeout: float = 1800,
702 point: str = "",
703) -> int:
704 """Print the head commit's runs and their state; return that state's exit.
705
706 One line per run, workflow, status or conclusion, event, commit,
707 and page, then the word for all of them. With *point* the one run
708 read is the newest of that point's workflow, whatever commit it
709 checked: the nightly's verdict, which no commit of a branch
710 carries. With *wait* the read repeats every *interval* seconds
711 while the state is pending, until *timeout*, which exits 14; a
712 transport error under *wait* is a retry, and a run of them that
713 reaches the budget exits 15. Without *wait* an unreadable forge
714 is named in one line and exits 15 at once.
715 """
716 sha = ""
717 if point:
718 subject = f"the {point} point"
719 nothing = f"no {point} runs yet"
720 else:
721 sha = _head_sha(repo, root)
722 subject = sha[:10]
723 nothing = f"no runs yet for {sha[:10]}"
724 seen_runs: dict[int, str] = {}
725 jobs = JobWatch()
726
727 def read() -> tuple[str, int, int]:
728 if point:
729 runs = point_runs(repo, point, limit=1)
730 else:
731 runs = repo.checks.runs(head_sha=sha)
732 # A line per run when it moves, and a line per job likewise;
733 # a poll that changes nothing prints nothing.
734 for run in runs:
735 if seen_runs.get(run.id) != (run.conclusion or run.status):735 ↛ 734line 735 didn't jump to line 734 because the condition on line 735 was always true
736 seen_runs[run.id] = run.conclusion or run.status
737 print(jobs.stamp() + _run_line(run).lstrip())
738 if not point:
739 jobs.report(repo, sha)
740 word, code = runs_state(runs)
741 if not runs:
742 print(f" {nothing}")
743 return word, code, len(runs)
744
745 def settled() -> tuple[str, int, int] | Pending:
746 answer = read()
747 return Pending() if answer[1] == EXIT_PENDING else answer
748
749 if wait:
750 try:
751 word, code, count = follow(
752 repo,
753 settled,
754 subject=f"the runs for {subject}",
755 interval=interval,
756 timeout=timeout,
757 quiet=None,
758 )
759 except ForgeUnreachable as exc:
760 print(exc.line)
761 return EXIT_UNREACHABLE
762 except FollowTimedOut:
763 print(f" still pending after {timeout:.0f}s")
764 return EXIT_TIMEOUT
765 else:
766 # One read, no retry: an unreadable forge is named in one line
767 # with the retrying form, never a traceback.
768 try:
769 word, code, count = read()
770 except ForgeError as exc:
771 print(
772 f" the forge answered {exc} while reading the runs for {subject};"
773 f" run again, or `{footman_prog()} ci.status --wait` to retry"
774 )
775 return EXIT_UNREACHABLE
776 print(f" {word}: {count} run(s) for {subject}")
777 return code
778
779
780@ci.task(name="status")
781def ci_status(
782 wait: Annotated[bool, doc("poll until the runs are no longer pending")] = False,
783 interval: Annotated[int, doc("poll seconds under --wait")] = 15,
784 timeout: Annotated[int, doc("deadline seconds under --wait")] = 1800,
785 point: Annotated[str, doc("read the newest run of this point instead")] = "",
786) -> None:
787 """Say where the head commit's runs stand; exit that state's code.
788
789 Green exits 0, red 13, pending 18, a wait that runs out 14, and
790 a forge unreachable for five polls in a row under the wait 15,
791 so a script can ask whether main's push has finished after a
792 merge. The head is the pull request's when the branch has one,
793 else the local HEAD, so on main after a pull it is the merge.
794 ``--point=nightly`` reads the newest nightly run instead, by
795 workflow rather than by commit, so a failure only the nightly
796 meets reaches a person here.
797 """
798 repo, root = _resolved()
799 code = runs_status_flow(
800 repo, root, wait=wait, interval=interval, timeout=timeout, point=point
801 )
802 if code:
803 raise SystemExit(code)
804
805
806#: The marks of a line that names a failure: pytest's short summary,
807#: the runner's error annotations, and a verb's own refusal line.
808_FAILURE_MARKS = ("FAILED ", "##[error]", "fm: ")
809
810
811def failure_lines(log: str, *, limit: int = 40) -> list[str]:
812 """The lines of *log* that name a failure, the first *limit* of them.
813
814 A long leg buries pytest's ``FAILED`` summary hundreds of lines
815 above its end, behind the coverage table, so a tail alone shows
816 the exit code and not the test.
817 """
818 found = [
819 line
820 for line in log.splitlines()
821 if any(mark in line for mark in _FAILURE_MARKS)
822 ]
823 return found[:limit]
824
825
826def logs_flow(
827 repo: Repository,
828 root: Path,
829 *,
830 lines: int = 80,
831 failed_only: bool = True,
832 point: str = "",
833) -> None:
834 """Print the head commit's job logs, failed jobs first and by default.
835
836 The tail of each log, newest run first, so a red branch explains
837 itself without leaving the terminal; the log's failure lines
838 (livery.workshop._ci_tasks.failure_lines) come first when the tail
839 does not reach them. With *point* the one run read is the newest
840 of that point's workflow.
841 """
842 if point:
843 runs = point_runs(repo, point, limit=1)
844 subject = f"the {point} point"
845 else:
846 sha = _head_sha(repo, root)
847 runs = repo.checks.runs(head_sha=sha)
848 subject = sha[:10]
849 if not runs:
850 print(f" no {point} runs" if point else f" no runs for {subject}")
851 return
852 printed = 0
853 for run in runs:
854 for job in repo.checks.jobs(run.id):
855 failed = job.conclusion not in ("", "success", "skipped")
856 if failed_only and not failed:856 ↛ 857line 856 didn't jump to line 857 because the condition on line 856 was never true
857 continue
858 state = job.conclusion or job.status
859 print(f" {run.workflow} / {job.name}: {state}")
860 try:
861 log = repo.checks.job_log(job.id)
862 except ForgeError as exc:
863 # A running job's log is not stored yet; github answers
864 # 404 from its blob store until the job completes.
865 print(f" log not available yet ({exc.status or 'error'})")
866 continue
867 tail = log.splitlines()[-lines:]
868 named = [line for line in failure_lines(log) if line not in tail]
869 if named:
870 print(" failures named above the tail:")
871 for line in named:
872 print(f" {line}")
873 print(" tail:")
874 for line in tail:
875 print(f" {line}")
876 printed += 1
877 if not printed:877 ↛ 878line 877 didn't jump to line 878 because the condition on line 877 was never true
878 which = "failed " if failed_only else ""
879 print(f" no {which}jobs for {subject}")
880
881
882@ci.task(name="logs")
883def ci_logs(
884 lines: Annotated[int, doc("log lines per job, from the tail")] = 80,
885 failed_only: Annotated[bool, doc("only jobs that did not succeed")] = True,
886 point: Annotated[str, doc("read the newest run of this point instead")] = "",
887) -> None:
888 """Print the head commit's job logs, failed jobs by default.
889
890 ``--point=nightly`` prints the newest nightly run's logs instead.
891 """
892 repo, root = _resolved()
893 logs_flow(repo, root, lines=lines, failed_only=failed_only, point=point)
894
895
896@ci.task(name="run")
897def ci_run(
898 *,
899 point: Annotated[str, doc("the point: gate, merge, nightly, or release")],
900 job: Annotated[str, doc("the point's job this runner executes")],
901 os: Annotated[str, doc("the matrix runner label, on a matrix job")] = "",
902 python: Annotated[str, doc("the matrix python, on a matrix job")] = "",
903 shard: Annotated[
904 str, doc("the leg's shard, <i>of<n>, on a check leg the contract splits")
905 ] = "",
906) -> None:
907 """Run one job of a point: every task scheduled there, in order.
908
909 The one verb the emitted workflows call, once per job. What the
910 job does is data: the workshop's builtin schedule plus the
911 contract's ``[[ci.schedule]]`` entries. A push to main promotes
912 the gate to the merge point, so the shell spells no decision. On
913 a check leg the contract splits, ``--shard=2of4`` names the shard
914 this job runs as: it ends the leg's label and the job's display
915 name, and the walk runs that shard's share; without it the job
916 runs as one shard of one. A person may run a job by hand; the
917 tasks that only CI may do refuse or fail open on their own.
918 """
919 from livery.workshop._points import run_point
920
921 root = workspace_root()
922 if root is None:922 ↛ 923line 922 didn't jump to line 923 because the condition on line 922 was never true
923 fail("no workspace: no workshop.toml above the working directory")
924 try:
925 run_point(root, point, job, os_label=os, python=python, shard=shard)
926 finally:
927 note = sweep_tool_store()
928 if note:928 ↛ 929line 928 didn't jump to line 929 because the condition on line 928 was never true
929 print(f" {note}")
930
931
932def sweep_tool_store(environ: dict[str, str] | None = None) -> str:
933 """Sweep the tool store on a GitHub job, before the post-job save; the note.
934
935 A tool's archive stays in the store as an object once its tree is
936 extracted and named, and nothing reaches it again: the tree is what
937 a pinned version runs from. The workflow saves the whole store,
938 so those objects would ride in every archive and cost every restore.
939 Off a GitHub job nothing runs and the note is empty; on Gitea's
940 runner, which sets the same variable, shares one store across its
941 jobs and saves none, the store is kept whole and the note says so.
942 A failed sweep is named and never decides the job.
943 """
944 import os as _os
945
946 found = _os.environ if environ is None else environ
947 if not found.get("GITHUB_ACTIONS"):
948 return ""
949 if found.get("GITEA_ACTIONS"):
950 # Gitea's runner sets GITHUB_ACTIONS for compatibility, shares
951 # one store across its jobs and saves no archive, so a sweep
952 # there only makes the next job fetch the catalogue again.
953 return (
954 "tool store: kept whole; this runner shares it across jobs and saves none"
955 )
956 from livery.strongroom import LockTimeout, StoreError
957 from livery.workshop._tools import _home # pyright: ignore[reportPrivateUsage]
958
959 try:
960 report = _home().open_store().sweep()
961 except (LockTimeout, StoreError, OSError) as error:
962 return f"tool store: sweep failed ({error}); the archive saves unswept"
963 return (
964 f"tool store: {len(report.removed)} unreached object(s) swept before the save"
965 )
966
967
968def _conclusion_words(repo: Repository, run: Run, job: Job) -> str:
969 """A red job's conclusion, with the run that superseded a cancelled one named."""
970 words = job.conclusion or job.status
971 if job.conclusion != "cancelled":971 ↛ 972line 971 didn't jump to line 972 because the condition on line 971 was never true
972 return words
973 from livery.workshop._runs import successor
974
975 found = successor(repo, run)
976 if found is None:976 ↛ 977line 976 didn't jump to line 977 because the condition on line 976 was never true
977 return "cancelled; no newer run for this head or its pull request is known"
978 where = (
979 "for the same head"
980 if found.same_head
981 else f"for the moved head {found.run.head_sha[:12]}"
982 )
983 return f"cancelled, superseded by run {found.run.id} {where}"
984
985
986def verdict_flow(repo: Repository, *, needs: tuple[str, ...], job: str) -> list[str]:
987 """The red jobs of this run among *needs*, and any other red job; empty is green.
988
989 A needed job is matched by name or, for a matrix job, by its
990 display name's prefix (``check (ubuntu-latest, 3.14)`` answers to
991 ``check``). A needed job the forge does not list at all counts as
992 red: the verdict never passes on a job it could not see.
993 """
994 from livery.workshop._ci_context import run_context
995
996 run = run_context()
997 if run is None:
998 return ["not a CI run: the verdict reads the run's jobs from the forge"]
999 runs = repo.checks.runs(head_sha=run.head_sha) if run.head_sha else ()
1000 forge_run = next((r for r in runs if str(r.id) == run.run_id), None)
1001 if forge_run is None:
1002 return [f"the forge lists no run {run.run_id} for {run.head_sha[:12]}"]
1003 jobs = repo.checks.jobs(forge_run.id)
1004 red: list[str] = []
1005 for needed in needs:
1006 matching = [
1007 j for j in jobs if j.name == needed or j.name.startswith(f"{needed} (")
1008 ]
1009 if not matching:
1010 red.append(f"{needed}: not among the run's jobs")
1011 for found in matching:
1012 if found.status != "completed" or found.conclusion != "success":
1013 red.append(f"{found.name}: {_conclusion_words(repo, forge_run, found)}")
1014 for found in jobs:
1015 if found.name == job or any(line.startswith(found.name) for line in red):
1016 continue
1017 if found.status == "completed" and found.conclusion not in (1017 ↛ 1021line 1017 didn't jump to line 1021 because the condition on line 1017 was never true
1018 "success",
1019 "skipped",
1020 ):
1021 red.append(f"{found.name}: {found.conclusion}")
1022 return red
1023
1024
1025@ci.task(name="verdict", hidden=True)
1026def ci_verdict(
1027 *,
1028 needs: Annotated[str, doc("the jobs this verdict requires, comma-separated")] = "",
1029) -> None:
1030 """Judge the run: green when every needed job succeeded, red naming the rest.
1031
1032 Runs in the gate job, the one required context, after the jobs
1033 it needs have completed. It asks the forge for the run's jobs, so
1034 the YAML carries no expression over them; a job it cannot see is
1035 red, never assumed.
1036 """
1037 import os as _os
1038
1039 repo, _ = _resolved()
1040 names = tuple(name.strip() for name in needs.split(",") if name.strip())
1041 red = verdict_flow(repo, needs=names, job=_os.environ.get("GITHUB_JOB", "gate"))
1042 if red:
1043 fail("the run is red:\n " + "\n ".join(red))
1044 print(f" green: {', '.join(names) or 'every job'} succeeded")
1045
1046
1047verified = ci.group("verified", help="The trees a green gate proved", hidden=True)
1048
1049
1050def verified_stamp_flow(root: Path) -> None:
1051 """Stamp the run's tree when every check leg ran the full gate; print why not."""
1052 from livery.git import remote_snapshot
1053 from livery.workshop._ci_context import run_context
1054 from livery.workshop._verified import stamp_from_metrics
1055
1056 run = run_context()
1057 if run is None:
1058 print(" not a CI run: the verified record is written by CI only")
1059 return
1060 with remote_snapshot(state_store(root), fetch=("metrics", "verified")):
1061 print(stamp_from_metrics(root, run, sha=git.head_sha(root)))
1062
1063
1064@verified.task(name="stamp")
1065def ci_verified_stamp() -> None:
1066 """Record this run's tree as proved green, for the runs that share it.
1067
1068 Runs in the gate job after the verdict, so only a green run
1069 reaches it. Every check leg's metrics row must say it ran the
1070 full gate; a narrowed leg leaves the tree unstamped, and a later
1071 run of the same tree pays the gate. Fails open loudly: every
1072 reason is printed and the exit stays 0.
1073 """
1074 _repo, root = _resolved()
1075 verified_stamp_flow(root)
1076
1077
1078def leg_flow(root: Path, *, job: str, trace: Path) -> None:
1079 """Put this leg's row and its spool on its per-run ref; red when the write fails."""
1080 from livery.workshop._ci_context import run_context
1081 from livery.workshop._metrics import put_leg
1082
1083 run = run_context()
1084 if run is None:1084 ↛ 1085line 1084 didn't jump to line 1085 because the condition on line 1084 was never true
1085 print(" not a CI run: a check leg's row is put by CI only")
1086 return
1087 lines, why = put_leg(
1088 root,
1089 run,
1090 job=job,
1091 label=run.leg,
1092 trace=trace if trace.is_absolute() else root / trace,
1093 )
1094 for line in lines:
1095 print(line)
1096 if why:
1097 fail(
1098 f"the leg's row could not be put on its per-run ref ({why}): a leg"
1099 " whose write fails is red, since the gate job would judge it by"
1100 " what it never put"
1101 )
1102
1103
1104@ci.task(name="leg", hidden=True)
1105def ci_leg(
1106 *,
1107 job: Annotated[str, doc("the job's name as the forge lists it")] = "",
1108 trace: Annotated[Path, doc("the trace the gate wrote")] = Path(TRACE),
1109) -> None:
1110 """Put this check leg's row and the files its entries left on its ref, in one write.
1111
1112 The check leg's last entry. The row is the leg's timing half, read
1113 from the trace the profiled gate wrote, with the scope the gate
1114 ran; beside it go the files the leg's other entries left in its
1115 spool, ``.workshop/leg/``, each under its own name. A trace that
1116 cannot be read prints its reason and the write goes on without
1117 the row. A write that fails is red.
1118 """
1119 root = workspace_root()
1120 if root is None:
1121 fail("no workspace: no workshop.toml above the working directory")
1122 leg_flow(root, job=job, trace=trace)
1123
1124
1125def fold_flow(root: Path) -> None:
1126 """Fold each platform's shards on the run's refs into one leg; print every line."""
1127 from livery.git import remote_snapshot
1128 from livery.workshop._ci_context import run_context
1129 from livery.workshop._fold import fold_run
1130
1131 run = run_context()
1132 if run is None:
1133 print(" not a CI run: a platform's shards are folded by CI only")
1134 return
1135 with remote_snapshot(state_store(root), fetch=(f"run/{run.run_id}/",)):
1136 fold_run(root, run)
1137
1138
1139@ci.task(name="fold", hidden=True)
1140def ci_fold() -> None:
1141 """Fold each platform's shards into one check leg on the run's refs.
1142
1143 Runs in the gate job before the judges, the collect, the verdict
1144 and the stamp read the run's refs. A platform whose check leg ran
1145 as several shards has one ref per shard; the fold proves the
1146 shards are one leg's work (one scope, one item set from one
1147 durations row, every item in exactly one shard, every shard the
1148 matrix produces present) and puts one ref under the platform
1149 leg's key, the timing sums added and every other file folded by
1150 the extension that leaves it, then drops the shards' refs. A
1151 platform whose shards disagree, or one of whose shards left no
1152 ref, is red naming the platform, the shards and what differs. A
1153 run with no sharded platform says so and changes nothing.
1154 """
1155 root = workspace_root()
1156 if root is None:
1157 fail("no workspace: no workshop.toml above the working directory")
1158 fold_flow(root)
1159
1160
1161metrics = ci.group("metrics", help="The timing rows CI writes", hidden=True)
1162
1163
1164def metrics_collect_flow(root: Path, repo: Repository) -> None:
1165 """Collect the run's rows into the metrics series; print every line."""
1166 from livery.git import remote_snapshot
1167 from livery.workshop._ci_context import run_context
1168 from livery.workshop._metrics import collect
1169
1170 run = run_context()
1171 if run is None:
1172 print(" not a CI run: the run's timing rows are collected by CI only")
1173 return
1174 with remote_snapshot(state_store(root), fetch=(f"run/{run.run_id}/", "metrics")):
1175 for line in collect(root, repo, run, sha=git.head_sha(root)):
1176 print(line)
1177
1178
1179@metrics.task(name="collect")
1180def ci_metrics_collect() -> None:
1181 """Join the legs' timing rows with the forge's times into the run's file.
1182
1183 Runs in the gate job before its verdict. Reads the per-run refs
1184 the legs wrote, asks the forge for the run's jobs and their
1185 steps, and puts the run's file on the metrics series under its
1186 window. The per-run refs stay for `ci.metrics.drop`, the job's
1187 last entry, so a red verdict leaves them for the forge's re-run
1188 of the job, which runs without its legs. Fails open loudly: every
1189 reason is printed and the exit stays 0.
1190 """
1191 repo, root = _resolved()
1192 metrics_collect_flow(root, repo)
1193
1194
1195def drop_flow(root: Path, repo: Repository, *, point: str, job: str) -> None:
1196 """Drop the run's per-run refs once every half is in; print every line."""
1197 from livery.git import remote_snapshot
1198 from livery.workshop._ci_context import run_context
1199 from livery.workshop._metrics import drop_halves
1200 from livery.workshop._points import half_writers
1201
1202 run = run_context()
1203 if run is None:1203 ↛ 1204line 1203 didn't jump to line 1204 because the condition on line 1203 was never true
1204 print(" not a CI run: the run's per-run refs are dropped by CI only")
1205 return
1206 with remote_snapshot(state_store(root), fetch=(f"run/{run.run_id}/",)):
1207 for line in drop_halves(
1208 root,
1209 repo,
1210 run,
1211 sha=git.head_sha(root),
1212 job=job,
1213 writers=half_writers(root, point),
1214 ):
1215 print(line)
1216
1217
1218@metrics.task(name="drop")
1219def ci_metrics_drop(
1220 *,
1221 point: Annotated[str, doc("the point whose jobs put the run's halves")] = "",
1222 job: Annotated[str, doc("this job's name as the forge lists it")] = "",
1223) -> None:
1224 """Drop the run's per-run refs, once every job that put a half is green.
1225
1226 The last entry of a job that collected the run's row, after its
1227 verdict: this job is green when it runs. The refs stay while
1228 another job that puts a half (a check leg, or a sibling job of a
1229 point with no gate job) has not completed green, or any completed
1230 job is red, so a re-run finds what the legs put and the last of a
1231 point's collecting jobs is the one that drops. Fails open loudly:
1232 every reason is printed and the exit stays 0.
1233 """
1234 repo, root = _resolved()
1235 drop_flow(root, repo, point=point, job=job)
1236
1237
1238def record_flow(root: Path, repo: Repository, *, point: str, job: str) -> None:
1239 """Put this job's half from its box, collect the run's row, drop when last.
1240
1241 Every line is printed and none is a verdict: the row is a record
1242 of what the job did, and a job with no gate job to judge it stays
1243 green whatever its record could not do. The three steps share one
1244 listing of the store, taken here: the put records its ref in it,
1245 so the collect and the drop after it see the half it just put.
1246 """
1247 from livery.footman.profile import as_trace, parent_box, swept
1248 from livery.git import remote_snapshot
1249 from livery.workshop._ci_context import run_context
1250 from livery.workshop._metrics import put_leg
1251 from livery.workshop._points import trace_file
1252
1253 run = run_context()
1254 if run is None:
1255 print(" not a CI run: a job's row is put by CI only")
1256 return
1257 with remote_snapshot(state_store(root), fetch=(f"run/{run.run_id}/", "metrics")):
1258 # The job's box, where the entries before this one left their
1259 # fragments: this process was handed it, and runs profiled
1260 # itself, so the environment names its own, empty, box instead.
1261 box = parent_box()
1262 if box is None:
1263 print(
1264 " no profile box in this job's environment; the job's half stays"
1265 " unwritten"
1266 )
1267 else:
1268 events, origin = swept(box)
1269 trace = root / trace_file("job row")
1270 trace.write_text(as_trace(events, origin=origin), encoding="utf-8")
1271 lines, why = put_leg(root, run, job=job, label=run.leg, trace=trace)
1272 for line in lines:
1273 print(line)
1274 if why:1274 ↛ 1275line 1274 didn't jump to line 1275 because the condition on line 1274 was never true
1275 print(
1276 f" {job}: the half could not be put ({why}); the row goes"
1277 " without it"
1278 )
1279 metrics_collect_flow(root, repo)
1280 drop_flow(root, repo, point=point, job=job)
1281
1282
1283@metrics.task(name="record")
1284def ci_metrics_record(
1285 *,
1286 point: Annotated[str, doc("the point this job belongs to")] = "",
1287 job: Annotated[str, doc("this job's name as the forge lists it")] = "",
1288) -> None:
1289 """Record this job's run: its half, the run's row, the refs dropped when last.
1290
1291 The last entry of every job of a point with no gate job (the
1292 nightly, the release, a point a package contributes). The half is
1293 the union of the traces the job's entries left in its profile
1294 box, put on the job's per-run ref; the collect then writes the
1295 run's row from every half present, so each job's write carries
1296 the jobs before it; the drop takes the refs once every job of the
1297 point has completed green. Fails open loudly: every reason is
1298 printed and the exit stays 0.
1299 """
1300 repo, root = _resolved()
1301 record_flow(root, repo, point=point, job=job)
1302
1303
1304profile = ci.group("profile", help="A run as one timeline")
1305
1306
1307@profile.default
1308def ci_profile(
1309 *,
1310 run: Annotated[str, doc("the run's id; the newest for HEAD by default")] = "",
1311 from_: Annotated[
1312 str, doc("walk every run about this commit, and what its merge caused")
1313 ] = "",
1314 into: Annotated[str, doc("where the file lands; the contract's own by default")] = (
1315 ""
1316 ),
1317) -> None:
1318 """Write one CI run as one timeline, to open at ui.perfetto.dev.
1319
1320 Every job of the run is a track at the forge's own times, its steps
1321 inside it, and each leg that kept a trace brings its own tasks, steps
1322 and tests, laid on the run's clock by the origin each side recorded.
1323 This is what the legs' traces are kept for, and what the window bounds.
1324
1325 ``--from=<commit>`` writes the chain instead: every run about that
1326 commit, and the runs of the commit its merge produced. A release is
1327 two commits and one hop, so the pull request's run, the base's own
1328 run and the wave land in one file. Any commit-ish a checkout can
1329 resolve names it, ``HEAD`` included.
1330
1331 Refuses when no file could be written, naming why: a person asked for
1332 one. A run whose traces have aged out of the window still assembles from
1333 the forge's skeleton alone, and says which legs brought nothing.
1334 """
1335 from livery.workshop._traces import write_chain, write_run
1336
1337 repo, root = _resolved()
1338 where = Path(into) if into else None
1339 if from_:
1340 commit = git.sha_of(root, from_)
1341 if not commit:
1342 fail(f"--from={from_} names no commit this checkout can resolve")
1343 path, lines = write_chain(root, repo, commit, into=where)
1344 for line in lines:
1345 print(f" {line}")
1346 if path is None:
1347 fail(lines[0] if lines else f"nothing about {commit[:12]}")
1348 print(f" profile: {path}")
1349 return
1350 path, lines = write_run(
1351 root,
1352 repo,
1353 run_id=run,
1354 head_sha="" if run else git.head_sha(root),
1355 into=where,
1356 )
1357 for line in lines:
1358 print(f" {line}")
1359 if path is None:
1360 fail(lines[0] if lines else "no run to assemble")
1361 print(f" profile: {path}")
1362
1363
1364def timings_flow(root: Path, *, since: int, base: int) -> None:
1365 """Print the rendered timings, then the speed marks beside the newest run."""
1366 from livery.git import remote_snapshot
1367 from livery.workshop._metrics import render
1368 from livery.workshop._speed import render_marks
1369
1370 with remote_snapshot(state_store(root), fetch=("metrics", "speed/marks")):
1371 for line in render(root, since=since, base=base):
1372 print(line)
1373 for line in render_marks(root):
1374 print(line)
1375
1376
1377@ci.task(name="timings")
1378def ci_timings(
1379 since: Annotated[int, doc("the recent runs the movers judge")] = 1,
1380 base: Annotated[int, doc("the runs before them the movers compare against")] = 20,
1381) -> None:
1382 """Print the CI timings: per job and metric the latest, p50, p90, and the movers.
1383
1384 Reads the metrics series the gate job writes; CI is its only
1385 writer, a local run reads. Per job, every metric's latest value
1386 and its median and ninetieth percentile over the series' window;
1387 a platform that ran as shards is one job, its folded row, with a
1388 line per shard under it giving the shard's wall and queue times
1389 from the newest run. Then the biggest movers: the recent runs'
1390 median against the base runs' median. Then the speed marks, each
1391 package's mark on each leg beside the newest run's time.
1392 """
1393 root = workspace_root()
1394 if root is None:1394 ↛ 1395line 1394 didn't jump to line 1395 because the condition on line 1394 was never true
1395 fail("no workspace: no workshop.toml above the working directory")
1396 timings_flow(root, since=since, base=base)
1397
1398
1399def balance_flow(root: Path, *, cap: int, write: bool) -> None:
1400 """Print the balancer's window line, its tables and its proposal; *write* writes."""
1401 from livery.git import remote_snapshot
1402 from livery.workshop import _balance
1403
1404 with remote_snapshot(state_store(root), fetch=("metrics",)):
1405 balance = _balance.read(root, cap=cap)
1406 for line in balance.lines:
1407 print(line)
1408 if write and _balance.write(root, balance) is not None:
1409 print(f" run `{footman.prog()} sync` to render the workflows with the counts")
1410
1411
1412@ci.task(name="balance")
1413def ci_balance(
1414 max_: Annotated[
1415 int,
1416 doc(
1417 "the largest shard count a platform may take; 0 reads [ci] shards-max,"
1418 " 4 when the contract is silent"
1419 ),
1420 ] = 0,
1421 write: Annotated[
1422 bool, doc("rewrite [ci] shards and [ci] pull-request-shards when due")
1423 ] = False,
1424 submit: Annotated[
1425 bool,
1426 doc(
1427 "write, commit on the day's branch and open its pull request, unarmed"
1428 " (implies --write)"
1429 ),
1430 ] = False,
1431) -> None:
1432 """Predict each check platform's wall at one to several shards; propose the counts.
1433
1434 Reads the metrics series the gate job writes; CI is its only
1435 writer, a local run reads. Per platform, a runner and a Python,
1436 the newest ten rows whose check jobs ran the full gate and
1437 succeeded give the medians of four measures: F, the setup and
1438 teardown the forge timed around the call; O, the summed wall of
1439 the gate steps that are not test steps, which the first shard
1440 alone runs; S, the summed test time over every package; and p,
1441 the parallelism the test step reached, S over its wall. The
1442 shards of a platform fold into one reading. The first shard's
1443 slice is lighter by the tests O holds back, so the wall at n
1444 shards is predicted as F + max(O, (S / p + O) / n) and the runner
1445 seconds as about n F + S / p + O; the table prints
1446 both for every count up to the cap, ``[ci] shards-max`` or
1447 ``--max`` under it.
1448
1449 The proposal: the slowest platform, the largest wall at one
1450 shard, takes the largest count whose every added shard saved at
1451 least F; every other platform takes the smallest count whose
1452 wall is within 15% of the slowest platform's, or the cap, named.
1453 The pull request counts follow the same rule over
1454 ``[ci] pull-request-runners`` when that list is shorter than
1455 ``[ci] runners``. The proposal prints as the TOML ``[ci] shards``
1456 and ``[ci] pull-request-shards`` would hold, with the predicted
1457 wall of the slowest leg beside its wall at one shard. A platform
1458 with fewer than three such rows keeps its count and is named. A
1459 store that cannot be read prints why and proposes nothing.
1460
1461 Without a flag the verb changes nothing. ``--write`` rewrites the
1462 two keys in the root ``workshop.toml`` when a count differs from
1463 the contract's and the predicted wall at the proposed counts
1464 differs from the wall at the contract's counts by more than 10%,
1465 so a count does not move back and forth; every comment and every
1466 other key stay, a count of one is left out of a table, a table
1467 that would be empty is removed, and ``[ci] shards-max`` is read
1468 and never written. It prints what each key becomes, or why nothing
1469 was written: no platform with enough rows, no count changed, or a
1470 wall that moved less. The workflows follow at the next ``fm sync``,
1471 which the last line says. ``--submit`` writes the same change on
1472 the day's branch, ``chore/balance-<date>``, renders the workflows,
1473 commits it as ``chore(ci): the check legs run as <counts>`` and
1474 submits it unarmed, for a person to merge; it runs from origin's
1475 default branch alone and says so otherwise, and a day whose branch
1476 origin already holds at these counts submits nothing and says so.
1477 The nightly runs it once a day. A count can also be set by hand.
1478 """
1479 root = workspace_root()
1480 if root is None:
1481 fail("no workspace: no workshop.toml above the working directory")
1482 if submit:
1483 from livery.workshop import _balance
1484
1485 _balance.submit(root, cap=max_)
1486 return
1487 balance_flow(root, cap=max_, write=write)
1488
1489
1490def _github_public(root: Path, repo: Repository) -> tuple[bool, str]:
1491 """Whether *repo* is a public repository on GitHub, and why that is unknown.
1492
1493 False with no reason on another forge, or when GitHub says the
1494 repository is private. False with the reason when GitHub did not
1495 answer, or knows no such repository: the caller prints it, so a
1496 missing price is never silent on the one forge that prices.
1497 """
1498 from livery.workshop._forge_lane import forge_kind, this_forge
1499
1500 if forge_kind(root) != "github":
1501 return False, ""
1502 try:
1503 info = this_forge(root).get_repo(repo.owner, repo.name)
1504 except ForgeError as error:
1505 return False, (
1506 f"GitHub did not say whether {repo.owner}/{repo.name} is public: {error}"
1507 )
1508 if info is None:
1509 return False, f"GitHub knows no repository {repo.owner}/{repo.name}"
1510 return not info.private, ""
1511
1512
1513def footprint_flow(root: Path, repo: Repository, *, since: int) -> None:
1514 """Print the window's runner-minutes by runner label; price them where GitHub would.
1515
1516 Fails when the window holds no row, naming what writes one.
1517 """
1518 from livery.git import remote_snapshot
1519 from livery.workshop._footprint import default_runner, footprint, price, render
1520 from livery.workshop._metrics import SERIES
1521
1522 if since < 0:
1523 fail(f"--since={since} names no count of runs: 0 reads the whole window")
1524 with remote_snapshot(state_store(root), fetch=("metrics",)):
1525 found = SERIES.rows(state_store(root))
1526 if found.failed:1526 ↛ 1527line 1526 didn't jump to line 1527 because the condition on line 1526 was never true
1527 fail(found.reason)
1528 for skipped in found.skipped:1528 ↛ 1529line 1528 didn't jump to line 1529 because the loop on line 1528 never started
1529 print(f" {skipped}")
1530 entries = [row.data for row in found.rows[: since or None]]
1531 if not entries:
1532 fail(
1533 "the metrics window is empty: the gate job of a run on the forge"
1534 " writes one row per run; nothing to sum"
1535 )
1536 runner = default_runner(root)
1537 summed = footprint(entries, default_runner=runner)
1538 for line in render(summed, default_runner=runner):
1539 print(line)
1540 public, why = _github_public(root, repo)
1541 if why:
1542 print(f" not priced: {why}")
1543 elif public:
1544 for line in price(summed):
1545 print(line)
1546
1547
1548@ci.task(name="footprint")
1549def ci_footprint(
1550 since: Annotated[int, doc("the newest runs read; 0 reads the whole window")] = 0,
1551) -> None:
1552 """Sum the window's runner-minutes by runner label; price them where GitHub would.
1553
1554 Reads the metrics series the gate job writes; CI is its only
1555 writer, a local run reads. Every job of every run in the window,
1556 or of the newest ``--since`` runs, is counted on one runner label:
1557 a check leg on the runner its job name lists, every other job (the
1558 gate job, the docs build, the release jobs) on the first runner
1559 ``[ci] runners`` names, where the workflow places them. Per label
1560 and kind the jobs and the minutes, then the total and the minutes
1561 per 30 days from the window's span. A job without a wall time,
1562 still running when the gate job collected or never started, is
1563 counted apart and priced as nothing.
1564
1565 On GitHub, when the repository is public, the same minutes are
1566 priced at GitHub's rates for the standard runners of a private
1567 repository, by the class the label names (linux, windows, macos),
1568 with the total, the USD per 30 days and the source page: a public
1569 repository is not billed for them, and the price says what the
1570 matrix would cost one that is. On another forge, or on a private
1571 GitHub repository, nothing is priced.
1572
1573 Fails when the window holds no row: the gate job of a run on the
1574 forge writes one.
1575 """
1576 repo, root = _resolved()
1577 footprint_flow(root, repo, since=since)