Coverage for packages/workshop/src/livery/workshop/_e2e.py: 64%

Measured by coverage.py 7.16.2 on CPython 3.14 while the suites of tests, packages/workshop, packages/extensions/ty, packages/extensions/ruff, packages/extensions/python, packages/extensions/pytest, packages/extensions/pyrefly, packages/extensions/nanobind, packages/extensions/mypy, packages/extensions/mkdocstrings, packages/extensions/lfs, packages/extensions/docs, packages/extensions/cpp, packages/extensions/coverage, packages/extensions/conan, packages/extensions/cmake, packages/extensions/claude, packages/extensions/clang-tidy, packages/extensions/clang-format, packages/extensions/changelog, packages/extensions/basedpyright ran.

1"""``fm ci.e2e``: the CI and release story against a local forge.
2
3The workshop's own development substrate: provision a scratch
4repository on the seeded local forge, and drive the emitted
5workflows through a real runner against the forge's own package
6registry, so the whole second world runs consequence-free before
7anything reaches a public forge. The loop's workspace splits its one
8runner's check leg into two shards, so every pass proves the slice,
9the gate job's fold and the durations row beside the gate itself.
10
11The verb registers only where the workshop develops itself, the way
12``fm forge.fixtures.record`` does: a consumer checkout does not
13carry it at all.
14"""
15
16from __future__ import annotations
17
18import contextlib
19import functools
20import os
21import re
22import shutil
23from collections.abc import (
24 Callable,
25 Collection,
26 Generator,
27 Iterable,
28 Mapping,
29 MutableMapping,
30 Sequence,
31)
32from dataclasses import dataclass, field, replace
33from datetime import UTC, datetime
34from pathlib import Path
35from typing import TYPE_CHECKING, TypeVar
36
37import livery.footman as footman
38from livery import git
39from livery.footman import fail
40from livery.forge import ForgeError
41from livery.workshop._ci_context import state_store
42from livery.workshop._ci_tasks import ci
43from livery.workshop._e2e_rows import (
44 RunFacts,
45 durations_row,
46 marks,
47 record_units,
48 run_facts,
49 verified_by,
50 verified_tree,
51)
52from livery.workshop._series import LOOP
53from livery.workshop._verdict import FollowTimedOut, ForgeUnreachable, Pending, follow
54
55if TYPE_CHECKING:
56 from livery.forge import Forge, Job, Registry, Repository, Run
57 from livery.workshop._devenv import Environment
58 from livery.workshop._packages import Package
59
60#: What a runs reader answers.
61_A = TypeVar("_A")
62
63#: The seeded organisation and the loop's one scratch repository.
64E2E_OWNER = "livery"
65E2E_REPO = "ci-e2e-loop"
66
67#: The workshop's own test suite, present only in a source checkout:
68#: this file is src/livery/workshop/_e2e.py, so the package directory
69#: holding tests/ is three parents up.
70_WORKSHOP_TESTS = Path(__file__).resolve().parents[3] / "tests"
71
72#: The member the loop eats the dev wheels of; its dependency closure
73#: rides along, read from the contracts by `dev_members`.
74DEV_MEMBER = "packages/workshop"
75
76#: What a reset of the loop's tree keeps: the directories `fm sync`
77#: materialises, gitignored and read by the gate on the desk.
78KEPT_BY_SYNC = (".venv", "typings", ".workshop")
79
80#: How many shards the loop's workspace splits its one runner's check
81#: leg into: every check leg of a pass runs as this many jobs, which
82#: the gate job folds into one leg before anything reads the run.
83SHARDS = 2
84
85
86@dataclass
87class RunCount:
88 """How many runner runs the pass has followed so far; the timing table reads it."""
89
90 count: int = 0
91
92
93#: The pass-wide count every wait on a run adds to, so a scenario's
94#: row says how many runs it cost, not only how long it took.
95RUNS = RunCount()
96
97
98@dataclass
99class Current:
100 """The environment a pass runs on, as the loop reads its forge from it.
101
102 Attributes:
103 name: The environment's name; empty outside a pass.
104 mode: `host` or `docker`.
105 url: The forge's URL in host mode; empty in docker mode, where
106 the cascade's shared file names it.
107 token: The seeded API token in host mode.
108 label: The runner label the loop's workspace names as its
109 runner and its wheel platform; the hosted name in docker
110 mode, the environment's own label in host mode.
111 """
112
113 name: str = ""
114 mode: str = ""
115 url: str = ""
116 token: str = ""
117 label: str = "ubuntu-latest"
118
119
120#: The environment of the running pass. `ci.e2e` sets it from
121#: `--env` before anything reads the forge; the tests set it by hand.
122CURRENT = Current()
123
124
125#: The variables that make the workshop read a process as a CI run: a
126#: fix is refused inside CI, a tool the store cannot supply refuses, and
127#: a run is stamped as the runner's. The pass drives its project as a
128#: desk does, so its children run without them, on a hosted runner too.
129CI_MARKERS: tuple[str, ...] = ("CI", "GITHUB_ACTIONS", "GITEA_ACTIONS", "GITLAB_CI")
130
131
132def scrub_ci_markers(environ: MutableMapping[str, str]) -> tuple[str, ...]:
133 """Remove CI's markers from *environ*; the names removed, in declared order."""
134 removed = tuple(name for name in CI_MARKERS if name in environ)
135 for name in removed:
136 del environ[name]
137 return removed
138
139
140def child_environment(**extra: str) -> dict[str, str]:
141 """The environment a child the pass starts by hand begins with, *extra* over it.
142
143 The process environment over the task's. The pass is a serial task,
144 so the process environment is the real one, where the pass writes
145 its forge's own token after the task started; the task's carries
146 what the git plugin wrote at the task's start, the tools' identity
147 and the unsigned setting, which the process environment lacks. A
148 spawn given an explicit environment takes neither by itself.
149 """
150 return {**footman.current().env, **os.environ, **extra}
151
152
153def edit_environments(
154 values: Mapping[str, str], *, remove: Sequence[str] = ()
155) -> tuple[str, ...]:
156 """Write *values* and drop *remove* where this task and every child read them.
157
158 The pass is a serial task, so ``os.environ`` in it is the real
159 environment, which a child given an explicit environment built from
160 it inherits; footman's own ``run`` and the store's tool handles
161 spawn from the task's ``ctx.env``, the environment the task started
162 with, so the same edit lands there too. Without the second write a
163 git the pass runs through the store commits without the identity
164 and the unsigned setting, which a hosted runner showed. Returns the
165 names removed from the process environment, in *remove*'s order.
166 """
167 from livery.footman import current
168
169 environments: list[MutableMapping[str, str]] = [os.environ, current().env]
170 removed = tuple(name for name in remove if name in os.environ)
171 for environ in environments:
172 environ.update(values)
173 for name in remove:
174 environ.pop(name, None)
175 return removed
176
177
178def dev_members(root: Path, extensions: Sequence[str] | None = None) -> tuple[str, ...]:
179 """The members whose dev wheels the loop eats: the workshop and what a birth lists.
180
181 Directory names, the workshop first, then each member that ships
182 an extension a birth lists, and each dependency after the first
183 member that names it, once. An edge of every kind counts: the
184 wheel the runner installs resolves all of them from the loop's
185 registry, and a member the closure does not reach publishes
186 nothing there. An extension depends on the workshop, never the
187 other way round, so the workshop's closure alone leaves the
188 newborn's listed extensions nothing to install. *extensions* is
189 the stack a birth lists, the stock list when None.
190
191 Raises:
192 Failed: when *root* has no workshop member to eat.
193 """
194 from livery.workshop._extensions import SELF, distribution_of, listing
195 from livery.workshop._new_project import birth_extensions
196 from livery.workshop._packages import discover_packages
197
198 by_path = {package.path: package for package in discover_packages(root)}
199 workshop = by_path.get(DEV_MEMBER)
200 if workshop is None:
201 fail(f"{root}: no {DEV_MEMBER} member; the loop eats the workshop's dev wheels")
202 listed = birth_extensions([SELF]) if extensions is None else extensions
203 stock = {distribution_of(listing(entry).name) for entry in listed}
204 shipping = sorted(
205 (p for p in by_path.values() if p.name in stock and p is not workshop),
206 key=lambda package: package.path,
207 )
208 ordered: list[str] = []
209 pending = [workshop, *shipping]
210 while pending:
211 package = pending.pop(0)
212 if package.member in ordered:
213 continue
214 ordered.append(package.member)
215 pending.extend(
216 by_path[edge.path] for edge in package.depends if edge.path in by_path
217 )
218 return tuple(ordered)
219
220
221@dataclass(frozen=True)
222class Lane:
223 """One local forge the loop runs against, as the host and the runner reach it.
224
225 Attributes:
226 kind: The forge kind, ``gitea`` or ``gitlab``.
227 alias: The forge as CI sees it, the compose service name and
228 port, true on the host too through its hosts entry.
229 url_var: The shared env key the seed writes the host-side URL
230 to.
231 token_var: The shared env key the seed writes the lane token to.
232 version_path: The API path the hosts-entry probe reads.
233 runner: The compose service of the lane's runner.
234 """
235
236 kind: str
237 alias: str
238 url_var: str
239 token_var: str
240 version_path: str
241 runner: str
242
243 @property
244 def host(self) -> str:
245 """The compose hostname alone, for the hosts entry."""
246 return self.alias.removeprefix("http://").rsplit(":", 1)[0]
247
248 def publish(self, base: str = "") -> str:
249 """The PyPI upload base for the loop's packages at *base*, the alias by default.
250
251 Gitea keeps an owner's registry; GitLab a project's, addressed
252 by its URL-encoded path.
253 """
254 base = base or self.alias
255 if self.kind == "gitlab":
256 from urllib.parse import quote
257
258 project = quote(f"{E2E_OWNER}/{E2E_REPO}", safe="")
259 return f"{base}/api/v4/projects/{project}/packages/pypi"
260 return f"{base}/api/packages/{E2E_OWNER}/pypi"
261
262 def index(self, base: str = "") -> str:
263 """The simple index the loop's packages are read from."""
264 return f"{self.publish(base)}/simple"
265
266
267#: The lanes, by forge kind.
268LANES: dict[str, Lane] = {
269 "gitea": Lane(
270 "gitea",
271 "http://gitea:3000",
272 "GITEA_URL",
273 "GITEA_TOKEN",
274 "/api/v1/version",
275 "act_runner",
276 ),
277 "gitlab": Lane(
278 "gitlab",
279 "http://gitlab:8929",
280 "GITLAB_URL",
281 "GITLAB_TOKEN",
282 "/api/v4/version",
283 "gitlab-runner",
284 ),
285}
286
287#: The compose project the dev containers belong to; a container is
288#: named ``<project>-<service>-1``.
289_DEV_PROJECT = "livery-forge-dev"
290
291
292def _require_runner_docker(kind: str) -> None:
293 """Refuse until the lane's runner has the host's docker socket.
294
295 The loop's native member builds its wheel through cibuildwheel on
296 the runner, which needs a daemon, and the runners get the socket
297 only from ``fm forge.dev.up --with-docker``. A runner that is not
298 up at all is named the same way, so the refusal comes before the
299 minutes a pass spends reaching the release act.
300 """
301 import json
302
303 import livery.toolroom.tools as tools
304
305 if CURRENT.mode == "host":305 ↛ 309line 305 didn't jump to line 309 because the condition on line 305 was never true
306 # The runner is a process of this machine, and cibuildwheel
307 # reaches the machine's own daemon: the one question is
308 # whether it answers.
309 info = tools.docker.opts(nofail=True, recorded=False)("info")
310 if info.code != 0:
311 fail(
312 "the machine's docker daemon does not answer, and the loop's"
313 " native member builds its wheel through cibuildwheel on it:"
314 " start Docker Desktop, or the daemon"
315 )
316 return
317 lane = _lane(kind)
318 container = f"{_DEV_PROJECT}-{lane.runner}-1"
319 result = tools.docker.opts(nofail=True, recorded=False)(
320 "inspect", "--format", "{{json .Mounts}}", container
321 )
322 if result.code != 0:
323 fail(
324 f"the {kind} runner ({container}) is not up:"
325 f" `{footman.prog()} forge.dev.up --profile={kind} --with-docker`"
326 )
327 try:
328 mounts = json.loads(result.stdout or "[]")
329 except ValueError:
330 mounts = []
331 destinations = {
332 str(mount.get("Destination", "")) for mount in mounts if isinstance(mount, dict)
333 }
334 if "/var/run/docker.sock" not in destinations:
335 fail(
336 f"the {kind} runner has no docker socket, and the loop's native"
337 " member builds its wheel through cibuildwheel on it:"
338 f" `{footman.prog()} forge.dev.up --profile={kind} --with-docker`"
339 )
340
341
342def _lane(kind: str) -> Lane:
343 """The lane for *kind*; refuses a kind with no local containers.
344
345 On a host-mode environment the lane's alias is the environment's
346 own URL: the forge and the runner are processes of this machine,
347 so one localhost URL is true on both sides of the loop.
348 """
349 from dataclasses import replace
350
351 lane = LANES.get(kind)
352 if lane is None:
353 fail(
354 f"--forge={kind} is not a local lane: the loop runs against"
355 f" {', '.join(LANES)}"
356 )
357 if CURRENT.mode == "host" and CURRENT.url:
358 return replace(lane, alias=CURRENT.url)
359 return lane
360
361
362def _enter_host(values: Mapping[str, str], env: str) -> None:
363 """Run the pass on the host-mode environment *env*, whose *values* these are.
364
365 The forge's URL and token go into the pass's own environment as
366 well as `CURRENT`. The cascade filled them when the pass started,
367 before the bring-up seeded the environment, and every child the
368 pass starts (the birth, the loop's own fm) copies that
369 environment, so each would otherwise reach the forge with a token
370 the environment no longer holds. The generic forge variables go
371 too: the workshop reads ``FORGE_TOKEN`` and ``FORGE_ADMIN_TOKEN``
372 before a forge's own, and a hosted job carries the real forge's in
373 them, so the newborn would otherwise hand GitHub's token to the
374 local Gitea and meet a 401.
375 """
376 CURRENT.name, CURRENT.mode = env, "host"
377 CURRENT.url, CURRENT.token = values["GITEA_URL"], values["GITEA_TOKEN"]
378 CURRENT.label = values["LABELS"].split(",")[0]
379 lane = LANES["gitea"]
380 os.environ[lane.url_var], os.environ[lane.token_var] = CURRENT.url, CURRENT.token
381 os.environ["FORGE_TOKEN"] = os.environ["FORGE_ADMIN_TOKEN"] = CURRENT.token
382
383
384def enter_environment(
385 name: str, extensions: Sequence[str], *, fresh: bool = False
386) -> Environment:
387 """Run the pass on environment *name*, brought up first; the environment.
388
389 A host-mode environment is brought up; with *fresh* it is removed
390 first, so the pass gets a new forge, and so is a directory that a
391 refused removal left. A removal that cannot take the whole
392 directory refuses, and the pass stops there
393 ([livery.workshop._devenv.remove][]). The forge's URL and token then
394 go into the pass's environment (`_enter_host`). A docker-mode
395 environment is the compose rig, used as it is, fresh or not.
396
397 In both modes the pass and every process it starts then run with
398 the variables the set of *extensions*, the list the loop's
399 workspace is born with, declares, each one naming ``{place}``
400 placed in the environment's directory. A tool's home placed there
401 goes with the environment: a release writes a remote into it that
402 names the environment's forge, and a home shared with an earlier
403 pass would keep that remote after the forge is gone, so a later
404 install that asks the remotes tries it and fails. A fresh pass
405 starts with nothing an earlier one wrote there.
406 """
407 from livery.workshop import _devenv
408
409 place = _devenv.environment(name)
410 if fresh and place.mode != "docker" and place.directory.is_dir():
411 for line in _devenv.remove(place):
412 print(line)
413 place = _devenv.environment(name)
414 if place.mode == "docker":
415 CURRENT.name, CURRENT.mode = name, "docker"
416 else:
417 _devenv.up_host(place, ("host",))
418 place = _devenv.environment(name)
419 _enter_host(place.values(), name)
420 from livery.workshop._composition import closure
421 from livery.workshop._extensions import installed_declaration
422
423 held = closure(extensions, installed_declaration)
424 variables = place.variables(held, os.environ)
425 edit_environments(variables)
426 for variable, value in variables.items():
427 print(f" {variable}: {value}")
428 return place
429
430
431def _dev_forge(kind: str) -> tuple[Forge, str]:
432 """The seeded local forge and its token; refusal teaches.
433
434 The credentials are the ones ``fm forge.dev.up`` writes into the
435 shared env file the cascade reads, so a warm machine needs
436 nothing beyond the containers being up.
437 """
438 lane = _lane(kind)
439 url = CURRENT.url or os.environ.get(lane.url_var, "")
440 token = CURRENT.token or os.environ.get(lane.token_var, "")
441 if not url or not token:
442 fail(
443 f"the local {kind}'s credentials are not in the environment."
444 f" Run `{footman.prog()} devenv.up`: it starts and seeds the"
445 f" environment and writes {lane.url_var} and {lane.token_var}"
446 " into the shared env file the cascade reads"
447 )
448 from livery.workshop._forge_lane import _connect
449
450 return _connect(kind, url, token), token
451
452
453def provision(kind: str = "gitea") -> None:
454 """Ensure the loop's repository, idempotently, with its secrets.
455
456 The repository is created on the seeded organisation when absent
457 and reused when present. ``UV_PUBLISH_TOKEN`` is written as an
458 Actions secret with the forge token's value, because the forge's
459 own package registry authenticates with the same credential as
460 its API. Re-running is the recovery procedure.
461 """
462 from livery.forge import ForgeError, RepoConfig
463
464 forge, token = _dev_forge(kind)
465 try:
466 existing = forge.get_repo(E2E_OWNER, E2E_REPO)
467 except ForgeError as error:
468 fail(
469 f"the local forge did not answer: {error}. Is the container"
470 f" up? `{footman.prog()} forge.dev.up --profile=gitea`"
471 )
472 if existing is None:
473 forge.create_repo(
474 E2E_OWNER,
475 E2E_REPO,
476 private=False,
477 description="The workshop's local CI loop. Scratch; recreated freely.",
478 )
479 print(f" created {E2E_OWNER}/{E2E_REPO}")
480 else:
481 print(f" reusing {E2E_OWNER}/{E2E_REPO}")
482 repo = forge.repository(E2E_OWNER, E2E_REPO)
483 if kind == "gitlab":
484 _make_project_public(os.environ.get(_lane(kind).url_var, ""), token)
485 print(" project: public, so its registry reads without a credential")
486 # Three secrets, all the lane token: the registry credential the
487 # publish reads, the forge lane's everyday token, and the admin
488 # ladder's, because the emitted governance job asks for it and
489 # the seeded admin account holds every grant anyway. GitLab's jobs
490 # push through a fourth, a project access token, since the job
491 # token cannot push.
492 secrets = {
493 "UV_PUBLISH_TOKEN": token,
494 "FORGE_TOKEN": token,
495 "FORGE_ADMIN_TOKEN": token,
496 }
497 if kind == "gitlab":
498 secrets["GITLAB_PUSH_TOKEN"] = _mint_push_token(
499 os.environ.get(_lane(kind).url_var, ""), token
500 )
501 repo.configure(RepoConfig(secrets=secrets))
502 print(f" secrets set: {', '.join(secrets)}")
503
504
505#: One GitLab API call for the loop's provisioning: (method, path,
506#: token, body) to (status, parsed body or text).
507GitlabCall = Callable[[str, str, str, "dict[str, object] | None"], "tuple[int, object]"]
508
509
510def _gitlab_call(
511 method: str, url: str, token: str, body: dict[str, object] | None = None
512) -> tuple[int, object]:
513 """One GitLab API call for the loop's provisioning; (status, parsed body)."""
514 import json
515 import urllib.error
516 import urllib.request
517
518 request = urllib.request.Request(
519 url,
520 data=json.dumps(body).encode() if body is not None else None,
521 method=method,
522 headers={"PRIVATE-TOKEN": token, "Content-Type": "application/json"},
523 )
524 try:
525 with urllib.request.urlopen(request, timeout=30) as response:
526 text = response.read().decode()
527 return int(response.status), (json.loads(text) if text else None)
528 except urllib.error.HTTPError as exc:
529 return exc.code, exc.read().decode(errors="replace")
530
531
532def _make_project_public(url: str, token: str, api: GitlabCall | None = None) -> None:
533 """Make the loop's GitLab project public, so its registry reads anonymously.
534
535 The runner installs the dev wheels from the project's PyPI index
536 and the loop locks against it here, both without a credential, as
537 both read Gitea's owner registry. A private project's index
538 answers 401 to both, whatever the group's visibility. *api* is the
539 call, the live one by default.
540 """
541 from urllib.parse import quote
542
543 api = api or _gitlab_call
544 path = f"{url}/api/v4/projects/{quote(f'{E2E_OWNER}/{E2E_REPO}', safe='')}"
545 status, body = api("PUT", path, token, {"visibility": "public"})
546 if status != 200:
547 fail(f"making {E2E_OWNER}/{E2E_REPO} public answered HTTP {status}: {body}")
548
549
550def _mint_push_token(url: str, token: str, api: GitlabCall | None = None) -> str:
551 """A project access token that may push to the loop's project; its value.
552
553 GitLab's job token cannot push, so the state store, the receipt
554 tags and the template artifact ride a project access token with
555 ``write_repository``. A token's value is readable at minting
556 alone, so every provisioning mints one and revokes the ones it
557 minted before, by name. *api* is the call, the live one by
558 default.
559 """
560 from datetime import UTC, datetime, timedelta
561 from urllib.parse import quote
562
563 api = api or _gitlab_call
564 base = f"{url}/api/v4/projects/{quote(f'{E2E_OWNER}/{E2E_REPO}', safe='')}"
565 name = "livery-loop-push"
566
567 def call(
568 path: str, *, method: str = "GET", body: dict[str, object] | None = None
569 ) -> tuple[int, object]:
570 return api(method, base + path, token, body)
571
572 status, listed = call("/access_tokens")
573 if status == 200 and isinstance(listed, list):573 ↛ 581line 573 didn't jump to line 581 because the condition on line 573 was always true
574 for item in listed:
575 if (575 ↛ 574line 575 didn't jump to line 574 because the condition on line 575 was always true
576 isinstance(item, dict)
577 and item.get("name") == name
578 and item.get("active")
579 ):
580 call(f"/access_tokens/{item['id']}", method="DELETE")
581 expires = (datetime.now(UTC) + timedelta(days=30)).date().isoformat()
582 status, made = call(
583 "/access_tokens",
584 method="POST",
585 body={
586 "name": name,
587 "scopes": ["api", "write_repository"],
588 "access_level": 40,
589 "expires_at": expires,
590 },
591 )
592 if status != 201 or not isinstance(made, dict) or not made.get("token"):
593 fail(f"the loop's push token was not minted: HTTP {status}: {made}")
594 return str(made["token"])
595
596
597@dataclass(frozen=True)
598class LoopMember:
599 """One member of the loop: its directory and the template it is born from.
600
601 Attributes:
602 name: The member's directory under ``packages/``.
603 template: The template whose verb, ``fm new.package.<template>``,
604 births it, and whose ``extensions`` the member's contract
605 lists.
606 """
607
608 name: str
609 template: str
610
611 def listed(self) -> tuple[str, ...]:
612 """The extensions the member's contract lists, from its template.
613
614 The template's ``extensions`` in their canonical form, which a
615 birth writes into the contract, so the loop and the birth name
616 one list. The template is read from every available one,
617 whatever the driving checkout's lists hold: the loop's
618 workspace, not the driver's, creates the member.
619 """
620 from livery.workshop._composition import canonical
621 from livery.workshop._extensions import installed_declaration
622 from livery.workshop._seeds import available_templates
623
624 declared = available_templates()[self.template]
625 return canonical(declared.extensions, installed_declaration)
626
627 def needs(self) -> tuple[str, ...]:
628 """The extensions the loop's root lists before the member is created.
629
630 Those its template's chain needs: each template's
631 ``extensions`` and the extension declaring it, since a creation
632 installs nothing.
633 """
634 from livery.workshop._seeds import chain_needs, template_chain
635
636 return chain_needs(template_chain(self.template))
637
638 def holds(self) -> tuple[str, ...]:
639 """The extensions the member's set holds: its list and what it requires."""
640 from livery.workshop._composition import closure
641 from livery.workshop._extensions import installed_declaration
642
643 return closure(self.listed(), installed_declaration)
644
645
646#: The loop's members. The nanobind member forces the wheels path on
647#: every pass: its wheel is built on the runner through cibuildwheel,
648#: published, and installed in the isolated leg. The cpp-conan member
649#: forces the native measurement and the conan path: its gate build
650#: is measured by gcov beside the runner image's gcc and its lines
651#: join the union, the release leg creates it into conan's cache, and
652#: the wave publishes it through the forge's conan registry.
653LOOP_MEMBERS: tuple[LoopMember, ...] = (
654 LoopMember("loop-echo", "python-library"),
655 LoopMember("loop-native", "nanobind-module"),
656 LoopMember("loop-cpp", "cpp-package"),
657)
658
659
660#: The member that contributes a point, and the point's name.
661CONTRIBUTING_MEMBER = "loop-echo"
662CONTRIBUTED_POINT = "echo-audit"
663
664#: The verb the loop's contract schedules at the nightly point: it
665#: verifies the member's released wheel against its own tests.
666VERIFY_TASK = "release.verify"
667
668
669def _contract_name(root: Path, member: str) -> str:
670 """The name the loop member *member*'s contract declares, as the verbs print it."""
671 import tomllib
672
673 contract = root / "packages" / member / "workshop.toml"
674 return str(tomllib.loads(contract.read_text("utf-8"))["name"])
675
676
677def _serving_probe(root: Path) -> Callable[[str], tuple[str, ...]]:
678 """What the loop's registries serve of a member, by name: its versions.
679
680 Each member is probed where its extensions release it, through the
681 probe they answer for that registry, under the name its contract
682 declares, which the wave publishes it under: the python index
683 normalises a name, and a conan registry matches it exactly. Each
684 probe is built once and polled through the wave's wait.
685 """
686 from livery.workshop import _lifecycle
687 from livery.workshop._packages import discover_packages
688
689 members = {package.member: package for package in discover_packages(root)}
690 probes: dict[str, Registry] = {}
691
692 def versions(name: str) -> tuple[str, ...]:
693 package = members[name]
694 probe = probes.get(name)
695 if probe is None:
696 probe = probes[name] = _lifecycle.registry(package)
697 return probe.versions(package.name)
698
699 return versions
700
701
702def _require_host_alias(kind: str = "gitea") -> None:
703 """Refuse until the host resolves the lane's compose hostname.
704
705 One registry URL must be true on both sides of the loop: the
706 runner resolves the compose service name, the host does not, and
707 a split URL forces an unlocked workspace and a hostname fork
708 through every file. The one-line alias makes the compose name
709 true on the host too, and the lock then carries a URL both sides
710 can read.
711 """
712 import urllib.error
713 import urllib.request
714
715 if CURRENT.mode == "host":
716 # A localhost URL needs no alias: the same address is true for
717 # the driver, the forge and the runner, all on this machine.
718 return
719 lane = _lane(kind)
720 try:
721 with urllib.request.urlopen(lane.alias + lane.version_path, timeout=2):
722 return
723 except urllib.error.HTTPError:
724 # The host answered: GitLab's version endpoint refuses an
725 # anonymous read with 401, and the alias resolved all the same.
726 return
727 except (urllib.error.URLError, TimeoutError, OSError):
728 fail(
729 f"the host cannot reach {lane.alias}, the compose"
730 " hostname CI uses for the registry (a network resolving"
731 f" '{lane.host}' elsewhere fails the same way; /etc/hosts wins"
732 " over DNS). Add the alias once:"
733 f" sudo sh -c 'echo \"127.0.0.1 {lane.host}\" >> /etc/hosts'"
734 )
735
736
737def _loop_home(kind: str) -> Path:
738 """Where the lane's workspace lives: durable, per machine and per forge.
739
740 One directory per lane: the workspaces have different remotes,
741 different registries and different histories, and one pass must
742 never adopt the other lane's checkout.
743 """
744 from livery.footman import data_dir
745
746 home = data_dir() / "workshop-e2e"
747 if CURRENT.name:
748 home = home / CURRENT.name
749 return home / kind
750
751
752def _dev_pins(root: Path, head: str, members: tuple[str, ...]) -> dict[str, str]:
753 """The dev versions this pass built for *members*, by distribution name.
754
755 Read from the wheels [livery.workshop._e2e._dev_wheels][] finds.
756 """
757 return {
758 name: wheel.name.split("-", 2)[1]
759 for name, wheel in _dev_wheels(root, head, members).items()
760 }
761
762
763def _dev_wheels(root: Path, head: str, members: tuple[str, ...]) -> dict[str, Path]:
764 """The wheel this pass built for each of *members*, by distribution name.
765
766 The newest wheel in each member's ``dist``, which the dev act just
767 filled, checked against *head*: a dev version's local segment is
768 ``<branch>.<sha>.<date>``, ``.dirty`` appended for a tree no
769 commit describes, so a wheel a previous pass left behind can never
770 pin the loop. Refuses, naming the member, when a dist holds no
771 wheel or its newest wheel is another commit's.
772 """
773 found: dict[str, Path] = {}
774 for member in members:
775 dist = root / "packages" / member / "dist"
776 wheels = sorted(dist.glob("*.whl"), key=lambda wheel: wheel.stat().st_mtime)
777 if not wheels:
778 fail(f"no dev wheel in {dist}: the dev act built nothing for {member}")
779 name, version = wheels[-1].name.split("-", 2)[:2]
780 local = version.partition("+")[2].split(".")
781 if local and local[-1] == "dirty":
782 local.pop()
783 # The sha as git describe spells it, ``g`` first; a hex sha
784 # never starts with a ``g`` of its own.
785 sha = local[-2].removeprefix("g") if len(local) >= 2 else ""
786 if not sha or not head.startswith(sha):
787 fail(
788 f"the newest wheel in {dist} is {wheels[-1].name}, built from"
789 f" {sha or 'no commit'}, not from HEAD {head[:12]}: the dev"
790 f" act did not build this pass's {member}"
791 )
792 found[name.replace("_", "-")] = wheels[-1]
793 return found
794
795
796def _dev_split(
797 root: Path, stack: Sequence[str] = ()
798) -> tuple[list[str], dict[str, str]]:
799 """The dev members this pass builds, and the release each other one pins, by member.
800
801 A member nothing unreleased touches cannot be built as a dev
802 wheel: that number would sort below its release and satisfy no
803 floor naming it. The loop pins the release instead, and drops the
804 member's stale rehearsal wheels from the registry, which a
805 first-index resolve would otherwise pick over the release.
806 """
807 from livery.workshop._dev_release import unchanged_since_release
808 from livery.workshop._packages import discover_packages
809
810 packages = {package.member: package for package in discover_packages(root)}
811 changed: list[str] = []
812 released: dict[str, str] = {}
813 for member in dev_members(root, tuple(stack) or None):
814 version = unchanged_since_release(root, packages[member])
815 if version:
816 released[member] = version
817 else:
818 changed.append(member)
819 return changed, released
820
821
822def write_dev_index(folder: Path, wheels: Collection[Path]) -> Path:
823 """Lay *wheels* out at *folder* as a simple index uv reads; *folder*.
824
825 One directory per project, named as PEP 503 normalises it, holding
826 the project's wheels and a page that links them, and a root page
827 that links the projects. Whatever *folder* held before goes, so a
828 wheel an earlier pass built is never offered again.
829 """
830 if folder.exists():
831 shutil.rmtree(folder)
832 folder.mkdir(parents=True)
833 projects: dict[str, list[Path]] = {}
834 for wheel in wheels:
835 name = re.sub(r"[-_.]+", "-", wheel.name.split("-", 1)[0]).lower()
836 projects.setdefault(name, []).append(wheel)
837 for name, files in sorted(projects.items()):
838 (folder / name).mkdir()
839 for wheel in files:
840 shutil.copy2(wheel, folder / name / wheel.name)
841 _index_page(folder / name, sorted(wheel.name for wheel in files))
842 _index_page(folder, [f"{name}/" for name in sorted(projects)])
843 return folder
844
845
846def _index_page(directory: Path, targets: list[str]) -> None:
847 """Write *directory*'s ``index.html``: one link per target."""
848 links = "".join(f'<a href="{target}">{target}</a>\n' for target in targets)
849 (directory / "index.html").write_text(
850 f"<!DOCTYPE html>\n<html><body>\n{links}</body></html>\n", "utf-8"
851 )
852
853
854def checkout_index(
855 root: Path, folder: Path, extensions: Sequence[str] | None = None
856) -> str:
857 """Build what a newborn installs from *root* as a local index at *folder*; its URL.
858
859 The members are [livery.workshop._e2e.dev_members][]: the workshop,
860 what it depends on, and the extensions a birth lists. Each is built
861 by `build_content_named`, at the version its manifest declares with
862 a local segment naming its content, and
863 [livery.workshop._e2e.write_dev_index][] lays the wheels out. Named
864 first in ``UV_INDEX``, the index serves every distribution it holds,
865 whatever its version, because uv takes a package from the first
866 index that has it; every other package comes from the next.
867 ``UV_FIND_LINKS`` would not do: a folder named there only adds
868 candidates, and a stable release on PyPI beats a dev wheel
869 (measured). This is how a workspace installs this checkout's code
870 before a release, never the dev act, which on a main-family branch
871 is the release train. *extensions* is the stack the birth lists,
872 the stock list when None.
873 """
874 import tempfile
875
876 from livery.workshop._packages import discover_packages
877
878 members = set(dev_members(root, extensions))
879 with tempfile.TemporaryDirectory() as scratch:
880 built = Path(scratch) / "wheels"
881 for package in discover_packages(root):
882 if package.member in members:
883 build_content_named(package, root, built)
884 write_dev_index(folder, sorted(built.glob("*.whl")))
885 return folder.as_uri()
886
887
888def build_content_named(package: Package, root: Path, out: Path) -> Path:
889 """Build *package*'s wheel into *out*, at a version naming its content; the wheel.
890
891 The build reads a copy of the package's directory, without its build
892 outputs and environments. The copy's version gains a local segment,
893 a digest of the copy's files, so the wheel's file name changes
894 whenever its content does. uv caches an installed wheel by its file
895 name and the place it came from, and installs the cached copy again
896 without reading a newer file of that name (measured). The package's
897 own files stay as they are. *root* is the workspace *package*
898 belongs to, whose stamp phase writes the version.
899
900 Raises:
901 Failed: when nothing in the package's set answers its version,
902 or adds a step to the stamp phase.
903 """
904 import tempfile
905
906 import livery.toolroom.tools as toolroom
907
908 skipped = shutil.ignore_patterns(
909 ".venv", "dist", "build", "__pycache__", "_docs", "*.egg-info"
910 )
911 with tempfile.TemporaryDirectory() as scratch:
912 copy = Path(scratch) / "copy" / package.member
913 shutil.copytree(package.directory, copy, ignore=skipped)
914 _stamp_content(replace(package, directory=copy), root)
915 built = Path(scratch) / "built"
916 toolroom.uv.opts(cwd=root, recorded=False)(
917 "build", "--wheel", "--out-dir", str(built), str(copy)
918 )
919 (wheel,) = built.glob("*.whl")
920 out.mkdir(parents=True, exist_ok=True)
921 return Path(shutil.copy2(wheel, out / wheel.name))
922
923
924def _stamp_content(package: Package, root: Path) -> None:
925 """Give *package*'s version a local segment naming its tree's content.
926
927 A rebuilt wheel at an unchanged version carries the last one's file
928 name, and uv installs a cached wheel by its name: a birth installed
929 the code from before a fix (measured). The segment is a digest of
930 the package's files, so a change gets a new name and an unchanged
931 tree the same one, and every floor the version met still holds, a
932 local version sorting with its public one. The stamp phase of the
933 package's set writes it, into the files the package keeps its
934 version in.
935
936 Raises:
937 Failed: when nothing in the package's set answers its version,
938 or adds a step to the stamp phase.
939 """
940 import hashlib
941
942 from livery.workshop import _lifecycle
943
944 tree = package.directory
945 digest = hashlib.sha256()
946 for path in sorted(path for path in tree.rglob("*") if path.is_file()):
947 digest.update(path.relative_to(tree).as_posix().encode("utf-8"))
948 digest.update(b"\0")
949 digest.update(path.read_bytes())
950 digest.update(b"\0")
951 version = _lifecycle.current_version(package)
952 _lifecycle.stamp(package, root, f"{version}+checkout.{digest.hexdigest()[:12]}")
953
954
955def _dev_index(kind: str, stack: Sequence[str] = ()) -> str:
956 """This checkout's index under the lane's home, for the birth to read; its URL.
957
958 The birth locks the newborn before the pass publishes anything, and
959 every extension the birth lists is in that first lock, released or
960 not: [livery.workshop._e2e.checkout_index][] builds them.
961 """
962 from livery.workshop._extensions import workspace_root
963
964 root = workspace_root()
965 if root is None:
966 fail("no workspace: no workshop.toml above the working directory")
967 folder = _loop_home(kind).parent / f"{kind}-dev-index"
968 url = checkout_index(root, folder, tuple(stack) or None)
969 held = sum(1 for entry in folder.iterdir() if entry.is_dir())
970 print(
971 f" checkout index: {held} distributions at {folder}, read first by the birth"
972 )
973 return url
974
975
976def require_dev_branch(root: Path) -> None:
977 """Refuse unless *root* stands on a branch whose release verb is the dev act.
978
979 The loop builds the dev wheels it installs with `workflow.release`
980 in this checkout, and the branch decides that verb's act. On
981 ``main`` or a reserved ``workflow/`` branch it is the release train,
982 which opens a real release on this repository's forge, and a merged
983 one publishes to the real index; a detached HEAD names no act at all.
984 """
985 from livery.workshop._release_driver import runs_the_train
986
987 branch = git.current_branch(root)
988 if branch and not runs_the_train(branch):
989 return
990 where = f"'{branch}'" if branch else "a detached HEAD"
991 prog = footman.prog()
992 fail(
993 f"the loop builds its dev wheels with `{prog} workflow.release`, and on"
994 f" {where} that is the release train, which opens a real release: start a"
995 f" branch at this commit (`{prog} start chore/<slug>`) and run the loop there"
996 )
997
998
999@contextlib.contextmanager
1000def scratch_branch(root: Path | None, *, enabled: bool) -> Generator[None, None, None]:
1001 """Run a pass from a scratch branch where a build would be the release train.
1002
1003 ``--from-main``: a nightly's checkout stands on ``main``, where the
1004 loop's first build is the release train, so the pass moves to
1005 ``chore/loop-<stamp>`` at the same commit before anything builds
1006 and returns afterwards, deleting the branch when it gained no
1007 commit. A checkout already on a dev branch, or none, is left
1008 alone. A dirty tree refuses naming the paths, so the scratch branch
1009 holds exactly the commit it started from.
1010
1011 Args:
1012 root: The driver checkout, None when the pass runs outside one.
1013 enabled: Whether ``--from-main`` was asked.
1014
1015 Yields:
1016 Nothing; the pass runs inside.
1017 """
1018 from livery.workshop._release_driver import runs_the_train
1019
1020 if root is None or not enabled:
1021 yield
1022 return
1023 branch = git.current_branch(root)
1024 if branch and not runs_the_train(branch):
1025 yield
1026 return
1027 dirty = git.dirty_paths(root)
1028 if dirty:
1029 fail(
1030 "--from-main starts a scratch branch at this commit, and the working"
1031 f" tree has changes: {', '.join(dirty)}; commit or discard them first"
1032 )
1033 origin = branch or git.head_sha(root)
1034 scratch = f"chore/loop-{datetime.now(UTC):%Y%m%dT%H%M%SZ}"
1035 git.create_branch(root, scratch)
1036 where = f"'{branch}'" if branch else "a detached HEAD"
1037 print(f" from main: left {where} for {scratch} at {git.head_sha(root)[:12]}")
1038 try:
1039 yield
1040 finally:
1041 _leave_scratch(root, origin, scratch, branch)
1042
1043
1044def _leave_scratch(root: Path, origin: str, scratch: str, branch: str) -> None:
1045 """Return *root* to *origin*; *scratch* goes when it gained nothing."""
1046 if git.current_branch(root) != scratch:1046 ↛ 1047line 1046 didn't jump to line 1047 because the condition on line 1046 was never true
1047 print(
1048 f" from main: the checkout left {scratch} during the pass;"
1049 " nothing restored"
1050 )
1051 return
1052 if not git.is_clean(root):1052 ↛ 1053line 1052 didn't jump to line 1053 because the condition on line 1052 was never true
1053 print(f" from main: the tree has changes, so {scratch} stays checked out")
1054 return
1055 gained = git.commits_since(root, origin)
1056 if branch:1056 ↛ 1059line 1056 didn't jump to line 1059 because the condition on line 1056 was always true
1057 git.switch(root, branch)
1058 else:
1059 git.detach(root, origin)
1060 back = branch or origin[:12]
1061 if gained:
1062 print(
1063 f" from main: back on {back}; {scratch} holds {gained} commit(s) and stays"
1064 )
1065 else:
1066 git.delete_branch(root, scratch)
1067 print(f" from main: back on {back}; {scratch} deleted")
1068
1069
1070def _publish_dev_wheels(kind: str, stack: Sequence[str] = ()) -> dict[str, str]:
1071 """Publish the workspace's dev wheels to the loop's registry; the pins.
1072
1073 The loop installs the workshop being edited, so every pass
1074 publishes fresh dev wheels first: a lock pinned to an older dev
1075 version would test yesterday's code with today's templates. The
1076 dev act is idempotent at a given commit, and a re-publish of the
1077 same version walks past. A dirty tree is the exception: its dev
1078 version is the same for every edit, so the wheels it published
1079 before are dropped first, and the pass installs the tree as it is
1080 now. Returns the published versions by distribution name, for the
1081 loop's lock to pin exactly.
1082 """
1083 from livery.footman import run
1084 from livery.workshop._extensions import workspace_root
1085 from livery.workshop._packages import discover_packages
1086
1087 root = workspace_root()
1088 if root is None:1088 ↛ 1089line 1088 didn't jump to line 1089 because the condition on line 1088 was never true
1089 fail("no workspace: no workshop.toml above the working directory")
1090 require_dev_branch(root)
1091 _, token = _dev_forge(kind)
1092 packages = {package.member: package for package in discover_packages(root)}
1093 changed, pinned = _dev_split(root, stack)
1094 released = {packages[member].name: version for member, version in pinned.items()}
1095 for member, version in pinned.items():
1096 print(
1097 f" {member}: nothing unreleased since {version}; the loop pins the release"
1098 )
1099 if released:1099 ↛ 1107line 1099 didn't jump to line 1107 because the condition on line 1099 was always true
1100 purged = _purge(
1101 kind, os.environ.get(_lane(kind).url_var, ""), token, names=released
1102 )
1103 print(
1104 f" registry: {len(purged)} stale rehearsal release(s) of the pinned"
1105 " member(s) dropped"
1106 )
1107 if changed and not git.is_clean(root):
1108 purged = _purge(
1109 kind,
1110 os.environ.get(_lane(kind).url_var, ""),
1111 token,
1112 names={packages[member].name for member in changed},
1113 )
1114 print(
1115 f" registry: {len(purged)} dev wheel(s) of the dirty tree dropped,"
1116 " so this pass installs the tree as it is now"
1117 )
1118 if changed:1118 ↛ 1130line 1118 didn't jump to line 1130 because the condition on line 1118 was always true
1119 run(
1120 # The runner under its own name: a branded instance is not
1121 # called `fm`, and a literal would spawn nothing there.
1122 [footman.prog(), "--yes", "workflow.release", *changed],
1123 cwd=root,
1124 # The whole environment, extended: env= replaces, and a bare
1125 # pair would strip PATH from under the child fm.
1126 env=child_environment(
1127 PYTHON_PUBLISH_INDEX=_lane(kind).publish(), UV_PUBLISH_TOKEN=token
1128 ),
1129 )
1130 pins = _dev_pins(root, git.head_sha(root), tuple(changed)) | released
1131 print(" dev wheels: published to the loop's registry")
1132 return pins
1133
1134
1135def _purge(
1136 kind: str, url: str, token: str, *, names: Collection[str] | None = None
1137) -> list[str]:
1138 """Drop the loop's packages from the registry at *url*: every one, or *names*.
1139
1140 Gitea keeps them under the owner, GitLab under the project.
1141 """
1142 from livery.forge._registry import purge_gitlab_packages, purge_packages
1143
1144 project = f"{E2E_OWNER}/{E2E_REPO}"
1145 if names is None:
1146 if kind == "gitlab":1146 ↛ 1147line 1146 didn't jump to line 1147 because the condition on line 1146 was never true
1147 return purge_gitlab_packages(url, project, token=token)
1148 return purge_packages(url, E2E_OWNER, token=token)
1149 if kind == "gitlab":
1150 return purge_gitlab_packages(url, project, token=token, names=names)
1151 return purge_packages(url, E2E_OWNER, token=token, names=names)
1152
1153
1154def start_over(
1155 lane: Forge,
1156 token: str,
1157 root: Path,
1158 *,
1159 url: str,
1160 wait: float = 120.0,
1161 kind: str = "gitea",
1162) -> list[str]:
1163 """Delete the loop's repository, its registry releases, and *root*; the lines.
1164
1165 Refuses while *root* holds commits its origin has not seen, since
1166 the repository they would land in is about to go. A repository or
1167 a release already gone is not an error: the next birth wants them
1168 absent, and a re-run of the reset is the recovery procedure. A
1169 delete the forge answers too slowly for the client completes on
1170 the server anyway, so the reset waits up to *wait* seconds for
1171 the repository to be gone before it refuses.
1172 """
1173 if (root / ".git").is_dir():
1174 unpushed = _unpushed_commits(root)
1175 if unpushed:
1176 listed = "\n".join(f" {line}" for line in unpushed)
1177 fail(
1178 f"{root} holds commits its origin has not seen:\n{listed}\n"
1179 " push or discard them before starting over"
1180 )
1181 lines: list[str] = []
1182 try:
1183 lane.delete_repo(E2E_OWNER, E2E_REPO)
1184 except ForgeError as error:
1185 if not _gone_within(lane, wait):
1186 raise ForgeError(
1187 f"{E2E_OWNER}/{E2E_REPO} is still on the dev forge after the"
1188 f" delete's wait: {error}"
1189 ) from error
1190 lines.append(
1191 f" deleted {E2E_OWNER}/{E2E_REPO} on the dev forge (the delete"
1192 " outran the client's wait and finished on the server)"
1193 )
1194 else:
1195 lines.append(f" deleted {E2E_OWNER}/{E2E_REPO} on the dev forge")
1196 purged = _purge(kind, url, token)
1197 lines.append(
1198 f" purged {len(purged)} release(s) from the registry"
1199 + (f": {', '.join(purged)}" if purged else "")
1200 )
1201 if root.exists():
1202 _rmtree(root)
1203 lines.append(f" removed {root}")
1204 return lines
1205
1206
1207def _gone_within(lane: Forge, wait: float) -> bool:
1208 """Whether the loop's repository is gone from the forge within *wait* seconds."""
1209
1210 def gone() -> bool | Pending:
1211 return True if lane.get_repo(E2E_OWNER, E2E_REPO) is None else Pending()
1212
1213 try:
1214 return follow(
1215 None,
1216 gone,
1217 subject="the loop's repository",
1218 interval=5.0,
1219 timeout=wait,
1220 quiet=None,
1221 retried=(),
1222 )
1223 except FollowTimedOut:
1224 return False
1225
1226
1227def _rmtree(path: Path) -> None:
1228 """Remove *path* whole; git's pack files are read-only, which Windows honours.
1229
1230 A file that goes between the walk and its chmod is left to the
1231 removal: git's background maintenance drops its lock under
1232 ``.git/objects`` while a fresh clone is being torn down.
1233 """
1234 import stat
1235
1236 for entry in path.rglob("*"):
1237 if entry.is_file() and not entry.is_symlink():
1238 try:
1239 entry.chmod(entry.stat().st_mode | stat.S_IWRITE)
1240 except FileNotFoundError:
1241 continue
1242 shutil.rmtree(path)
1243
1244
1245def _origin_main(root: Path) -> str:
1246 """The commit origin's main is at, asked of origin; empty when it cannot say."""
1247 try:
1248 return git.ls_remote(root, "refs/heads/main").get("refs/heads/main", "")
1249 except git.GitError:
1250 return ""
1251
1252
1253def _unpushed_commits(root: Path) -> list[str]:
1254 """The commits on any local branch of *root* that its origin does not hold."""
1255 try:
1256 git.fetch(root)
1257 except git.GitError:
1258 return [] # no reachable origin: nothing there could hold them anyway
1259 return git.unpushed(root)
1260
1261
1262def _birth(
1263 kind: str,
1264 url: str,
1265 index: str = "",
1266 stack: Sequence[str] = (),
1267 driver: Path | None = None,
1268) -> Path:
1269 """Birth or resume the loop's workspace; the root it lives at.
1270
1271 ``fm new.project`` owns the whole half: seeds, git, repository,
1272 protection, and the setup pull request. It runs as a child of the
1273 pass, this interpreter's own footman with the pass's environment:
1274 a task run in-process starts from the run's pinned environment,
1275 and the pass's own settings, its unsigned commits, would never
1276 reach the birth's git. A workspace already there is resumed, so
1277 this is the recovery procedure too. *index*, when given, is
1278 searched before any other index the environment names
1279 ([livery.workshop._e2e._dev_index][]); *stack*, when given, is the
1280 extensions the workspace lists instead of the stock list, which the
1281 birth takes through ``--extensions``. The verb
1282 runs from *driver*, the checkout running the pass, whose own project
1283 mounts the workshop being edited, so a bare runner needs nothing
1284 self-added and the birth runs this checkout's code; the folder is
1285 then absolute, and the verb hands the finish to the newborn's own
1286 runner. Without a driver the machine's footman answers from the
1287 loop's home.
1288 """
1289 import sys
1290
1291 home = _loop_home(kind)
1292 env = child_environment()
1293 if index:
1294 env["UV_INDEX"] = " ".join(filter(None, (index, os.environ.get("UV_INDEX"))))
1295 home.mkdir(parents=True, exist_ok=True)
1296 # A workspace already there is a birth to resume: the verb refuses
1297 # to start a second one in its folder.
1298 resume = ["--resume"] if (home / E2E_REPO / "workshop.toml").is_file() else []
1299 result = footman.run(
1300 [
1301 sys.executable,
1302 "-m",
1303 "livery.footman",
1304 "--yes",
1305 "new.project",
1306 str(home / E2E_REPO) if driver is not None else E2E_REPO,
1307 f"--forge={kind}",
1308 f"--owner={E2E_OWNER}",
1309 f"--url={_lane(kind).alias}",
1310 "--description=The workshop's local CI loop. Scratch; recreated freely.",
1311 *([f"--extensions={','.join(stack)}"] if stack else []),
1312 *resume,
1313 ],
1314 cwd=driver if driver is not None else home,
1315 env=env,
1316 nofail=True,
1317 timeout=900.0,
1318 )
1319 print(result.stdout.rstrip("\n"))
1320 if result.code != 0:1320 ↛ 1321line 1320 didn't jump to line 1321 because the condition on line 1320 was never true
1321 fail(f"the loop's birth exited {result.code}:\n{result.stderr}")
1322 return home / E2E_REPO
1323
1324
1325def _enable_steps(contract_text: str, steps: tuple[str, ...]) -> str:
1326 """*contract_text* with a table turning on each of *steps* it does not have yet.
1327
1328 A step is ``<phase>.<step>``, and its table is
1329 ``[phases.<phase>.<step>] enabled = true``, the root contract's
1330 spelling for a step its extension declares off.
1331 """
1332 for name in steps:
1333 phase, _, step = name.partition(".")
1334 table = f"[phases.{phase}.{step}]"
1335 if table in contract_text:
1336 continue
1337 contract_text = (
1338 contract_text.rstrip("\n")
1339 + f"\n\n# The {name} step, declared off, turned on for the pass.\n"
1340 + f"{table}\n"
1341 + "enabled = true\n"
1342 )
1343 return contract_text
1344
1345
1346def _has_remote(root: Path) -> bool:
1347 """Whether the loop's workspace at *root* has its forge remote."""
1348 return git.has_remote(root)
1349
1350
1351def _authenticate_remote(root: Path, token: str, kind: str = "gitea") -> None:
1352 """Embed the lane token in the scratch workspace's remote.
1353
1354 Birth writes a credential-free remote by design and a human's
1355 pushes ride their credential helper; the loop is automation on a
1356 scratch workspace, so the token rides the remote URL instead.
1357 The URL is a [livery.footman.Secret][]: a profile trace and every
1358 other place a command line is shown carry ``***`` in its place,
1359 and git alone receives the value. Gitea validates the token and
1360 ignores the username (measured), and ``oauth2`` is the
1361 conventional stand-in.
1362 """
1363 bare = _lane(kind).alias.removeprefix("http://")
1364 url = footman.Secret(f"http://oauth2:{token}@{bare}/{E2E_OWNER}/{E2E_REPO}.git")
1365 try:
1366 git.set_remote_url(root, "origin", url)
1367 except git.GitError as error:
1368 fail(str(error))
1369
1370
1371def _lock_pins(root: Path, pins: dict[str, str]) -> None:
1372 """Lock the loop onto exactly the dev wheels this pass published.
1373
1374 The dev number counts commits since the release tag, so a longer
1375 branch publishes a higher number and a lock left to upgrade
1376 freely keeps resolving that branch's wheels (measured: the loop
1377 ran one branch's emitter under another branch's wheels). The
1378 pins name the four versions outright; everything else keeps its
1379 locked version.
1380 """
1381 import livery.toolroom.tools as toolroom
1382
1383 args = [f"--upgrade-package={name}=={version}" for name, version in pins.items()]
1384 result = toolroom.uv.opts(cwd=root, nofail=True)("lock", *args)
1385 if result.code != 0:
1386 fail(
1387 f"uv lock in the loop workspace exited {result.code}:"
1388 f"\n{result.stdout}{result.stderr}"
1389 )
1390
1391
1392def _eat_dev_wheels(
1393 root: Path,
1394 pins: dict[str, str],
1395 kind: str = "gitea",
1396 *,
1397 enabled: tuple[str, ...] = (),
1398) -> str:
1399 """Point the workspace at this pass's dev wheels; the pushed head sha.
1400
1401 The registry joins the contract, the lock pins the wheels the
1402 pass just published, and only then does the workspace re-render
1403 through its own fm, so the files the runner executes come from
1404 the templates and the emitter under test. Measured necessity: a
1405 released workshop predating the emitted verbs fails the docs job
1406 with "no task named", and a render before the re-lock ships the
1407 previous pass's workflows. The loop's fm then writes the tool lock
1408 against the contract the wiring edited, since CI's setup refuses a
1409 lock the contract has moved past. *enabled* names the steps the
1410 contract turns on, each at its own address, the steps the extension
1411 under test declares off. Idempotent: an already-wired workspace
1412 pushes nothing.
1413 """
1414 from livery.workshop._new_project import _SETUP_BRANCH
1415
1416 # main is protected by the birth's own assertion, so the wiring
1417 # rides the setup branch and its pull request proves the gate.
1418 # The branch is pass-owned: rebuilt from current main every
1419 # time, because a reused branch atop stale bases conflicts with
1420 # the very squashes it produced (measured: green statuses,
1421 # mergeable false, the merge API answering 'try again later'
1422 # forever).
1423 _fresh_branch(root, _SETUP_BRANCH)
1424 lane = _lane(kind)
1425 host_url = os.environ.get(lane.url_var, "")
1426 contract = root / "workshop.toml"
1427 if contract.is_file():1427 ↛ 1431line 1427 didn't jump to line 1431 because the condition on line 1427 was always true
1428 body = contract.read_text("utf-8")
1429 if host_url and host_url in body:1429 ↛ 1430line 1429 didn't jump to line 1430 because the condition on line 1429 was never true
1430 contract.write_text(body.replace(host_url, lane.alias), "utf-8")
1431 loop_index, loop_publish = lane.index(), lane.publish()
1432 # The registry lives in the contract, and the template renders it
1433 # into pyproject from there: every re-render preserves the wiring
1434 # (a roster change re-renders pyproject too, and an unwired
1435 # render re-locks the workshop back to the released index).
1436 contract_file = root / "workshop.toml"
1437 original = contract_file.read_text("utf-8")
1438 contract_text = original
1439 # No prerelease declaration: uv's default admits a prerelease
1440 # where only prereleases exist or a requirement names one, and
1441 # the first-index strategy serves the workshop from the loop
1442 # index alone, so the dev wheels resolve with nothing declared
1443 # (measured). A global "allow" is the measured trap: it resolved
1444 # mkdocs 2.0.dev3 from PyPI and the docs job lost
1445 # mkdocs.exceptions. The heal below removes any mode an earlier
1446 # pass declared.
1447 if "[registries.python]" not in contract_text:
1448 contract_text = (
1449 contract_text.rstrip("\n")
1450 + "\n\n# The loop eats the workshop's dev wheels from the local\n"
1451 + "# registry; the compose hostname is true on both sides,\n"
1452 + "# the host through its /etc/hosts alias. The publish base\n"
1453 + "# is where the release wave uploads; without it the wave\n"
1454 + "# refuses rather than default an upload endpoint.\n"
1455 + "[registries.python]\n"
1456 + f'url = "{loop_index}"\n'
1457 + f'publish = "{loop_publish}"\n'
1458 )
1459 elif f'publish = "{loop_publish}"' not in contract_text:1459 ↛ 1460line 1459 didn't jump to line 1460 because the condition on line 1459 was never true
1460 contract_text = contract_text.replace(
1461 f'url = "{loop_index}"\n',
1462 f'url = "{loop_index}"\npublish = "{loop_publish}"\n',
1463 1,
1464 )
1465 contract_text = re.sub(r'prerelease = "[^"]*"\n', "", contract_text)
1466 if "[docs]" not in contract_text and "docs" in _listed_extensions(root):1466 ↛ 1472line 1466 didn't jump to line 1472 because the condition on line 1466 was never true
1467 # The loop builds the site for real and publishes nowhere:
1468 # the runner container has no docker, and the release act
1469 # needs main green (an undeclared seam defaults to the forge
1470 # kind's, container on gitea, which dies on the missing
1471 # docker).
1472 contract_text = (
1473 contract_text.rstrip("\n")
1474 + "\n\n# The site builds for real; nothing serves it here.\n"
1475 + "[docs]\n"
1476 + 'publish = "none"\n'
1477 )
1478 if "python-versions" not in contract_text:
1479 # The fast shape: one python per point, the loop's whole
1480 # matrix; livery's own contract keeps the derived pair.
1481 marker = "\n[ci]\n"
1482 if marker not in contract_text:1482 ↛ 1483line 1482 didn't jump to line 1483 because the condition on line 1482 was never true
1483 fail(
1484 "the loop's contract has no [ci] table to declare its"
1485 " python on; birth seeds one, so this workspace was not"
1486 " born by the loop"
1487 )
1488 contract_text = contract_text.replace(
1489 marker, marker + 'python-versions = ["3.14"]\naffected-legs = true\n', 1
1490 )
1491 if CURRENT.label != "ubuntu-latest":
1492 # The environment's runner answers to its own label, so the
1493 # workspace names it: the check legs run there, and the wheels
1494 # job builds there.
1495 contract_text = contract_text.replace(
1496 'runners = ["ubuntu-latest"]', f'runners = ["{CURRENT.label}"]', 1
1497 )
1498 if "speed-marks" not in contract_text:
1499 # Seeded on its own: an adopted loop born before the key
1500 # exists keeps its contract, and the judge stays off without
1501 # it, which the scoped-leg proof pins.
1502 contract_text = contract_text.replace(
1503 "\n[ci]\n", "\n[ci]\nspeed-marks = true\n", 1
1504 )
1505 if re.search(r"^shards *=", contract_text, re.MULTILINE) is None:
1506 # The check leg runs as shards on the environment's runner, the
1507 # one `[ci] runners` names, so every pass proves the slice, the
1508 # fold and the durations row; a contract that declares a count
1509 # keeps it.
1510 contract_text = contract_text.replace(
1511 "\n[ci]\n", f'\n[ci]\nshards = {{ "{CURRENT.label}" = {SHARDS} }}\n', 1
1512 )
1513 if "host-allowed" not in contract_text:
1514 # The allowance for the build tools: a runner with cmake and
1515 # ninja on PATH serves them from the host, one without takes
1516 # the store's, and the sync says which. The loop's container
1517 # runner has neither, so the pass proves the store's path.
1518 marker = "\n[toolroom]\n"
1519 if marker not in contract_text:1519 ↛ 1520line 1519 didn't jump to line 1520 because the condition on line 1519 was never true
1520 fail(
1521 "the loop's contract has no [toolroom] table to allow host"
1522 " tools in; birth seeds one, so this workspace was not"
1523 " born by the loop"
1524 )
1525 contract_text = contract_text.replace(
1526 marker, marker + 'host-allowed = ["cmake", "ninja"]\n', 1
1527 )
1528 if "[[ci.schedule]]" not in contract_text:
1529 # The schedule seam's first entry: the nightly point verifies
1530 # the member's released wheel, so a dispatched nightly proves
1531 # a task scheduled through the contract with no YAML change.
1532 contract_text = (
1533 contract_text.rstrip("\n")
1534 + "\n\n# The nightly point verifies the released member.\n"
1535 + "[[ci.schedule]]\n"
1536 + 'point = "nightly"\n'
1537 + f'task = "{VERIFY_TASK}"\n'
1538 + 'args = ["loop-echo", "--python={python}"]\n'
1539 )
1540 # An adopted loop keeps the entry its birth wrote: the entry's task
1541 # is set to the verb that verifies a release, whatever it named.
1542 contract_text = re.sub(
1543 r'task = "release\.[a-z-]+"\nargs = \["loop-echo"',
1544 f'task = "{VERIFY_TASK}"\nargs = ["loop-echo"',
1545 contract_text,
1546 )
1547 contract_text = _enable_steps(contract_text, enabled)
1548 if contract_text != original:
1549 contract_file.write_text(contract_text, "utf-8")
1550 # Birth's pyproject predates the registry in the contract, and the
1551 # lock below must already read the loop index to find the dev
1552 # wheels, before the loop's own fm can run them. The pass's own
1553 # workshop renders it, the code the dev wheels are built from, as
1554 # it rendered the birth. From nothing: an edit an earlier pass
1555 # committed would be a local override the render keeps and drift
1556 # refuses.
1557 (root / "pyproject.toml").unlink(missing_ok=True)
1558 _render_with_the_pass(root)
1559 # Lock first, render second: the loop's fm syncs its venv from
1560 # the lock, so the render runs the workshop this pass published,
1561 # and the workflows it emits are the emitter under test. The
1562 # render may rewrite pyproject (the registry wiring, the roster),
1563 # so the lock settles on the same pins once more afterwards; an
1564 # unchanged pyproject makes that a no-op.
1565 _lock_pins(root, pins)
1566 _loop_fm(root, "drift.check", "--fix")
1567 # The tool lock follows the contract as the wiring left it: an
1568 # allowance above joins its tool's entry, so a lock the birth wrote
1569 # before it moves, and CI's setup (`sync --locked`) refuses a lock
1570 # that would move. The push below skips the lock check `fm submit`
1571 # makes, so the wiring writes the lock itself.
1572 _loop_fm(root, "toolroom.lock")
1573 _lock_pins(root, pins)
1574 # Cleanliness is the truth, not this run's edits: a resumed
1575 # half-wired workspace still commits and pushes here.
1576 if git.is_clean(root):1576 ↛ 1577line 1576 didn't jump to line 1577 because the condition on line 1576 was never true
1577 print(" dev wheels: already wired")
1578 else:
1579 git.commit_all(
1580 root,
1581 "chore: the loop eats the dev wheels\n\nThe registry index"
1582 " joins uv's config on the compose hostname, true on both"
1583 " sides, and the lock pins the workshop's own dev wheels,"
1584 " so the installed workshop matches the emitter that"
1585 " rendered these workflows.",
1586 )
1587 # Force: the branch is rebuilt from main each pass, so the
1588 # remote's copy is always superseded, like everything scratch.
1589 # The head this push supersedes keeps its runs moving with
1590 # nothing left to read them: they are cancelled after the push.
1591 previous = ""
1592 try:
1593 previous = git.remote_head(root, _SETUP_BRANCH)
1594 except Exception: # no such branch on origin yet
1595 previous = ""
1596 git.push_force(root, _SETUP_BRANCH)
1597 if previous and previous != git.head_sha(root):1597 ↛ 1598line 1597 didn't jump to line 1598 because the condition on line 1597 was never true
1598 from livery.workshop._ci_tasks import cancel_superseded_runs
1599
1600 forge_lane, _ = _dev_forge(kind)
1601 for line in cancel_superseded_runs(
1602 forge_lane.repository(E2E_OWNER, E2E_REPO), previous
1603 ):
1604 print(line)
1605 print(" dev wheels: wired and pushed")
1606 return git.head_sha(root)
1607
1608
1609def _runs_followed(
1610 repo: Repository,
1611 read: Callable[[tuple[Run, ...]], _A | Pending],
1612 *,
1613 head_sha: str = "",
1614 subject: str,
1615 timeout: float,
1616 interval: float,
1617 timed_out: Callable[[Pending], str],
1618) -> _A:
1619 """Poll the runs (of *head_sha*, or all) until *read* answers from them.
1620
1621 A loaded local forge answers a runs listing in minutes, and the
1622 client's timeout is shorter: an unreadable poll is retried as the
1623 ``ci`` verbs retry theirs, and only a spent budget or the deadline
1624 is the verdict, each failing with its words. *read* answers a
1625 pending whose key says what it still waits for; at the deadline
1626 *timed_out* turns the last one into the failure's words.
1627 """
1628 try:
1629 return follow(
1630 repo,
1631 lambda: read(repo.checks.runs(head_sha=head_sha)),
1632 subject=subject,
1633 interval=interval,
1634 timeout=timeout,
1635 quiet=None,
1636 )
1637 except ForgeUnreachable as exc:
1638 fail(exc.line.strip())
1639 except FollowTimedOut as exc:
1640 fail(timed_out(exc.last))
1641
1642
1643def _watch_latest(kind: str, workflow: str, *, timeout: float = 900.0) -> None:
1644 """Follow the newest run of *workflow* to its verdict; red fails verbatim."""
1645 forge, _ = _dev_forge(kind)
1646 repo = forge.repository(E2E_OWNER, E2E_REPO)
1647
1648 def newest_completed(listed: tuple[Run, ...]) -> Run | Pending:
1649 runs = [run for run in listed if run.workflow.endswith(workflow)]
1650 latest = max(runs, key=lambda run: run.id, default=None)
1651 if latest is not None and latest.status == "completed":
1652 return latest
1653 return Pending(key="pending")
1654
1655 latest = _runs_followed(
1656 repo,
1657 newest_completed,
1658 subject=f"{workflow}'s runs",
1659 timeout=timeout,
1660 interval=5,
1661 timed_out=lambda _last: (
1662 f"{workflow} did not complete within {timeout:.0f}s:"
1663 f" {repo.web_url()}/actions"
1664 ),
1665 )
1666 if latest.conclusion != "success":
1667 fail(
1668 f"{workflow} run {latest.id} ended {latest.conclusion}:"
1669 f" {repo.web_url()}/actions/runs/{latest.id}"
1670 )
1671 print(f" {workflow:<14} {latest.conclusion}")
1672 RUNS.count += 1
1673
1674
1675def judge_runs(
1676 repo: Repository, runs: tuple[Run, ...], *, branch: str = ""
1677) -> list[str]:
1678 """One line per run; red fails with every red run named.
1679
1680 A failed run is red. A cancelled run is red too, unless a twin
1681 for the same head took over and completed green, in which case
1682 the twin's verdict stands and the line says so; a successor for a
1683 moved head means the watched sha is no longer the branch's and is
1684 named as red, since the loop watches a sha of its own. A
1685 cancellation nobody took over stays red rather than quietly
1686 passing.
1687 """
1688 from livery.workshop._runs import successor
1689
1690 lines: list[str] = []
1691 failed: list[str] = []
1692 for run in runs:
1693 verdict: str = run.conclusion or run.status
1694 if run.conclusion == "cancelled":
1695 found = successor(repo, run, branch=branch)
1696 if found is None:
1697 verdict = "cancelled; no newer run for its head is known"
1698 failed.append(run.workflow)
1699 elif found.same_head and found.run.conclusion == "success":
1700 verdict = (
1701 f"cancelled, superseded by run {found.run.id} for the same"
1702 " head (success)"
1703 )
1704 elif found.same_head:
1705 verdict = (
1706 f"cancelled, superseded by run {found.run.id} for the same head"
1707 f" ({found.run.conclusion or found.run.status})"
1708 )
1709 failed.append(run.workflow)
1710 else:
1711 verdict = (
1712 f"cancelled, superseded by run {found.run.id} for the moved head"
1713 f" {found.run.head_sha[:12]}; the watched sha is no longer"
1714 " the branch's"
1715 )
1716 failed.append(run.workflow)
1717 elif run.conclusion == "failure":
1718 failed.append(run.workflow)
1719 lines.append(f" {run.workflow:14} {verdict}")
1720 if failed:
1721 names = ", ".join(failed)
1722 fail(f"red runs on the loop: {names}; logs: {repo.web_url()}/actions")
1723 return lines
1724
1725
1726def _jobs_settled(repo: Repository, run: Run, *, timeout: float = 600.0) -> None:
1727 """Wait until every job of *run* has completed.
1728
1729 A job still moving past *timeout* seconds fails, named.
1730 """
1731
1732 def settled() -> bool | Pending:
1733 moving = [
1734 job.name for job in repo.checks.jobs(run.id) if job.status != "completed"
1735 ]
1736 if not moving:1736 ↛ 1738line 1736 didn't jump to line 1738 because the condition on line 1736 was always true
1737 return True
1738 return Pending(
1739 f" waiting for {moving[0]} of run {run.id} before its re-run",
1740 key=", ".join(moving),
1741 )
1742
1743 try:
1744 follow(
1745 repo,
1746 settled,
1747 subject=f"the jobs of run {run.id}",
1748 interval=5,
1749 timeout=timeout,
1750 quiet=None,
1751 )
1752 except ForgeUnreachable as exc:
1753 fail(exc.line.strip())
1754 except FollowTimedOut as exc:
1755 fail(
1756 f"run {run.id} ended failure but {exc.last.key} still"
1757 f" moved {timeout:.0f}s later; the re-run would race it"
1758 )
1759
1760
1761def _retry_red_once(repo: Repository, runs: tuple[Run, ...]) -> list[str]:
1762 """Re-run the failed jobs of every red run in *runs*; the runs re-queued.
1763
1764 The loop's runner has failed a job on a build that left nothing
1765 and passed the same job on the re-run, so a red run gets one more
1766 attempt before it is judged. Returns the workflows re-run, empty
1767 when nothing was red.
1768 """
1769 retried: list[str] = []
1770 for run in runs:
1771 if run.conclusion != "failure":
1772 continue
1773 # A job of the old attempt still running would collect the new
1774 # attempt's halves: the re-run waits until every job has
1775 # completed, naming the one it waits on.
1776 _jobs_settled(repo, run)
1777 # The re-run keeps the latest attempt's logs alone, so the first
1778 # attempt's failure is said here or nowhere.
1779 print(red_jobs_text(repo, run))
1780 repo.checks.rerun(run.id, failed_only=True)
1781 print(f" re-running {run.workflow} (run {run.id}) once: it ended failure")
1782 retried.append(run.workflow)
1783 return retried
1784
1785
1786def _watch(
1787 kind: str,
1788 url: str,
1789 sha: str,
1790 *,
1791 timeout: float = 900.0,
1792 require: tuple[str, ...] = (),
1793 branch: str = "",
1794 interval: float = 5.0,
1795) -> None:
1796 """Follow *sha*'s runs to their verdicts; red fails verbatim after one re-run.
1797
1798 The runs judged are the ones this pass caused: every run the
1799 commit's own events started, and the dispatched runs *require*
1800 names. A run dispatched at the same commit by something else (the
1801 merge point re-dispatching a wave that already failed there) is
1802 named once and left to the scenario that owns it, so a birth is
1803 never judged by a wave it did not cause. *require* names
1804 workflows that must appear before the wait ends: a workflow
1805 triggered by the push registers its run a beat after the others,
1806 and a watch that settles on the early arrivals would call the
1807 commit green while the required one is still unstarted. A run
1808 that ended failure is re-run once and followed again; red twice
1809 is the verdict.
1810 """
1811 forge, _ = _dev_forge(kind)
1812 repo = forge.repository(E2E_OWNER, E2E_REPO)
1813 retried = False
1814 foreign: set[int] = set()
1815
1816 def own(runs: tuple[Run, ...]) -> tuple[Run, ...]:
1817 mine: list[Run] = []
1818 for run in runs:
1819 if run.event != "workflow_dispatch" or run.workflow in require:
1820 mine.append(run)
1821 elif run.id not in foreign:1821 ↛ 1818line 1821 didn't jump to line 1818 because the condition on line 1821 was always true
1822 foreign.add(run.id)
1823 print(
1824 f" {run.workflow} run {run.id} is not this pass's: dispatched"
1825 f" at {sha[:10]} by something else; left to its own scenario"
1826 )
1827 return tuple(mine)
1828
1829 def all_completed(runs: tuple[Run, ...]) -> tuple[Run, ...] | Pending:
1830 nonlocal retried
1831 mine = own(runs)
1832 present = {run.workflow for run in mine}
1833 if (1833 ↛ 1841line 1833 didn't jump to line 1841 because the condition on line 1833 was always true
1834 mine
1835 and all(r.status == "completed" for r in mine)
1836 and all(name in present for name in require)
1837 ):
1838 if retried or not _retry_red_once(repo, mine):
1839 return mine
1840 retried = True
1841 missing = ", ".join(sorted(set(require) - present))
1842 return Pending(key=missing)
1843
1844 def never_completed(last: Pending) -> str:
1845 waited = f"; never appeared: {last.key}" if last.key else ""
1846 return (
1847 f"the loop's runs did not complete within {timeout:.0f}s"
1848 f"{waited}: {repo.web_url()}/actions"
1849 )
1850
1851 runs = _runs_followed(
1852 repo,
1853 all_completed,
1854 head_sha=sha,
1855 subject=f"{sha[:10]}'s runs",
1856 timeout=timeout,
1857 interval=interval,
1858 timed_out=never_completed,
1859 )
1860 RUNS.count += len(runs)
1861 for line in judge_runs(repo, runs, branch=branch):
1862 print(line)
1863
1864
1865def _align_main(root: Path) -> None:
1866 """Align the loop's main hard onto origin, superseded commits named.
1867
1868 Origin is the loop's truth: every local byte is regenerable
1869 render output, and local main gathers aftercare commits the
1870 squashes supersede, so a fast-forward regularly cannot. What
1871 goes is printed, never silently vanished.
1872 """
1873 git.fetch(root)
1874 with contextlib.suppress(git.GitError):
1875 for line in git.oneline(root, "origin/main..main"):1875 ↛ 1876line 1875 didn't jump to line 1876 because the loop on line 1875 never started
1876 print(f" superseded local commit: {line}")
1877 git.switch(root, "main")
1878 git.reset_hard(root, "origin/main")
1879 # Residue too: a failed pass's branch leaves untracked leftovers
1880 # (a member directory without its contract refuses discovery).
1881 # What `fm sync` materialises survives: the venv, the stubs under
1882 # typings/ and the receipts and fragments under .workshop/, which
1883 # the gate reads and no verb between two syncs rewrites (a member
1884 # is wired with `uv sync` alone); everything else is regenerable
1885 # by charter.
1886 residue = git.clean(root, keep=KEPT_BY_SYNC, dry_run=True)
1887 if residue:1887 ↛ 1895line 1887 didn't jump to line 1895 because the condition on line 1887 was always true
1888 for line in residue:
1889 print(f" cleaned: {line}")
1890 git.clean(root, keep=KEPT_BY_SYNC)
1891 # Local branches too: main is the loop's only durable ref, and
1892 # every other local branch is a past pass's residue (a prepared
1893 # release branch whose PR already merged makes the driver refuse
1894 # as stale). What goes is printed, never silently vanished.
1895 for name in git.local_branches(root, ""):1895 ↛ 1896line 1895 didn't jump to line 1896 because the loop on line 1895 never started
1896 if name == "main":
1897 continue
1898 with contextlib.suppress(git.GitError):
1899 for line in git.oneline(root, f"main..{name}"):
1900 print(f" superseded branch commit: {line} ({name})")
1901 with contextlib.suppress(git.GitError):
1902 git.delete_branch(root, name)
1903 print(f" pruned pass-owned branch: {name}")
1904
1905
1906def _main_settled(root: Path, kind: str) -> None:
1907 """Wait until main's push run at the aligned tip has completed.
1908
1909 The loop cuts its next branch right after a merge, and the one
1910 runner may run the new pull request's check job before main's push
1911 job has written main's record; a leg then reads a record that holds
1912 nothing and measures every suite afresh. The push run is waited for
1913 here, so what a branch's leg reuses is on the record before the leg
1914 reads it. A red push run fails the pass as any red run does.
1915 """
1916 forge, _ = _dev_forge(kind)
1917 repo = forge.repository(E2E_OWNER, E2E_REPO)
1918 _completed_run(repo, git.head_sha(root), event="push")
1919
1920
1921def _fresh_branch(root: Path, name: str) -> None:
1922 """A pass-owned branch, recreated from an aligned main.
1923
1924 ``switch -C`` from a dirty tree drags or clobbers state, so the
1925 alignment and the switch are one operation here, never two calls
1926 a future edit can separate: the tree is reset to origin, cleaned,
1927 and swept of stale local branches before this one is recreated.
1928 """
1929 _align_main(root)
1930 git.switch_new(root, name)
1931
1932
1933def _render_with_the_pass(root: Path) -> None:
1934 """Render the loop's files at *root* with this pass's own workshop.
1935
1936 A child of the pass's interpreter, as the birth is, with the uv
1937 handoff off: the loop's lock pins the workshop the loop last
1938 locked, and the handoff would run that one instead. The caller has
1939 deleted the loop's ``pyproject.toml`` so the render starts from
1940 nothing, and footman's project rung reads that file for the
1941 built-ins a project's dependencies offer, so here it offers none.
1942 The child mounts the workshop as a user built-in instead, through
1943 a config directory of its own: what a desk's global fm carries in
1944 its user config and a bare runner lacks. A failure also prints
1945 footman's ``--plugins`` listing from the same child: what mounted,
1946 and from which rung.
1947 """
1948 import sys
1949 import tempfile
1950
1951 runner = [sys.executable, "-m", "livery.footman"]
1952 with tempfile.TemporaryDirectory(prefix="loop-render-") as config:
1953 (Path(config) / "config.toml").write_text(
1954 "# The pass's own workshop, mounted for the render alone.\n"
1955 '[builtins]\nuser = ["livery.workshop"]\n',
1956 "utf-8",
1957 )
1958 env = child_environment(FOOTMAN_NO_UV="1", FOOTMAN_CONFIG_DIR=config)
1959 result = footman.run(
1960 [*runner, "--yes", "drift.check", "--fix"],
1961 cwd=root,
1962 env=env,
1963 nofail=True,
1964 timeout=900.0,
1965 )
1966 print(result.stdout.rstrip("\n"))
1967 if result.code != 0:
1968 plugins = footman.run(
1969 [*runner, "--plugins"], cwd=root, env=env, nofail=True, timeout=120.0
1970 )
1971 fail(
1972 f"the pass's render of the loop exited {result.code}:"
1973 f"\n{result.stdout}{result.stderr}"
1974 f"\nfootman's plugins in the loop:\n{plugins.stdout}{plugins.stderr}"
1975 )
1976
1977
1978def _loop_fm(
1979 root: Path, *args: str, timeout: float = 900.0, nofail: bool = False
1980) -> int:
1981 """Run the loop's own fm, the dev wheels' one, inside the loop.
1982
1983 The whole point of the substrate: the workspace under test runs
1984 the workshop being edited, so submit, release, and every other
1985 verb exercise the dev wheels end to end. ``--yes`` rides every
1986 call, because the loop is automation and silence never confirms.
1987 """
1988 import livery.toolroom.tools as toolroom
1989
1990 # The caller's VIRTUAL_ENV points at the worktree; the loop's uv
1991 # must resolve the loop's own venv, so the variable stays behind.
1992 env = {k: v for k, v in os.environ.items() if k != "VIRTUAL_ENV"}
1993 # Named here, not only in the pass's environment: a write to
1994 # os.environ inside a task is the task's, and the environment this
1995 # child gets is built from the process's, so the loop's own fm
1996 # (whose submit can amend a commit) would ask a developer's signer.
1997 # No --no-sync: the wiring moves the lock onto each pass's fresh
1998 # dev wheels, and a frozen venv would keep running yesterday's
1999 # workshop under today's lock (measured: a fixed bug stayed red
2000 # in the loop while the fix sat published in the registry).
2001 result = toolroom.uv.opts(cwd=root, nofail=True, timeout=timeout, env=env)(
2002 "run", "fm", "--yes", *args
2003 )
2004 if result.code != 0:
2005 if nofail:
2006 # The evidence prints either way; the caller owns the
2007 # verdict for one bounded retry, never a silent swallow.
2008 print(
2009 f" the loop's `{footman.prog()} {' '.join(args)}` exited"
2010 f" {result.code}:\n{result.stdout}{result.stderr}"
2011 )
2012 return result.code
2013 fail(
2014 f"the loop's `{footman.prog()} {' '.join(args)}` exited"
2015 f" {result.code}:\n{result.stdout}{result.stderr}"
2016 )
2017 return 0
2018
2019
2020def members_for(names: Sequence[str] = ()) -> tuple[LoopMember, ...]:
2021 """The loop's members named *names*, in the loop's order; all when empty."""
2022 return tuple(member for member in LOOP_MEMBERS if not names or member.name in names)
2023
2024
2025def landed_members(named: Iterable[str]) -> tuple[str, ...]:
2026 """The loop's members a pass lands for steps naming *named*, by name, in order.
2027
2028 A member lands for an extension a step names that its set holds and
2029 that no member whose set lies within its own holds too: the smaller
2030 member stands for that extension. loop-native's set holds
2031 loop-echo's, so a step naming python lands loop-echo alone, and one
2032 naming conan lands loop-native and loop-cpp. A pass carries the
2033 members its steps were written for, and no slower one besides.
2034 """
2035 wanted = set(named)
2036 sets = {member.name: frozenset(member.holds()) for member in LOOP_MEMBERS}
2037 landed: list[str] = []
2038 for member in LOOP_MEMBERS:
2039 held = sets[member.name]
2040 own = set(held)
2041 for other, smaller in sets.items():
2042 if other != member.name and smaller < held:
2043 own -= smaller
2044 if own & wanted:
2045 landed.append(member.name)
2046 return tuple(landed)
2047
2048
2049def steps_declared_off(name: str) -> tuple[str, ...]:
2050 """The steps extension *name* declares ``enabled = false``, by ``<phase>.<step>``.
2051
2052 An extension pass turns each on in the newborn's root contract, so
2053 every step of the extension under test registers and runs.
2054 """
2055 from livery.workshop._extensions import declaration
2056
2057 found = declaration(name)
2058 if found is None:
2059 return ()
2060 return tuple(record.name for record in found.additions.steps if not record.enabled)
2061
2062
2063def extension_under_test(name: str) -> tuple[tuple[str, ...], tuple[str, ...]]:
2064 """The list a pass births to test extension *name*, and the members it lands.
2065
2066 The members are the loop's whose sets hold an extension *name*'s
2067 steps name, by name (`landed_members`), so each step has something
2068 of its own to judge. The list is *name* and what those members'
2069 templates need (`LoopMember.needs`), as their canonical list: a
2070 creation installs nothing, so the root lists them before the
2071 members are created, and the set takes in what each requires. The
2072 steps *name* declares off are turned on in the newborn's contract
2073 (`steps_declared_off`), so each of its steps registers.
2074
2075 Raises:
2076 Failed: when no installed distribution declares *name*, naming
2077 the installed ones; when *name* registers no step.
2078 """
2079 from livery.workshop._extensions import declaration, installed_extensions
2080 from livery.workshop._new_project import canonical_extensions
2081
2082 found = declaration(name)
2083 if found is None:
2084 installed = ", ".join(installed_extensions()) or "none"
2085 fail(
2086 f"extension {name!r}: no installed distribution declares it;"
2087 f" the installed ones are {installed}"
2088 )
2089 steps = found.additions.steps
2090 if not steps:
2091 fail(
2092 f"extension {name!r} registers no step: a pass would have"
2093 " nothing of its own to run"
2094 )
2095 named = {extension for record in steps for extension in record.extensions}
2096 members = landed_members(named)
2097 needs = [needed for member in members_for(members) for needed in member.needs()]
2098 return tuple(canonical_extensions([name, *needs])), members
2099
2100
2101def _ensure_members(root: Path, members: Sequence[str] = ()) -> None:
2102 """Give the loop its *members*, by name, all when empty, in one pull request.
2103
2104 Its template's verb, ``new.package.<template>``, renders and wires
2105 each missing member, a `[release] baseline` seeds the first
2106 release's number, the nanobind member's
2107 ``[nanobind] wheel-platforms`` names the loop's one runner, and the
2108 loop's own ``fm submit --armed`` lands them together: one branch,
2109 one pull request, one gate on the runner over every kind at once
2110 (it compiles the native member's editable install), and one merge,
2111 all on the dev wheels. One pull request rather than one per member:
2112 every member edits the same rendered root files, so pull requests
2113 in flight together conflict, and landing them one after another
2114 costs a gate run and a merge each; what the one gate drops is each
2115 kind's narrowed gate and its own submit. Idempotent: a member
2116 already on main is skipped, and nothing is submitted when none is
2117 missing.
2118 """
2119 # Landed means on main: a failed pass leaves the working tree on
2120 # the feature branch with the member present, and judging that
2121 # tree would skip straight to the release act with nothing
2122 # merged. The alignment makes the glob read main's truth.
2123 _align_main(root)
2124 missing: list[tuple[str, str]] = []
2125 for wanted in members_for(members):
2126 if (root / "packages" / wanted.name / "workshop.toml").is_file():
2127 print(f" member {wanted.name}: already landed")
2128 else:
2129 missing.append((wanted.name, wanted.template))
2130 if not missing:
2131 return
2132 _fresh_branch(root, "feat/members")
2133 for name, template in missing:
2134 _loop_fm(root, f"new.package.{template}", name)
2135 member = root / "packages" / name / "workshop.toml"
2136 body = member.read_text("utf-8")
2137 # The nanobind template names the hosted runners under
2138 # [nanobind] wheel-platforms; the loop's fleet is its one runner,
2139 # and the leg must be schedulable. Another template names none,
2140 # and nothing changes.
2141 body = body.replace(
2142 'wheel-platforms = ["ubuntu-latest", "macos-latest", "windows-latest"]',
2143 f'wheel-platforms = ["{CURRENT.label}"]',
2144 )
2145 if "[release]" not in body:2145 ↛ 2151line 2145 didn't jump to line 2151 because the condition on line 2145 was always true
2146 body = (
2147 body.rstrip("\n")
2148 + "\n\n[release]\n# The first release lands at the baseline.\n"
2149 + 'baseline = "0.1.0"\n'
2150 )
2151 if name == CONTRIBUTING_MEMBER and "[[ci.point]]" not in body:2151 ↛ 2155line 2151 didn't jump to line 2155 because the condition on line 2151 was never true
2152 # The member contributes a point: the loop dispatches it
2153 # by hand, so the point is proven against the real runner
2154 # rather than against the renderer's own output.
2155 body = (
2156 body.rstrip("\n")
2157 + "\n\n# A point of this member's own, run by hand in the loop.\n"
2158 + "[[ci.point]]\n"
2159 + f'name = "{CONTRIBUTED_POINT}"\n'
2160 + 'task = "extensions"\n'
2161 # The environment's runner, as the root's runners name
2162 # it: the hosted label on the docker rig, the host
2163 # runner's own label in host mode.
2164 + f'runners = ["{CURRENT.label}"]\n'
2165 )
2166 member.write_text(body, "utf-8")
2167 # The baseline is a render input: cliff.toml was rendered before
2168 # the append, so it must settle again or the gate names it as
2169 # drift.
2170 _loop_fm(root, "drift.check", "--fix")
2171 names = ", ".join(name for name, _template in missing)
2172 git.commit_all(
2173 root,
2174 f"feat: the loop's members\n\n{names}: born through their templates' verbs on"
2175 " the dev wheels, with the release baseline seeded so the"
2176 " first release lands at 0.1.0.",
2177 )
2178 # Force for the same reason the setup branch pushes force: the
2179 # branch is pass-owned, rebuilt from main every time, so the
2180 # remote's copy is always superseded.
2181 _loop_fm(root, "submit", "--force", "--armed")
2182 _align_main(root)
2183 print(f" members {names}: landed through the loop's own gate, one pull request")
2184
2185
2186def red_jobs_text(repo: Repository, run: Run, *, lines: int = 60) -> str:
2187 """The failed jobs of *run* with the last *lines* of each log, as text to print.
2188
2189 A run that went red on the loop's forge is readable at its page on
2190 a desk; inside a hosted job the forge vanishes with the job, so the
2191 pass prints what the page would show. Every job that ended in
2192 failure is named with its log's tail; a run whose jobs all passed
2193 or never ran gives the header line alone.
2194 """
2195 out = [f" run {run.id} ended {run.conclusion}: {run.url or repo.web_url()}"]
2196 for job in repo.checks.jobs(run.id):
2197 if job.conclusion != "failure":
2198 continue
2199 tail = repo.checks.job_log(job.id).rstrip("\n").splitlines()[-lines:]
2200 out.append(f" {job.name}: failure; the last {len(tail)} line(s) of its log:")
2201 out.extend(f" {line}" for line in tail)
2202 return "\n".join(out)
2203
2204
2205def _completed_run(
2206 repo: Repository,
2207 sha: str,
2208 *,
2209 event: str,
2210 timeout: float = 900.0,
2211 interval: float = 5.0,
2212) -> tuple[Run, tuple[Job, ...]]:
2213 """The newest completed ``ci.yml`` run of *sha* for *event*, with its jobs.
2214
2215 Waits for the run to register and complete; a run that ended
2216 failure is re-run once and waited for again, and a red run then
2217 fails verbatim, naming its page. The jobs carry their names, the
2218 check legs by their matrix display (``check (ubuntu-latest,
2219 3.14)``), so a reader looks for the prefix it needs; the logs are
2220 read per job on demand, since a job the run skipped has none to
2221 serve.
2222 """
2223 retried = False
2224
2225 def newest_completed(listed: tuple[Run, ...]) -> Run | Pending:
2226 nonlocal retried
2227 runs = [
2228 run
2229 for run in listed
2230 if run.workflow.endswith("ci.yml") and run.event == event
2231 ]
2232 run = max(runs, key=lambda run: run.id, default=None)
2233 if run is not None and run.status == "completed":
2234 if retried or not _retry_red_once(repo, (run,)):
2235 return run
2236 retried = True
2237 return Pending(key="pending")
2238
2239 run = _runs_followed(
2240 repo,
2241 newest_completed,
2242 head_sha=sha,
2243 subject=f"{sha[:10]}'s {event} run",
2244 timeout=timeout,
2245 interval=interval,
2246 timed_out=lambda _last: (
2247 f"no completed {event} run for {sha[:10]} within {timeout:.0f}s:"
2248 f" {repo.web_url()}/actions"
2249 ),
2250 )
2251 RUNS.count += 1
2252 if run.conclusion != "success":
2253 print(red_jobs_text(repo, run))
2254 fail(
2255 f"run {run.id} ({event}, {sha[:10]}) ended {run.conclusion}:"
2256 f" {repo.web_url()}/actions/runs/{run.id}"
2257 )
2258 return run, repo.checks.jobs(run.id)
2259
2260
2261def _log_says(
2262 repo: Repository, run: Run, jobs: tuple[Job, ...], job: str, line: str
2263) -> None:
2264 """Fail unless the log of the job whose name starts with *job* carries *line*.
2265
2266 The one read of a job's log the loop keeps: the publish job's word
2267 on whether it collected what the wheels job built has no row until
2268 conan's per-host artifact (phase 13 of the destination note), which
2269 the destination's temporary table names.
2270 """
2271 found = next((item for item in jobs if item.name.startswith(job)), None)
2272 if found is None:
2273 fail(f"run {run.id} has no {job} job: {repo.web_url()}/actions/runs/{run.id}")
2274 if line not in repo.checks.job_log(found.id):
2275 fail(
2276 f"run {run.id}'s {job} job did not say {line!r};"
2277 f" {repo.web_url()}/actions/runs/{run.id}"
2278 )
2279
2280
2281def _require_task(facts: RunFacts, prefix: str, task: str) -> None:
2282 """Fail unless a job of *facts* named by *prefix* ran *task*, by its row.
2283
2284 A matrix job is several forge jobs under one name (``nightly
2285 (3.14)``), so any one of them running the task is the proof.
2286 """
2287 found = [job for name, job in facts.jobs.items() if name.startswith(prefix)]
2288 if not found:
2289 fail(
2290 f"run {facts.run}: the row names no job starting with {prefix!r};"
2291 f" its jobs are {', '.join(sorted(facts.jobs)) or 'none'}"
2292 )
2293 if not any(task in job.tasks for job in found):
2294 ran = sorted({t for job in found for t in job.tasks})
2295 fail(
2296 f"run {facts.run}: no {prefix} job ran {task}; the row names"
2297 f" {', '.join(ran) or 'no task'}"
2298 )
2299
2300
2301def _require_scope(
2302 run: Run,
2303 facts: RunFacts,
2304 *,
2305 scope: str,
2306 packages: Collection[str] = (),
2307 composed: bool | None = None,
2308) -> None:
2309 """Fail unless *run*'s check leg recorded *scope* over *packages*.
2310
2311 *composed* says whether the leg must name the tree it measured from
2312 (a narrowed gate) or must not; None leaves it unjudged.
2313 """
2314 leg = facts.job("check")
2315 if leg.scope != scope or set(leg.packages) != set(packages):
2316 fail(
2317 f"run {run.id}: the check leg's row says {leg.scope} over"
2318 f" {sorted(leg.packages) or 'no package'}, expected {scope} over"
2319 f" {sorted(packages) or 'no package'}; {run.url}"
2320 )
2321 if composed is not None and bool(leg.base_tree) != composed:
2322 named = (
2323 f"names base tree {leg.base_tree[:12]}"
2324 if leg.base_tree
2325 else "names no base tree"
2326 )
2327 fail(
2328 f"run {run.id}: the check leg's row {named}, expected"
2329 f" {'one' if composed else 'none'}; {run.url}"
2330 )
2331
2332
2333def _require_judged(run: Run, facts: RunFacts, percents: Mapping[str, float]) -> None:
2334 """Fail unless the gate job's coverage section judged exactly *percents*."""
2335 if facts.coverage != dict(percents):
2336 fail(
2337 f"run {run.id}: the gate job judged {facts.coverage}, expected"
2338 f" {dict(percents)}; {run.url}"
2339 )
2340
2341
2342def _require_record(
2343 root: Path,
2344 run: Run,
2345 base: str,
2346 leg: str,
2347 *,
2348 fresh: Collection[str],
2349 carried: Collection[str],
2350) -> None:
2351 """Fail unless *base*'s record on *leg* holds the units as measured.
2352
2353 *fresh* were measured by *run*; *carried* by another run.
2354 """
2355 units = record_units(root, base, leg)
2356 expected = set(fresh) | set(carried)
2357 if set(units) != expected:
2358 fail(
2359 f"run {run.id}: {base}'s record on {leg} holds {sorted(units)},"
2360 f" expected {sorted(expected)}"
2361 )
2362 stale = [unit for unit in fresh if units[unit].run != str(run.id)]
2363 remeasured = [unit for unit in carried if units[unit].run == str(run.id)]
2364 if stale or remeasured:
2365 fail(
2366 f"run {run.id}: {base}'s record on {leg} was to measure {sorted(fresh)}"
2367 f" afresh and carry {sorted(carried)}; measured by another run:"
2368 f" {stale}; measured again: {remeasured}"
2369 )
2370
2371
2372def _require_verified(
2373 root: Path, run: Run, *, composed: bool, branch: str = ""
2374) -> None:
2375 """Fail unless *run* stamped its tree as asked: composed or not, naming *branch*."""
2376 entry = verified_by(root, run.id)
2377 if bool(entry.base_tree) != composed:
2378 rests = "rests on no base"
2379 if entry.base_tree:
2380 rests = f"rests on {entry.base_tree[:12]}"
2381 fail(
2382 f"run {run.id}: the verified row of tree {entry.tree[:12]} {rests},"
2383 f" expected {'a composed row' if composed else 'a full row'}"
2384 )
2385 if branch and entry.branch != branch:
2386 fail(
2387 f"run {run.id}: the verified row of tree {entry.tree[:12]} names branch"
2388 f" {entry.branch!r}, expected {branch!r}"
2389 )
2390
2391
2392def _require_mark(
2393 root: Path, package: str, *, value: float, kind: str, by: str = ""
2394) -> None:
2395 """Fail unless *package*'s mark is *value* of *kind*, and set by *by* when given."""
2396 current = marks(root).get(package)
2397 if current is None:
2398 fail(f"no mark on the store for {package}")
2399 if current.value != value or current.kind != kind or (by and current.by != by):
2400 fail(
2401 f"{package}'s mark is {current.value} ({current.kind} by {current.by}),"
2402 f" expected {value} ({kind}{' by ' + by if by else ''})"
2403 )
2404
2405
2406def _require_shards(run: Run, facts: RunFacts, *, count: int, ran: bool) -> None:
2407 """Fail unless *run*'s check leg is the fold of *count* shards, each with its row.
2408
2409 The fold's row lists each shard with its job, and the forge's row of
2410 that job stands beside it. *ran* says whether the leg ran its gate:
2411 then the shards partitioned a set and ran its items; otherwise
2412 every shard skipped, so the fold names no item ran and no
2413 durations row read.
2414 """
2415 leg = facts.job("check")
2416 fold = leg.fold
2417 if fold is None:
2418 fail(
2419 f"run {run.id}: the check leg's row ({leg.leg}) is not a fold; the"
2420 f" loop's contract splits the leg into {count} shards; {run.url}"
2421 )
2422 labels = ", ".join(shard.leg for shard in fold.shards)
2423 if fold.count != count or len(fold.shards) != count:
2424 fail(
2425 f"run {run.id}: the check leg's row folds {fold.count} shard(s) and"
2426 f" lists {len(fold.shards)} ({labels or 'none'}), expected {count};"
2427 f" {run.url}"
2428 )
2429 missing = [shard.job for shard in fold.shards if shard.job not in facts.shard_jobs]
2430 if missing:
2431 fail(
2432 f"run {run.id}: the forge's rows of {', '.join(missing)} are not beside"
2433 " the folded row; the collect keeps each shard's job with its times;"
2434 f" {run.url}"
2435 )
2436 items = sum(shard.items for shard in fold.shards)
2437 if ran and (not fold.digest or not items):
2438 fail(
2439 f"run {run.id}: the check leg ran and its shards ({labels}) ran no"
2440 f" item; the fold recorded no partition; {run.url}"
2441 )
2442 if not ran and (items or fold.rows):
2443 fail(
2444 f"run {run.id}: every shard skipped, and the fold records {items}"
2445 f" item(s) ran and {len(fold.rows)} durations row(s) read ({labels});"
2446 f" {run.url}"
2447 )
2448
2449
2450def _require_durations(root: Path, run: Run, leg: str, *, written: bool) -> None:
2451 """Fail unless *run*'s durations row on *leg*'s platform is there, or is not.
2452
2453 *written* says the gate job's collect wrote the row, naming at
2454 least one item; otherwise the leg ran no item and the series holds
2455 no row of the run.
2456 """
2457 row = durations_row(root, run.id, leg)
2458 if written and not row.items:
2459 fail(
2460 f"run {run.id}: no row {row.name} naming an item on {row.ref}; the"
2461 " gate job's collect wrote no seconds per item for the leg;"
2462 f" {run.url}"
2463 )
2464 if not written and row.items is not None:
2465 fail(
2466 f"run {run.id}: {row.ref} holds {row.name} with {len(row.items)}"
2467 " item(s), and every shard skipped; a leg that ran no item leaves no"
2468 f" durations; {run.url}"
2469 )
2470
2471
2472def _prove_shards(root: Path, run: Run, facts: RunFacts, *, ran: bool) -> None:
2473 """Prove *run*'s check leg ran as the loop's shards and folded into one leg.
2474
2475 The loop's contract splits its one runner's check leg into `SHARDS`
2476 shards, so every run's metrics row has the fold's row for the
2477 platform leg, with the forge's rows of the shards beside it. Where
2478 the leg ran its gate (*ran*), the shards partitioned the test files
2479 and the gate job's collect wrote the run's durations row on the
2480 platform leg's series; where every shard skipped, the fold names no
2481 item and no row is written.
2482 """
2483 _require_shards(run, facts, count=SHARDS, ran=ran)
2484 leg = facts.job("check").leg
2485 _require_durations(root, run, leg, written=ran)
2486 if ran:
2487 print(
2488 f" shards: run {run.id}'s check leg ran as {SHARDS} shards folded into"
2489 f" {leg}; its durations row is on the store"
2490 )
2491 else:
2492 print(
2493 f" shards: run {run.id}'s check leg skipped on every shard, folded into"
2494 f" {leg}; no durations row written"
2495 )
2496
2497
2498def _units(root: Path) -> tuple[list[str], bool]:
2499 """The newborn's units: its members' paths, and whether its own tests are one."""
2500 from livery.workshop._packages import discover_packages, workspace_suite
2501
2502 members = [package.path for package in discover_packages(root)]
2503 return members, workspace_suite(root) is not None
2504
2505
2506def _require_prebuilt(repo: Repository, run: Run, distribution: str) -> None:
2507 """Fail unless the wave *run* published *distribution* from the wheels job's wheels.
2508
2509 The wave publishes a platform-wheel member either way: from the
2510 wheels the wheels job built and handed over, or from a build of its
2511 own on the publish job's runner. The served version is the same
2512 after both. The publish job's ``prebuilt:`` line tells them apart:
2513 the wave prints it for each member a job builds before the wave
2514 whose collected ``dist/`` held what the job built, naming the job.
2515 A publish job that did not end green proves nothing, so it fails
2516 here too, naming how it ended.
2517 """
2518 jobs = repo.checks.jobs(run.id)
2519 publish = next((job for job in jobs if job.name.startswith("publish")), None)
2520 if publish is not None and publish.conclusion != "success":
2521 fail(
2522 f"run {run.id}'s publish job ended {publish.conclusion or publish.status};"
2523 f" {repo.web_url()}/actions/runs/{run.id}"
2524 )
2525 _log_says(
2526 repo,
2527 run,
2528 jobs,
2529 "publish",
2530 f"prebuilt: {distribution}: the run collected what the wheels job built",
2531 )
2532 print(
2533 f" release: the wave published the wheels the wheels job built for"
2534 f" {distribution} (run {run.id})"
2535 )
2536
2537
2538def _prove_verified_skip(root: Path, kind: str) -> None:
2539 """Prove main's run after the setup squash skips the gate and copies the record.
2540
2541 The setup pull request's check leg ran the full gate, its gate
2542 job wrote the setup branch's coverage record, and its stamp put
2543 the tree on the verified record naming the branch; the squash
2544 lands the same tree on main, so main's push run finds it on the
2545 record, skips the gate, measures nothing, and its union takes
2546 every unit from the setup branch's record and copies it into
2547 main's. The rows say so: the check leg's scope is ``verified``,
2548 the tree's verified row names the setup branch, main's record
2549 holds every unit measured by another run, and the gate job judged
2550 each member; the leg's shards both skipped, folded into one leg,
2551 and no durations row was written. A skipped leg reddening the
2552 union, passing it with nothing counted, or measuring what the
2553 branch already measured is the failure this proof exists for.
2554 """
2555 _align_main(root)
2556 head = git.head_sha(root)
2557 forge, _ = _dev_forge(kind)
2558 repo = forge.repository(E2E_OWNER, E2E_REPO)
2559 from livery.workshop._new_project import _SETUP_BRANCH
2560 from livery.workshop._verified import tree_id
2561
2562 run, _jobs = _completed_run(repo, head, event="push")
2563 facts = run_facts(root, run.id)
2564 _require_scope(run, facts, scope="verified")
2565 proved = verified_tree(root, tree_id(root, head))
2566 if proved.branch != _SETUP_BRANCH:
2567 fail(
2568 f"run {run.id}: main's tree is proved by a row naming {proved.branch!r},"
2569 f" expected {_SETUP_BRANCH!r}, the branch whose record main copies"
2570 )
2571 # The units main's union reuses are the ones the workspace has at
2572 # this point: its members and its own tests. A fresh birth has
2573 # neither yet (the members and the tests land through their own
2574 # pull requests afterwards), and then the union has no unit and
2575 # judges nothing.
2576 members, tests = _units(root)
2577 units = [*members, *(["tests"] if tests else [])]
2578 leg = facts.job("check").leg
2579 _require_record(root, run, "main", leg, fresh=(), carried=units)
2580 _require_judged(run, facts, dict.fromkeys(members, 100.0))
2581 _prove_shards(root, run, facts, ran=False)
2582 print(
2583 f" verified skip: proven on main's run {run.id}; the gate skipped,"
2584 f" nothing was measured, and main copied {_SETUP_BRANCH}'s record"
2585 )
2586
2587
2588#: The member the loop keeps under auto-ratchet, so every pass proves
2589#: the mark's first record, an accepted lowering, and the ratchet up.
2590RATCHET_MEMBER = "loop-native"
2591
2592
2593def _prepare_ratchet(root: Path, kind: str) -> None:
2594 """Put the ratchet member under auto-ratchet, then lower its mark for the proof.
2595
2596 The member's template names no floor, so it is judged at the default
2597 floor until its contract gains ``[coverage] floor = "auto-ratchet"``,
2598 which lands through the loop's own gate: that pull request's run
2599 records the first mark, and main's run after the squash judges it;
2600 the pass waits for that run, or the accept below would be consumed
2601 there instead of by the member-only pull request. Then
2602 `fm coverage.accept` lowers the mark to 90 with a reason, so the
2603 pull request that follows judges from the accepted row and
2604 ratchets the mark back up. A refusal (a mark a pass that did not
2605 finish already lowered) is printed and the mark stands; the proof
2606 reads the same lines either way.
2607 """
2608 contract = root / "packages" / RATCHET_MEMBER / "workshop.toml"
2609 body = contract.read_text("utf-8")
2610 if 'floor = "auto-ratchet"' not in body:
2611 if "[coverage]" in body:
2612 fail(
2613 f"{contract}: a [coverage] table the template does not write; the"
2614 " loop adds the member's own, with its floor the mark"
2615 )
2616 _fresh_branch(root, "chore/ratchet-member")
2617 contract.write_text(
2618 body.rstrip("\n")
2619 + "\n\n[coverage]\n# The floor is the mark on the store.\n"
2620 + 'floor = "auto-ratchet"\n',
2621 "utf-8",
2622 )
2623 git.commit_all(
2624 root,
2625 f"chore({RATCHET_MEMBER}): the member's floor is the mark on the"
2626 " store\n\nUnder auto-ratchet the first gated run records the"
2627 " mark, and the loop proves the record, an accepted lowering,"
2628 " and the ratchet up on every pass.",
2629 )
2630 with contextlib.suppress(git.GitError):
2631 git.fetch(root, prune=True)
2632 _loop_fm(root, "submit", "--force", "--armed")
2633 _align_main(root)
2634 forge, _ = _dev_forge(kind)
2635 repo = forge.repository(E2E_OWNER, E2E_REPO)
2636 run, _jobs = _completed_run(repo, git.head_sha(root), event="push")
2637 print(
2638 f" ratchet member: {RATCHET_MEMBER} landed under auto-ratchet;"
2639 f" main's run {run.id} judged the first mark"
2640 )
2641 code = _loop_fm(
2642 root,
2643 "coverage.accept",
2644 f"packages/{RATCHET_MEMBER}",
2645 "90",
2646 "--reason=the loop proves an accepted lowering",
2647 nofail=True,
2648 )
2649 if code == 0:
2650 print(f" ratchet member: {RATCHET_MEMBER}'s mark accepted down to 90")
2651 else:
2652 print(
2653 f" ratchet member: the accept was refused (exit {code}); the mark stands"
2654 )
2655
2656
2657def _prove_scoped_leg(root: Path, kind: str) -> None:
2658 """Prove the scoped check leg on a member-only pull request.
2659
2660 The loop's other pull requests all touch the root (a template's verb
2661 wires the workspace, the release stamps the manifest), so their
2662 legs pay the full gate by the affected rule. This one rewrites a
2663 test file inside ``loop-echo`` and nothing else, lands it through
2664 the loop's gate, and reads the rows: the check leg narrowed to
2665 that one member against main; the branch's record holds that
2666 member measured by this run and the others carried from main's;
2667 the gate job judged every member; the ratchet member's mark, which
2668 the pass accepted down to 90, is back at 100 by this run; the stamp
2669 composed with main's verified tree; and the leg's shards
2670 partitioned the member's test files, folded into one leg, and the
2671 run's durations row is on the store. Main's push run after the
2672 squash then skips the gate on that composed row, measures nothing,
2673 and copies the branch's record into main's. Re-run on a pass-owned
2674 branch with main's tip as the stamp, so the diff is never empty.
2675 """
2676 _fresh_branch(root, "chore/scoped-leg")
2677 _main_settled(root, kind)
2678 stamp = git.head_sha(root)
2679 probe = root / "packages" / "loop-echo" / "tests" / "test_scoped_leg.py"
2680 probe.write_text(
2681 '"""A member-only change: the loop proves the scoped check leg on it."""\n'
2682 "\n"
2683 f'STAMP = "{stamp}"\n'
2684 "\n\n"
2685 "def test_the_stamp_is_a_commit():\n"
2686 " assert len(STAMP) == 40\n",
2687 "utf-8",
2688 )
2689 git.commit_all(
2690 root,
2691 "chore(loop-echo): a member-only change for the scoped leg\n\nOne file"
2692 " under one member, so the check leg narrows to that member and"
2693 " says so.",
2694 )
2695 head = git.head_sha(root)
2696 # The last pass's merge deleted the branch on origin, and the clone's
2697 # tracking ref still names its final commit; the forced submit pushes
2698 # with a lease on that ref, which git refuses as stale against a
2699 # branch that is gone. A pruning fetch drops the stale ref (or
2700 # refreshes it when a failed pass left the branch behind).
2701 with contextlib.suppress(git.GitError):
2702 git.fetch(root, prune=True)
2703 _loop_fm(root, "submit", "--force", "--armed")
2704 _align_main(root)
2705 # The workspace's own tests are a unit once the tests leg landed
2706 # them, and the member's change moves their closure, so the leg
2707 # runs them too; a fresh birth has none.
2708 members, tests = _units(root)
2709 others = [path for path in members if path != "packages/loop-echo"]
2710 fresh = ["packages/loop-echo", *(["tests"] if tests else [])]
2711 judged = dict.fromkeys(members, 100.0)
2712 forge, _ = _dev_forge(kind)
2713 repo = forge.repository(E2E_OWNER, E2E_REPO)
2714 run, _jobs = _completed_run(repo, head, event="pull_request")
2715 facts = run_facts(root, run.id)
2716 _require_scope(run, facts, scope="affected", packages=fresh, composed=True)
2717 leg = facts.job("check").leg
2718 _require_record(root, run, "chore/scoped-leg", leg, fresh=fresh, carried=others)
2719 _require_judged(run, facts, judged)
2720 _require_mark(
2721 root,
2722 f"packages/{RATCHET_MEMBER}",
2723 value=100.0,
2724 kind="ratchet",
2725 by=f"run {run.id}",
2726 )
2727 _require_verified(root, run, composed=True, branch="chore/scoped-leg")
2728 _prove_shards(root, run, facts, ran=True)
2729 print(
2730 " scoped leg: proven on a member-only pull request (affected:"
2731 " packages/loop-echo; the union reused the other members from main's"
2732 " record and wrote the branch's; the mark ratcheted; the stamp"
2733 " composed with main's tree)"
2734 )
2735 # The narrowed run rested on main's verified tree, so its stamp
2736 # composed naming the branch, and main's push after the squash
2737 # skips the gate, measures nothing, and copies the branch's
2738 # record, every unit at the closures the squash lands.
2739 from livery.workshop._verified import tree_id
2740
2741 landed = git.head_sha(root)
2742 run, _jobs = _completed_run(repo, landed, event="push")
2743 facts = run_facts(root, run.id)
2744 _require_scope(run, facts, scope="verified")
2745 proved = verified_tree(root, tree_id(root, landed))
2746 if proved.branch != "chore/scoped-leg":
2747 fail(
2748 f"run {run.id}: main's tree is proved by a row naming {proved.branch!r},"
2749 " expected 'chore/scoped-leg', the branch whose record main copies"
2750 )
2751 _require_record(
2752 root,
2753 run,
2754 "main",
2755 leg,
2756 fresh=(),
2757 carried=[*members, *(["tests"] if tests else [])],
2758 )
2759 _require_judged(run, facts, judged)
2760 _prove_shards(root, run, facts, ran=False)
2761 print(
2762 f" composed skip: proven on main's run {run.id} after the member-only"
2763 " squash; nothing was measured, and main copied the branch's record"
2764 )
2765
2766
2767def _prove_unproven_push(root: Path, kind: str) -> None:
2768 """Prove main's run narrows from its own record when a squash lands a new tree.
2769
2770 Every other squash the loop lands is a tree a pull request's run
2771 proved, so main's run skips. This branch is cut from the commit
2772 the scoped-leg branch stood on and touches one other member, so
2773 its squash onto main, which carries the scoped-leg squash already,
2774 is a tree no run proved: main's push run cannot skip, measures
2775 from the previous push's tree, narrows to the member the branch
2776 touched, reuses the other suites from main's record, and composes
2777 its stamp on that tree. A push that paid the whole gate here, or
2778 skipped, is the failure this proof exists for.
2779 """
2780 _align_main(root)
2781 _main_settled(root, kind)
2782 before = git.head_sha(root)
2783 git.switch_new(root, "chore/unproven-push", f"{before}~1")
2784 probe = root / "packages" / "loop-native" / "tests" / "test_unproven_push.py"
2785 probe.write_text(
2786 '"""A member-only change behind main: the loop proves a push narrows."""\n'
2787 "\n"
2788 f'STAMP = "{before}"\n'
2789 "\n\n"
2790 "def test_the_stamp_is_a_commit():\n"
2791 " assert len(STAMP) == 40\n",
2792 "utf-8",
2793 )
2794 git.commit_all(
2795 root,
2796 "chore(loop-native): a member-only change behind main\n\nCut from the"
2797 " commit before main's tip, so its squash lands a tree no run proved.",
2798 )
2799 with contextlib.suppress(git.GitError):
2800 git.fetch(root, prune=True)
2801 _loop_fm(root, "submit", "--force", "--armed")
2802 _align_main(root)
2803 landed = git.head_sha(root)
2804 forge, _ = _dev_forge(kind)
2805 repo = forge.repository(E2E_OWNER, E2E_REPO)
2806 run, _jobs = _completed_run(repo, landed, event="push")
2807 facts = run_facts(root, run.id)
2808 _require_scope(
2809 run, facts, scope="affected", packages=["packages/loop-native"], composed=True
2810 )
2811 members, tests = _units(root)
2812 carried = [path for path in members if path != "packages/loop-native"]
2813 if tests:
2814 carried.append("tests")
2815 _require_record(
2816 root,
2817 run,
2818 "main",
2819 facts.job("check").leg,
2820 fresh=["packages/loop-native"],
2821 carried=carried,
2822 )
2823 _require_judged(run, facts, dict.fromkeys(members, 100.0))
2824 _require_verified(root, run, composed=True)
2825 _prove_shards(root, run, facts, ran=True)
2826 print(
2827 f" unproven push: proven on main's run {run.id}; the leg measured from the"
2828 " previous push's tree, ran loop-native alone, and the stamp composed"
2829 )
2830
2831
2832def _prove_prose_leg(root: Path, kind: str) -> None:
2833 """Prove a note-only pull request pays no package gate.
2834
2835 The change is one markdown file under ``notes/`` and nothing
2836 else, landed through the loop's gate on a pass-owned branch with
2837 main's tip as the stamp so the diff is never empty. The check
2838 leg's row says nothing was affected; the branch's record holds
2839 every unit carried from main's; the gate job judged every member;
2840 the stamp composed with main's verified tree, so main's push run
2841 after the squash skips too.
2842 """
2843 _fresh_branch(root, "chore/prose-leg")
2844 _main_settled(root, kind)
2845 stamp = git.head_sha(root)
2846 note = root / "notes" / "loop-prose.md"
2847 note.parent.mkdir(exist_ok=True)
2848 note.write_text(
2849 "# The loop's prose probe\n\nA note-only change, so the check legs"
2850 f" skip their gate.\n\nStamp: {stamp}\n",
2851 "utf-8",
2852 )
2853 git.commit_all(
2854 root,
2855 "docs(notes): a note-only change for the prose leg\n\nOne markdown"
2856 " file under notes/, so the check leg finds nothing affected and"
2857 " skips its gate.",
2858 )
2859 head = git.head_sha(root)
2860 with contextlib.suppress(git.GitError):
2861 git.fetch(root, prune=True)
2862 _loop_fm(root, "submit", "--force", "--armed")
2863 _align_main(root)
2864 forge, _ = _dev_forge(kind)
2865 repo = forge.repository(E2E_OWNER, E2E_REPO)
2866 run, _jobs = _completed_run(repo, head, event="pull_request")
2867 facts = run_facts(root, run.id)
2868 _require_scope(run, facts, scope="nothing", composed=True)
2869 members, tests = _units(root)
2870 units = [*members, *(["tests"] if tests else [])]
2871 _require_record(
2872 root, run, "chore/prose-leg", facts.job("check").leg, fresh=(), carried=units
2873 )
2874 _require_judged(run, facts, dict.fromkeys(members, 100.0))
2875 _require_verified(root, run, composed=True, branch="chore/prose-leg")
2876 _prove_shards(root, run, facts, ran=False)
2877 print(
2878 f" prose leg: proven on a note-only pull request (run {run.id}: the"
2879 " check leg skipped, the union reused every unit from main's record"
2880 " and wrote the branch's, the stamp composed)"
2881 )
2882 landed = git.head_sha(root)
2883 run, _jobs = _completed_run(repo, landed, event="push")
2884 _require_scope(run, run_facts(root, run.id), scope="verified")
2885 print(f" composed skip: proven on main's run {run.id} after the note-only squash")
2886
2887
2888def _prove_tests_leg(root: Path, kind: str) -> None:
2889 """Prove a change under the workspace's own tests runs that unit alone.
2890
2891 A new test file under the root ``tests/`` and nothing else: the
2892 check leg's row says it narrowed to the workspace tests; the
2893 branch's record holds that unit measured by this run and every
2894 member carried from main's; the gate job judged every member; the
2895 stamp composed with main's tree. Re-run on a pass-owned branch
2896 with main's tip as the stamp.
2897 """
2898 _fresh_branch(root, "chore/tests-leg")
2899 _main_settled(root, kind)
2900 stamp = git.head_sha(root)
2901 probe = root / "tests" / "test_tests_leg.py"
2902 # A fresh newborn has no workspace tests yet: this file is the first.
2903 probe.parent.mkdir(exist_ok=True)
2904 probe.write_text(
2905 '"""A workspace-tests change: the loop proves the tests leg on it."""\n'
2906 "\n"
2907 f'STAMP = "{stamp}"\n'
2908 "\n\n"
2909 "def test_the_stamp_is_a_commit():\n"
2910 " assert len(STAMP) == 40\n",
2911 "utf-8",
2912 )
2913 git.commit_all(
2914 root,
2915 "chore(tests): a workspace-tests change for the tests leg\n\nOne file"
2916 " under the root tests directory, so the check leg narrows to the"
2917 " workspace tests and says so.",
2918 )
2919 head = git.head_sha(root)
2920 with contextlib.suppress(git.GitError):
2921 git.fetch(root, prune=True)
2922 _loop_fm(root, "submit", "--force", "--armed")
2923 _align_main(root)
2924 forge, _ = _dev_forge(kind)
2925 repo = forge.repository(E2E_OWNER, E2E_REPO)
2926 run, _jobs = _completed_run(repo, head, event="pull_request")
2927 facts = run_facts(root, run.id)
2928 _require_scope(run, facts, scope="affected", packages=["tests"], composed=True)
2929 members, _tests = _units(root)
2930 _require_record(
2931 root,
2932 run,
2933 "chore/tests-leg",
2934 facts.job("check").leg,
2935 fresh=["tests"],
2936 carried=members,
2937 )
2938 _require_judged(run, facts, dict.fromkeys(members, 100.0))
2939 _require_verified(root, run, composed=True, branch="chore/tests-leg")
2940 _prove_shards(root, run, facts, ran=True)
2941 print(
2942 f" tests leg: proven on a workspace-tests pull request (run {run.id}: the"
2943 " check leg ran the workspace tests alone, the union reused every member"
2944 " from main's record, the stamp composed)"
2945 )
2946 landed = git.head_sha(root)
2947 run, _jobs = _completed_run(repo, landed, event="push")
2948 _require_scope(run, run_facts(root, run.id), scope="verified")
2949 print(f" composed skip: proven on main's run {run.id} after the tests-leg squash")
2950
2951
2952def _prove_nightly(root: Path, kind: str) -> None:
2953 """Prove the nightly point by hand: dispatched, followed to green, read back.
2954
2955 The loop's contract schedules the released member's verification
2956 at the nightly point, so the dispatched run proves a task attached
2957 through the contract with no YAML change, and the verbs that
2958 start and read a point run against the real runner. After the
2959 release act, so the verification has a wheel to install.
2960 """
2961 code = _loop_fm(root, "ci.dispatch", "--point=nightly", "--interval=5", nofail=True)
2962 if code:
2963 fail(f"the loop's `{footman.prog()} ci.dispatch --point=nightly` exited {code}")
2964 if _loop_fm(root, "ci.status", "--point=nightly", nofail=True):
2965 fail(
2966 f"the loop's `{footman.prog()} ci.status --point=nightly` did not"
2967 " read the dispatched run green"
2968 )
2969 from livery.workshop._ci_tasks import point_runs
2970
2971 forge, _ = _dev_forge(kind)
2972 repo = forge.repository(E2E_OWNER, E2E_REPO)
2973 run = point_runs(repo, "nightly", limit=1)[0]
2974 # The nightly's jobs record the run's row themselves; the row names
2975 # the scheduled verification among the tasks a job ran.
2976 _require_task(run_facts(root, run.id), "nightly", VERIFY_TASK)
2977 print(
2978 f" nightly: proven by hand (run {run.id}: dispatched, followed to green,"
2979 " the scheduled verification ran, and the point read back green)"
2980 )
2981
2982
2983def _prove_dispatched_gate(root: Path, kind: str) -> None:
2984 """Prove the gate on command: dispatched on main, full, followed to green.
2985
2986 A dispatched gate pays the full gate whatever the contract's
2987 affected-legs key says: the check verb reads the event and
2988 narrows on a pull request alone, and sets the verified record
2989 aside the way the nightly does, so a tree main already proved is
2990 proved again rather than skipped. The check leg says so, and the
2991 run is read back green through the point's own reader.
2992 """
2993 code = _loop_fm(root, "ci.dispatch", "--point=gate", "--interval=5", nofail=True)
2994 if code:
2995 fail(f"the loop's `{footman.prog()} ci.dispatch --point=gate` exited {code}")
2996 if _loop_fm(root, "ci.status", "--point=gate", nofail=True):
2997 fail(
2998 f"the loop's `{footman.prog()} ci.status --point=gate` did not"
2999 " read the dispatched run green"
3000 )
3001 from livery.workshop._ci_tasks import point_runs
3002
3003 forge, _ = _dev_forge(kind)
3004 repo = forge.repository(E2E_OWNER, E2E_REPO)
3005 run = point_runs(repo, "gate", limit=1)[0]
3006 if run.event != "workflow_dispatch":
3007 fail(
3008 f"the newest gate run {run.id} is a {run.event} run, not the"
3009 f" dispatched one: {run.url}"
3010 )
3011 facts = run_facts(root, run.id)
3012 _require_scope(run, facts, scope="full")
3013 _prove_shards(root, run, facts, ran=True)
3014 print(
3015 f" gate: proven by hand (run {run.id}: dispatched, followed to green,"
3016 " the full gate ran, and the point read back green)"
3017 )
3018
3019
3020def _prove_contributed_point(root: Path, kind: str) -> None:
3021 """Prove the member's point: dispatched, followed to green, its task run.
3022
3023 The member's contract declares the point; the rendered shell is
3024 the loop's own output, so the proof is the real runner running
3025 the declared task through ``ci.run`` and the point reading back
3026 green through the point's own reader.
3027 """
3028 code = _loop_fm(
3029 root, "ci.dispatch", f"--point={CONTRIBUTED_POINT}", "--interval=5", nofail=True
3030 )
3031 if code:
3032 fail(
3033 f"the loop's `{footman.prog()} ci.dispatch --point={CONTRIBUTED_POINT}`"
3034 f" exited {code}"
3035 )
3036 if _loop_fm(root, "ci.status", f"--point={CONTRIBUTED_POINT}", nofail=True):
3037 fail(
3038 f"the loop's `{footman.prog()} ci.status --point={CONTRIBUTED_POINT}`"
3039 " did not read the dispatched run green"
3040 )
3041 from livery.workshop._ci_tasks import point_runs
3042
3043 forge, _ = _dev_forge(kind)
3044 repo = forge.repository(E2E_OWNER, E2E_REPO)
3045 run = point_runs(repo, CONTRIBUTED_POINT, root)[0]
3046 # The point's job records the run's row itself; the row names the
3047 # member's declared task among the tasks the job ran.
3048 _require_task(run_facts(root, run.id), CONTRIBUTED_POINT, "extensions")
3049 print(
3050 f" {CONTRIBUTED_POINT}: proven by hand (run {run.id}: the contributed"
3051 " point dispatched, its task run, and the point read back green)"
3052 )
3053
3054
3055def _release_act(root: Path, kind: str) -> None:
3056 """Release the member through the loop; verify wheel and receipt.
3057
3058 The armed release drives prepare, the isolated legs, the release
3059 pull request through the gate, and the merge; the emitted
3060 release workflow's wave then publishes to the forge's own
3061 registry and cuts the receipt tag. Done means measured: the
3062 served version and the annotated tag, never the exit code alone,
3063 and for the native member the publish job's line that the wave
3064 published the wheels the wheels job built.
3065 """
3066 from livery.workshop._release_driver import release_name
3067
3068 # Only the members without a receipt release: the driver refuses a
3069 # set with a member nothing unreleased touches, and a cut receipt
3070 # is exactly that. A fresh loop releases both in one set.
3071 all_tags = {
3072 member.name: f"packages/{member.name}/v0.1.0" for member in LOOP_MEMBERS
3073 }
3074 try:
3075 on_origin = git.ls_remote(root, *all_tags.values(), tags=True)
3076 except git.GitError:
3077 on_origin = {}
3078 cut = {name for name, tag in all_tags.items() if f"refs/tags/{tag}" in on_origin}
3079 for name in sorted(cut):
3080 _require_receipt_protected(root, all_tags[name], kind)
3081 print(f" release: receipt {all_tags[name]} already on the loop")
3082 names = tuple(member.name for member in LOOP_MEMBERS if member.name not in cut)
3083 if not names:
3084 return
3085 tags = {name: all_tags[name] for name in names}
3086 _align_main(root)
3087 # A prepared release PR may survive an earlier pass. Fresh (its
3088 # branch still contains main) it is the driver's own recovery:
3089 # left alone, the re-run arms and merges it, which is also how
3090 # the forge's long post-open conflict-check window is ridden out
3091 # (measured outlasting the merge retry budget). Stale (main has
3092 # moved past it) the driver refuses and teaches `abandon`; the
3093 # loop follows the teach through the verb before releasing.
3094 release_branch = f"workflow/{release_name(names)}"
3095 forge, _ = _dev_forge(kind)
3096 survivor = forge.repository(E2E_OWNER, E2E_REPO).pr.find_by_head(release_branch)
3097 if survivor is not None and survivor.state == "open":
3098 git.fetch(root, release_branch)
3099 if git.is_ancestor(root, "origin/main", f"origin/{release_branch}"):
3100 print(f" prepared release PR #{survivor.number} is fresh; recovering it")
3101 else:
3102 print(f" abandoning stale prepared release PR #{survivor.number}")
3103 git.switch_new(root, release_branch, f"origin/{release_branch}")
3104 _loop_fm(root, "abandon")
3105 _align_main(root)
3106 # Bounded self-heal for the forge's post-open window: right after
3107 # a release PR opens, the arm can answer 405 for minutes while
3108 # the forge computes mergeability (measured: still refusing at
3109 # fifty seconds, mergeable when probed later). The verb's own
3110 # recovery arms the surviving fresh PR, so re-running it is the
3111 # ride-out; a failure with no fresh survivor is real and final.
3112 import time
3113
3114 repo = forge.repository(E2E_OWNER, E2E_REPO)
3115 before_sha = _origin_main(root)
3116 for round_ in range(4):
3117 if round_:
3118 print(" waiting out the forge's post-open window (45s)")
3119 time.sleep(45)
3120 code = _loop_fm(
3121 root,
3122 "workflow.release",
3123 *names,
3124 "--armed",
3125 timeout=1800.0,
3126 nofail=True,
3127 )
3128 if code == 0:
3129 break
3130 # The wave's red run, readable here and nowhere else once a
3131 # hosted job's forge is gone.
3132 from livery.workshop._ci_tasks import point_runs
3133
3134 for run in point_runs(repo, "release", root, limit=3):
3135 if run.conclusion == "failure":
3136 print(red_jobs_text(repo, run))
3137 break
3138 survivor = repo.pr.find_by_head(release_branch)
3139 if survivor is None or survivor.state != "open":
3140 fail("the armed release failed with no fresh PR to recover; see above")
3141 _align_main(root)
3142 if not git.is_ancestor(root, "origin/main", f"origin/{release_branch}"):
3143 fail("the armed release failed and its PR went stale; see above")
3144 else:
3145 fail("the forge's post-open window never closed across 4 rounds")
3146 # The armed release returns at the merge, and the forge moves
3147 # the ref a beat later (measured: an immediate rev-parse watched
3148 # the pre-squash commit and called the wave skipped). Poll for
3149 # the movement; a main that never moves means the recovery arm
3150 # ran, which cuts receipts itself, and the probes below judge.
3151 squash = ""
3152 deadline = time.monotonic() + 90
3153 while time.monotonic() < deadline:
3154 moved = _origin_main(root)
3155 if moved and moved != before_sha:
3156 squash = moved
3157 break
3158 time.sleep(3)
3159 if squash:
3160 # The wave runs on the squash asynchronously; its red must
3161 # surface verbatim, because a registry poll alone cannot say
3162 # whether the wave failed or is merely slow.
3163 _watch(kind, _lane(kind).alias, squash, require=("release.yml",))
3164 else:
3165 # The recovery arm dispatched the wave at the stamping commit,
3166 # which is not main's tip, so the newest release run is the
3167 # one to follow: its red surfaces verbatim instead of a blind
3168 # registry wait that expires on a slow wheels leg.
3169 print(" main unmoved: the recovery arm dispatched; following its wave")
3170 _watch_latest(kind, "release.yml")
3171 # The served version is the same whether the wheels job built the
3172 # native member's wheels or the publish job built its own runner's;
3173 # the publish job's log says which. The wave just followed is the
3174 # release point's newest run.
3175 from livery.workshop._ci_tasks import point_runs
3176
3177 wave = point_runs(repo, "release", root, limit=1)[0]
3178 for member in members_for(names):
3179 if "nanobind" in member.holds():
3180 _require_prebuilt(repo, wave, _contract_name(root, member.name))
3181 served_versions = _serving_probe(root)
3182 for name in names:
3183 _until(
3184 functools.partial(_serves, served_versions, name),
3185 timeout=300,
3186 failing=(
3187 "the wave is green but the registry never served"
3188 f" {_contract_name(root, name)} 0.1.0"
3189 ),
3190 )
3191 # The receipt push follows the publish inside the wave, so the
3192 # tag gets the same patience as the serving probe.
3193 for tag in tags.values():
3194 _until(
3195 functools.partial(_tag_on_remote, root, tag),
3196 timeout=120,
3197 failing=f"served, but the receipt tag {tag} is not on the loop",
3198 retried=(git.GitError,),
3199 )
3200 _require_receipt_protected(root, tag, kind)
3201 served = ", ".join(f"{_contract_name(root, name)} 0.1.0" for name in names)
3202 print(f" release: {served} served, receipts {', '.join(tags.values())} cut")
3203
3204
3205def _serves(probe: Callable[[str], Collection[str]], name: str) -> bool:
3206 """Whether the registry *probe* serves *name* at the loop's release version."""
3207 return "0.1.0" in probe(name)
3208
3209
3210def _tag_on_remote(root: Path, tag: str) -> bool:
3211 """Whether *tag* is on the loop's remote."""
3212 return f"refs/tags/{tag}" in git.ls_remote(root, tag, tags=True)
3213
3214
3215def _until(
3216 condition: Callable[[], bool],
3217 *,
3218 timeout: float,
3219 failing: str,
3220 retried: tuple[type[Exception], ...] = (),
3221) -> None:
3222 """Poll *condition* every five seconds until it holds.
3223
3224 Past *timeout* seconds, fail with *failing*.
3225 """
3226
3227 def holds() -> bool | Pending:
3228 return True if condition() else Pending()
3229
3230 try:
3231 follow(
3232 None,
3233 holds,
3234 subject=failing,
3235 interval=5,
3236 timeout=timeout,
3237 quiet=None,
3238 retried=retried,
3239 )
3240 except (FollowTimedOut, ForgeUnreachable):
3241 fail(failing)
3242
3243
3244def _require_receipt_protected(root: Path, tag: str, kind: str = "gitea") -> None:
3245 """Prove the receipt's protection by attempting the crime.
3246
3247 A refused delete is the strongest proof and ends it; GitLab
3248 refuses a protected tag's deletion over git to everyone, so there
3249 it is the only proof. Gitea ties creation, deletion, and movement
3250 to one whitelist, so the loop's own lane (the whitelisted admin)
3251 can delete; there the tag is pushed straight back and the proof is
3252 the protection's presence with the lane on its whitelist, read
3253 from the API, which is what binds everyone else.
3254 """
3255 import fnmatch
3256 import json
3257 import urllib.request
3258
3259 # Forced: a receipt recut by a later wave is a new tag object, and
3260 # the workspace may still hold the one an earlier pass fetched.
3261 with contextlib.suppress(git.GitError):
3262 git.fetch_tag(root, tag)
3263 try:
3264 git.push_refspec(root, f":refs/tags/{tag}")
3265 except git.GitError:
3266 print(f" receipt {tag}: delete refused; protection holds")
3267 return
3268 with contextlib.suppress(git.GitError):
3269 git.push_refspec(root, f"refs/tags/{tag}")
3270 if kind != "gitea":3270 ↛ 3276line 3270 didn't jump to line 3276 because the condition on line 3270 was always true
3271 fail(
3272 f"the receipt tag {tag} was deletable on {kind}, whose protection"
3273 " refuses every deletion over git: the contract's assertion did"
3274 " not hold (the receipt was pushed back)"
3275 )
3276 url = os.environ.get("GITEA_URL", "")
3277 token = os.environ.get("GITEA_TOKEN", "")
3278 request = urllib.request.Request(
3279 f"{url}/api/v1/repos/{E2E_OWNER}/{E2E_REPO}/tag_protections",
3280 headers={"Authorization": f"token {token}"},
3281 )
3282 with urllib.request.urlopen(request, timeout=10) as answer:
3283 entries = json.load(answer)
3284 covering = [
3285 entry
3286 for entry in entries
3287 if fnmatch.fnmatch(tag, str(entry.get("name_pattern", "")))
3288 ]
3289 if not covering:
3290 fail(
3291 f"the receipt tag {tag} was deletable and no tag protection"
3292 " covers it: the contract's assertion did not hold (the"
3293 " receipt was pushed back)"
3294 )
3295 print(
3296 f" receipt {tag}: protected; the configuring lane stays on the"
3297 " whitelist (gitea's one-whitelist model), everyone else is bound"
3298 )
3299
3300
3301def _merge_setup(kind: str, sha: str) -> None:
3302 """Merge the setup pull request when its green head is *sha*.
3303
3304 Birth's own done-line prescribes it: merging the setup PR proves
3305 the gate and the protection wiring end to end. Patient through
3306 the forge's whole 405 family: the mergeability recompute's "try
3307 again later" and the beat where a finished run's required
3308 context has not propagated yet ("not all required status checks
3309 successful", measured seconds after the watch saw the run
3310 green). Already merged, or a head that moved on, is a quiet
3311 skip.
3312 """
3313 from livery.workshop._new_project import _SETUP_BRANCH
3314 from livery.workshop._submit import _follow_merge_state
3315
3316 forge, _ = _dev_forge(kind)
3317 repo = forge.repository(E2E_OWNER, E2E_REPO)
3318 pr = repo.pr.find_by_head(_SETUP_BRANCH)
3319 if pr is None or pr.state != "open":
3320 print(" setup PR: already merged or gone")
3321 return
3322 head = getattr(pr, "head_sha", "")
3323 if head and head != sha:
3324 print(" setup PR: head moved on; leaving it to the next run")
3325 return
3326 _follow_merge_state(
3327 repo, kind, pr.number, repo.pr.merge_now, pr.number, title=pr.title
3328 )
3329 print(f" setup PR #{pr.number}: merged; the gate is proven")
3330
3331
3332# --- the scenarios ---------------------------------------------------------------
3333
3334
3335@dataclass
3336class Pass:
3337 """What one pass carries between its scenarios.
3338
3339 Attributes:
3340 forge: The lane's kind, ``gitea`` or ``gitlab``.
3341 url: The forge's host-side URL from the shared environment.
3342 fresh: Whether the pass starts over from nothing.
3343 root: The loop's workspace once ``birth`` has run; None before.
3344 timings: One entry per scenario run, in order.
3345 extension: The extension under test; empty for a plain pass.
3346 stack: The extensions the birth lists: the loop's own
3347 ([livery.workshop._e2e.loop_stack][]) or the extension
3348 under test's; empty for the stock list.
3349 members: The members the members scenario lands, by name; empty
3350 for all.
3351 """
3352
3353 forge: str
3354 url: str
3355 fresh: bool = False
3356 root: Path | None = None
3357 timings: list[Timing] = field(default_factory=list)
3358 extension: str = ""
3359 stack: tuple[str, ...] = ()
3360 members: tuple[str, ...] = ()
3361
3362
3363@dataclass(frozen=True)
3364class Timing:
3365 """What one scenario cost.
3366
3367 Attributes:
3368 name: The scenario.
3369 seconds: Its wall time.
3370 runs: The runner runs it followed.
3371 failed: Whether it ended in a refusal or an error.
3372 """
3373
3374 name: str
3375 seconds: float
3376 runs: int
3377 failed: bool = False
3378
3379
3380@dataclass(frozen=True)
3381class Scenario:
3382 """One thing the loop proves, by name, with what must have run before it.
3383
3384 Attributes:
3385 name: The name ``--scenario`` takes.
3386 needs: The scenarios this one builds on; a pass runs them
3387 first, once, whether or not they were named.
3388 run: The proof, given the pass.
3389 daemon: Whether it needs the runner's docker socket: the
3390 release builds the native member's wheel through
3391 cibuildwheel on the runner, and nothing else does.
3392 """
3393
3394 name: str
3395 needs: tuple[str, ...]
3396 run: Callable[[Pass], None]
3397 daemon: bool = False
3398
3399
3400def _born(pass_: Pass) -> None:
3401 """Birth or resume the loop's workspace and merge its setup pull request.
3402
3403 The repository, its protection and the setup pull request through
3404 ``fm new.project``, the secrets, the wiring to the workshop's own
3405 dev wheels, the pushed head's runs followed to their verdicts on
3406 the real runner, and the setup squash merged.
3407 """
3408 forge = pass_.forge
3409 lane, lane_token = _dev_forge(forge)
3410 root = _loop_home(forge) / E2E_REPO
3411 if (root / ".git").is_dir() and _has_remote(root):
3412 # The origin names the forge of the pass that last wrote it,
3413 # which a fresh pass removed with its environment, so it is
3414 # pointed at this pass's forge before the start-over below asks
3415 # it for commits it has not seen.
3416 _authenticate_remote(root, lane_token, forge)
3417 listed = _listed_extensions(root)
3418 if listed and listed != _set_of(_stack_of(pass_)):
3419 # A workspace born for another list is not this pass's: an
3420 # extension pass and a plain one take turns in one environment.
3421 for line in start_over(lane, lane_token, root, url=pass_.url, kind=forge):3421 ↛ anywhereline 3421 didn't jump anywhere: it always raised an exception.
3422 print(line)
3423 elif pass_.fresh and CURRENT.mode != "host":3423 ↛ 3427line 3423 didn't jump to line 3427 because the condition on line 3423 was never true
3424 # A fresh host-mode pass took a new environment: its forge
3425 # holds nothing yet. The docker rig is long-lived, so its
3426 # repository, releases and workspace go here.
3427 for line in start_over(lane, lane_token, root, url=pass_.url, kind=forge):
3428 print(line)
3429 elif pass_.fresh and root.exists():3429 ↛ 3430line 3429 didn't jump to line 3430 because the condition on line 3429 was never true
3430 shutil.rmtree(root, ignore_errors=True)
3431 if (root / ".git").is_dir() and _has_remote(root):
3432 # A resumed birth pushes before it returns, so an existing
3433 # workspace authenticated above; birth resets the remote, so
3434 # it authenticates again after. Main then fast-forwards onto
3435 # the merges the loop itself made: without the reconcile,
3436 # birth's foreign-repo guard reads our own squash as a
3437 # stranger's history and refuses. A birth that failed before
3438 # the forge held the repository left no remote, and resumes
3439 # as a first birth does.
3440 _align_main(root)
3441 from livery.workshop._extensions import workspace_root
3442
3443 root = _birth(
3444 forge,
3445 pass_.url,
3446 index=_dev_index(forge, pass_.stack),
3447 stack=pass_.stack,
3448 driver=workspace_root(),
3449 )
3450 _authenticate_remote(root, lane_token, forge)
3451 provision(forge)
3452 # After the project exists: GitLab's registry belongs to the
3453 # project, and a fresh start's delete is asynchronous, so a
3454 # publish before the birth lands in the project being deleted.
3455 pins = _publish_dev_wheels(forge, pass_.stack)
3456 enabled = steps_declared_off(pass_.extension) if pass_.extension else ()
3457 sha = _eat_dev_wheels(root, pins, forge, enabled=enabled)
3458 print(f" watching {sha[:12]} on the runner")
3459 from livery.workshop._new_project import _SETUP_BRANCH
3460
3461 _watch(forge, pass_.url, sha, branch=_SETUP_BRANCH)
3462 print(" green: the loop's gate ran on the real runner")
3463 _merge_setup(forge, sha)
3464 pass_.root = root
3465
3466
3467def _stack_of(pass_: Pass) -> tuple[str, ...]:
3468 """The extensions *pass_*'s birth lists: its own stack, else the stock list."""
3469 from livery.workshop._extensions import SELF
3470 from livery.workshop._new_project import birth_extensions
3471
3472 return pass_.stack or tuple(birth_extensions([SELF]))
3473
3474
3475def _set_of(names: tuple[str, ...]) -> frozenset[str]:
3476 """*names* and every extension they require: the set a list of them names."""
3477 from livery.workshop._composition import closure
3478 from livery.workshop._extensions import installed_declaration
3479
3480 return frozenset(closure(names, installed_declaration))
3481
3482
3483#: The extension that builds the site, which a plain pass's birth leaves out.
3484SITE = "docs"
3485
3486
3487def loop_stack() -> tuple[str, ...]:
3488 """The extensions a plain pass's birth lists, as their canonical list.
3489
3490 The stock list without the site, and what the loop's members'
3491 templates need (`LoopMember.needs`): a creation installs nothing,
3492 so the root lists them before the members are created. The site is
3493 the docs extension and every stock entry whose declaration requires
3494 it, directly or through another: the docs extension puts a docs job
3495 on every run of the loop's project, for a site nothing in a pass
3496 reads, and an extension that requires it brings it into the root's
3497 set. The pass proves the gate, the release train and the points,
3498 and ``fm ci.e2e --extension=docs`` proves the docs extension on its
3499 own. A stock entry no installed distribution declares requires
3500 nothing and stays.
3501 """
3502 from livery.workshop._extensions import SELF, installed_declaration
3503 from livery.workshop._new_project import birth_extensions, canonical_extensions
3504
3505 def builds_the_site(name: str, seen: frozenset[str]) -> bool:
3506 if name == SITE:
3507 return True
3508 found = installed_declaration(name)
3509 needed = found.requires if found is not None else ()
3510 return any(
3511 builds_the_site(other, seen | {name})
3512 for other in needed
3513 if other not in seen
3514 )
3515
3516 stock = [
3517 entry
3518 for entry in birth_extensions([SELF])
3519 if not builds_the_site(entry, frozenset())
3520 ]
3521 needs = [needed for member in LOOP_MEMBERS for needed in member.needs()]
3522 return tuple(canonical_extensions([*stock, *needs]))
3523
3524
3525def _listed_extensions(root: Path) -> frozenset[str]:
3526 """The root's set of *root*'s contract: its list and what that requires.
3527
3528 Empty with no contract. The set, not the list, so an extension
3529 another one requires counts whether the list writes it or not.
3530 """
3531 from livery.workshop._extensions import root_set
3532
3533 if not (root / "workshop.toml").is_file():
3534 return frozenset()
3535 return frozenset(root_set(root))
3536
3537
3538def _at(pass_: Pass) -> Path:
3539 """The loop's workspace; a refusal when birth has not run in this pass."""
3540 if pass_.root is None:
3541 fail("the loop's workspace is not born yet; `birth` runs before every scenario")
3542 return pass_.root
3543
3544
3545#: Every scenario, in dependency order: a scenario's needs come
3546#: before it, so a chosen subset runs in this order too.
3547SCENARIOS: tuple[Scenario, ...] = (
3548 Scenario("birth", (), _born),
3549 Scenario(
3550 "verified-skip", ("birth",), lambda p: _prove_verified_skip(_at(p), p.forge)
3551 ),
3552 Scenario("members", ("birth",), lambda p: _ensure_members(_at(p), p.members)),
3553 Scenario("ratchet", ("members",), lambda p: _prepare_ratchet(_at(p), p.forge)),
3554 Scenario("scoped-leg", ("ratchet",), lambda p: _prove_scoped_leg(_at(p), p.forge)),
3555 Scenario(
3556 "unproven-push",
3557 ("scoped-leg",),
3558 lambda p: _prove_unproven_push(_at(p), p.forge),
3559 ),
3560 Scenario("prose-leg", ("members",), lambda p: _prove_prose_leg(_at(p), p.forge)),
3561 Scenario("tests-leg", ("members",), lambda p: _prove_tests_leg(_at(p), p.forge)),
3562 Scenario(
3563 "release", ("members",), lambda p: _release_act(_at(p), p.forge), daemon=True
3564 ),
3565 Scenario("nightly", ("release",), lambda p: _prove_nightly(_at(p), p.forge)),
3566 Scenario(
3567 "dispatched-gate",
3568 ("members",),
3569 lambda p: _prove_dispatched_gate(_at(p), p.forge),
3570 ),
3571 Scenario(
3572 "contributed-point",
3573 ("members",),
3574 lambda p: _prove_contributed_point(_at(p), p.forge),
3575 ),
3576)
3577
3578#: The named sets ``--scenario`` takes beside the scenarios' own names.
3579SETS: dict[str, tuple[str, ...]] = {
3580 "develop": ("birth", "verified-skip", "members", "scoped-leg", "unproven-push"),
3581 "release": (
3582 "birth",
3583 "verified-skip",
3584 "members",
3585 "scoped-leg",
3586 "unproven-push",
3587 "release",
3588 ),
3589 "points": ("nightly", "dispatched-gate", "contributed-point"),
3590 "extension": ("birth", "members"),
3591 "all": tuple(scenario.name for scenario in SCENARIOS),
3592}
3593
3594
3595def scenarios_for(spec: str) -> tuple[Scenario, ...]:
3596 """The scenarios *spec* names, their needs added, in the registry's order.
3597
3598 *spec* is comma-separated set names and scenario names. A name
3599 that is neither refuses naming both lists; an empty *spec*
3600 refuses too. A need is run once however many names reach it.
3601 """
3602 by_name = {scenario.name: scenario for scenario in SCENARIOS}
3603 wanted: set[str] = set()
3604 for token in (part.strip() for part in spec.split(",")):
3605 if not token:
3606 continue
3607 if token in SETS:
3608 wanted.update(SETS[token])
3609 elif token in by_name:
3610 wanted.add(token)
3611 else:
3612 fail(
3613 f"{token!r} is not a scenario or a set; the sets are"
3614 f" {', '.join(SETS)} and the scenarios {', '.join(by_name)}"
3615 )
3616 if not wanted:
3617 fail(f"--scenario names nothing; the sets are {', '.join(SETS)}")
3618 pending = list(wanted)
3619 while pending:
3620 for need in by_name[pending.pop()].needs:
3621 if need not in wanted:
3622 wanted.add(need)
3623 pending.append(need)
3624 return tuple(scenario for scenario in SCENARIOS if scenario.name in wanted)
3625
3626
3627def daemon_needed(scenarios: Sequence[Scenario]) -> bool:
3628 """Whether any of *scenarios* needs the runner's docker socket."""
3629 return any(scenario.daemon for scenario in scenarios)
3630
3631
3632def run_scenario(pass_: Pass, scenario: Scenario) -> None:
3633 """Run *scenario* for *pass_*, timed; a failure is timed and marked, then raised."""
3634 import time
3635
3636 started = time.monotonic()
3637 runs_before = RUNS.count
3638 print(f" scenario {scenario.name}")
3639 failed = True
3640 try:
3641 scenario.run(pass_)
3642 failed = False
3643 finally:
3644 pass_.timings.append(
3645 Timing(
3646 scenario.name,
3647 time.monotonic() - started,
3648 RUNS.count - runs_before,
3649 failed,
3650 )
3651 )
3652
3653
3654def timing_table(timings: Sequence[Timing]) -> list[str]:
3655 """The lines a pass ends with: each scenario's time and runs, then the total."""
3656 if not timings:
3657 return [" timing: nothing ran"]
3658 width = max(len(timing.name) for timing in timings)
3659 lines = [
3660 f" {timing.name:<{width}} {timing.seconds:7.1f}s {timing.runs} run(s)"
3661 + (" failed" if timing.failed else "")
3662 for timing in timings
3663 ]
3664 total = sum(timing.seconds for timing in timings)
3665 runs = sum(timing.runs for timing in timings)
3666 lines.append(f" {'total':<{width}} {total:7.1f}s {runs} run(s)")
3667 return lines
3668
3669
3670#: The local series a pass writes its rows to, declared with the
3671#: others so `fm store.show loop` reads it and the janitor bounds it.
3672LOOP_SERIES = LOOP
3673
3674
3675def record_pass(root: Path, pass_: Pass, asked: str) -> str:
3676 """Put the pass's timings on the local loop series; the line to print."""
3677 from datetime import UTC, datetime
3678
3679 when = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
3680 row: dict[str, object] = {
3681 "forge": pass_.forge,
3682 "asked": asked,
3683 **({"extension": pass_.extension} if pass_.extension else {}),
3684 "scenarios": [
3685 {
3686 "name": timing.name,
3687 "seconds": round(timing.seconds, 1),
3688 "runs": timing.runs,
3689 "failed": timing.failed,
3690 }
3691 for timing in pass_.timings
3692 ],
3693 }
3694 why = LOOP_SERIES.put(
3695 state_store(root),
3696 {f"pass-{when}.json": row},
3697 message=f"loop: pass {when} ({asked})",
3698 )
3699 if why:3699 ↛ 3700line 3699 didn't jump to line 3700 because the condition on line 3699 was never true
3700 return f" loop series: {why}; the pass is not recorded"
3701 return f" loop series: pass {when} recorded ({len(pass_.timings)} scenario(s))"
3702
3703
3704if _WORKSHOP_TESTS.is_dir():3704 ↛ exitline 3704 didn't exit the module because the condition on line 3704 was always true
3705 # serial: the driver births into and pushes from its own
3706 # directories, so it owns the process globals for the run.
3707 @ci.task(name="e2e", serial=True)
3708 @git.unsigned
3709 def e2e(
3710 forge: str = "gitea",
3711 fresh: bool = False,
3712 scenario: str = "",
3713 env: str = "e2e",
3714 purge_cache: bool = False,
3715 extension: str = "",
3716 from_main: bool = False,
3717 ) -> None:
3718 """Exercise the CI and release story on a local environment, by scenario.
3719
3720 ``--env`` names the environment the pass runs on, `e2e` by
3721 default: one that does not exist is brought up in host mode
3722 with one runner, ``--fresh`` removes it and brings it up
3723 again, so a fresh pass is a new forge, and ``--purge-cache``
3724 removes the rig's shared caches first. The docker-mode
3725 environment ``dev`` is the compose rig, addressed as before.
3726 Each variable an extension of the loop's set declares with a
3727 directory of its own is placed inside the environment, so what a
3728 release writes there goes with the environment's forge.
3729 ``--scenario`` names what the pass proves: scenario names, set
3730 names, or both, comma-separated. The sets are ``develop``
3731 (birth, the verified skip, the members, the scoped leg), the
3732 default, ``release`` (develop and the release act), ``points``
3733 (the nightly, the dispatched gate, the contributed point),
3734 ``extension`` (birth and the members) and ``all``; a scenario's
3735 needs run first, once. ``--extension`` tests one extension in
3736 isolation, ``extension`` the default set: the birth lists that
3737 extension and what it requires instead of the loop's stack, and
3738 the members are the loop's whose sets hold an extension its steps
3739 name. ``birth`` builds
3740 this checkout's workshop, its dependencies and the extensions a
3741 birth lists into a local index, births or resumes the loop's
3742 workspace through ``fm new.project`` from that index (so an
3743 extension the birth lists needs no release), with its
3744 repository, protection and setup pull request, provisions the
3745 secrets, wires the workspace to the dev wheels in the forge's
3746 registry, and follows the pushed head's runs to their verdicts
3747 on the real runner. Re-running is the recovery procedure at every
3748 step. The pass ends with a table, each scenario's time and
3749 runs, and records the same rows on the local ``loop`` series.
3750 ``--fresh`` starts over from nothing: the loop's repository on
3751 the dev forge, its releases in the forge's registry, and the
3752 local workspace go, then the pass births everything anew. It
3753 refuses while the workspace holds commits the forge has not
3754 seen. ``--from-main`` moves a checkout standing on ``main`` or a
3755 reserved ``workflow/`` branch, where a build would be the release
3756 train, to a scratch branch at the same commit for the pass and
3757 back afterwards; the nightly's checkout is one.
3758 """
3759 import os
3760
3761 from livery.workshop import _devenv
3762 from livery.workshop._extensions import workspace_root
3763
3764 driver = workspace_root()
3765 with scratch_branch(driver, enabled=from_main):
3766 if driver is not None:
3767 # Before anything is brought up: on a branch whose release
3768 # verb is the train, the first build would open a release.
3769 require_dev_branch(driver)
3770 asked = scenario or ("extension" if extension else "develop")
3771 chosen = scenarios_for(asked)
3772 stack, members = (
3773 extension_under_test(extension) if extension else (loop_stack(), ())
3774 )
3775 if forge != "gitea":
3776 fail(
3777 f"--env addresses a Gitea environment; --forge={forge} has none yet"
3778 )
3779 if purge_cache:
3780 for line in _devenv.purge_cache():
3781 print(line)
3782 enter_environment(env, stack, fresh=fresh)
3783 # Every commit the pass makes, the driver's, the birth's and
3784 # the loop's own fm's, is unsigned and carries the tools'
3785 # identity: the task's marks and the git plugin put both in
3786 # the task's environment, which rides into each child. On a
3787 # hosted runner the job's CI markers would reach every child,
3788 # which would then refuse a fix and stamp its runs as the
3789 # runner's; the pass drives its project as a desk.
3790 scrubbed = edit_environments({}, remove=CI_MARKERS)
3791 if scrubbed:
3792 names = ", ".join(scrubbed)
3793 print(f" ci markers removed for the pass's children: {names}")
3794 _require_host_alias(forge)
3795 if daemon_needed(chosen):
3796 _require_runner_docker(forge)
3797 pass_ = Pass(
3798 forge,
3799 CURRENT.url or os.environ.get(_lane(forge).url_var, ""),
3800 fresh,
3801 extension=extension,
3802 stack=stack,
3803 members=members,
3804 )
3805 RUNS.count = 0
3806 try:
3807 for item in chosen:
3808 run_scenario(pass_, item)
3809 finally:
3810 for line in timing_table(pass_.timings):
3811 print(line)
3812 driver = workspace_root()
3813 if driver is not None:
3814 print(record_pass(driver, pass_, asked))
3815 names = ", ".join(item.name for item in chosen)
3816 print(f" the loop proved {names}")