1"""The tools a workspace requires: six declaration sites, one lock, receipts.
3A tool requirement is a name with a floor, `git` or `git>=2.40`, and
4six sites declare them: the base, under `[toolroom] requires` in its
5contract file, for uv, which makes the venv every step runs in; the
6steps that judge a present package, each naming its tools; a listed
7extension, the workspace's or one a package's set holds, under
8`[toolroom] requires` in its `extension.toml`, for the tools it runs
9(cmake's cmake and ninja, conan's conan), and in a `[for.<target>]`
10table, for what it needs while that table applies; a plugin the
11project mounts, through its `workshop.tools` entry point; a package
12instance, in its `workshop.toml` under `[toolroom] requires`, for what
13its extensions cannot know; and the project, in the root contract's
14`[toolroom] requires`, for what belongs to the repository. The sites'
15requirements union, and `toolroom.lock` at the root holds one version
16per tool for the whole repository, the newest the catalogue lists that
17satisfies every floor and resolves on every supported host. The root
18contract's `[workspace] hosts` names those hosts, every host key when
19it is absent, and `[toolroom] index` names where the catalogue is read
20from: the published index by URL, a directory holding one, or the
21records that build one.
23Entering the environment materialises the bundle the sites require, on
24`fm sync` and on `fm toolroom.add`: each locked tool is supplied through
25the machine's store in one of three modes, its entry points linked
26into the checkout's bin directory, its own directories on PATH, or no
27PATH at all, and a receipt under `.workshop/receipts/` records the
28exact version, the host, the deployment's digest and what reached
29PATH. A receipt is a checkout's statement of what it installed, as the
30release train's receipt is a release's; `fm env.check` reads them
31back and names the drift when the lock's deployment has moved under
32one.
34The stubs the type checkers read are materialised too, into `typings/`
35at the root, pyright's default stub path and a search path the rendered
36configuration hands the other three checkers: one rendering per tool
37the lock holds, at the locked version, as `livery.toolroom.stubs`
38modules with `livery.toolroom.handles` beside them declaring each
39handle, which the installed tools package's index imports. Both live
40under the `livery.toolroom` namespace package, beside the tools package
41and never inside its directory: a tree inside it would shadow the
42package for a checker run on explicit paths. The store renders each
43stub from the locked version's own surface, from the records or from
44the index, so no source holds a stub. A tool the workspace does not
45deploy gets no stub. `fm toolroom.restub` writes them, and so do `fm sync`
46and every lock verb.
47"""
49from __future__ import annotations
51import functools
52from typing import TYPE_CHECKING
54if TYPE_CHECKING:
55 from livery.strongroom import Source
56 from livery.toolroom.store import (
57 Catalogue,
58 Deployment,
59 Ensured,
60 Graph,
61 Home,
62 Listed,
63 Lock,
64 Locked,
65 Requirement,
66 )
67 from livery.workshop._packages import Package
69import hashlib
70import json
71import os
72import shutil
73import sys
74import tempfile
75from dataclasses import dataclass, field, replace
76from pathlib import Path
77from typing import Any, NoReturn, cast
79from livery.footman import Failed, fail, prog, run
80from livery.workshop._contract import load_contract
81from livery.workshop._packages import discover_packages
83WORKSPACE = "workspace"
84"""The contract table the supported hosts live under."""
87DEFAULT_HOSTS: tuple[str, ...] = ()
88"""The hosts supported when `[workspace] hosts` is absent; empty for
89every host key, which the store names and which is read only then."""
91VERDICT_PHASES = ("format", "lint", "typecheck", "typecomplete", "test")
92"""The phases whose tools take no host allowance: the version is the verdict."""
94PINNED_TOOLS = ("uv",)
95"""The tools no allowance reaches by name: the entry pins uv."""
97BASE_SITE = "the base"
98"""The site of the base's own requirements, the ones every workspace has."""
100TYPINGS = "typings"
101"""The directory under the root the stubs are written into, pyright's default."""
103TYPINGS_PACKAGE = ("livery", "toolroom")
104"""The namespace package the stubs and handles live under, beside the tools package."""
107def tools_table(path: Path) -> dict[str, object]:
108 """The `[toolroom]` table of the contract at *path*, judged; empty when absent."""
109 from livery.toolroom.store import TABLE
111 if not path.is_file():
112 return {}
113 return dict(load_contract(path).get(TABLE) or {})
116def _requires(table: dict[str, object], *, site: str) -> list[Requirement]:
117 from livery.toolroom.store import LockError, Requirement
119 declared = cast("list[str]", table.get("requires", []))
120 try:
121 return [Requirement.parse(text, site=site) for text in declared]
122 except LockError as error:
123 fail(str(error))
126@functools.cache
127def base_requirements() -> tuple[Requirement, ...]:
128 """The tools the base requires in every workspace, at the site `BASE_SITE`.
130 Read from the `[toolroom] requires` of the base's contract file,
131 which spells them as an extension's `extension.toml` does: uv,
132 which makes the venv every step runs in, so a workspace with no
133 package requires it too.
135 Raises:
136 Failed: when an entry is not a requirement, naming it.
137 """
138 from livery.toolroom.store import TABLE
139 from livery.workshop._contract_keys import base_contract
141 data = base_contract()
142 return tuple(_requires(dict(data.get(TABLE, {})), site=BASE_SITE))
145def requirements(root: Path) -> tuple[Requirement, ...]:
146 """Every requirement the six sites declare, in site order.
148 The base's come first, at the site `BASE_SITE`: a workspace
149 requires uv with or without a package, since its own `tasks.py`
150 and tests run in the venv uv makes. The steps that judge a present
151 package bring the tools they run, each requirement naming
152 `step <name>` as its site, and so do the steps that judge the
153 root's own files, which are judged as a package holding the root's
154 set is.
155 An extension's site is `extension <name>`, read from its
156 ``extension.toml``'s ``[toolroom] requires`` and from each of its
157 ``[for.<target>]`` tables that applies now: cmake's build tools
158 are required while a package's set holds cmake, and a measurer's
159 tool while the workspace lists what it measures for. An extension
160 that neither the workspace lists nor a package's set holds
161 declares nothing here, since listing is the only activation
162 channel. A plugin the project mounts through its direct
163 dependencies is `plugin <name>`, read from its entry module
164 ([livery.workshop._tools.plugin_tools][]).
165 """
166 from livery.toolroom.store import LockError, Requirement
167 from livery.workshop._extensions import extension_tools
169 packages = discover_packages(root) if (root / "packages").is_dir() else ()
170 found: list[Requirement] = list(base_requirements())
171 # The steps that judge a present package bring their tools, each
172 # naming its step: unregistering a step removes its tool.
173 from livery.workshop._steps import tools_for
175 for tool, step in tools_for(_held(root, packages)):
176 found.append(Requirement.parse(tool, site=f"step {step}"))
177 try:
178 declared_by_extension = extension_tools(root)
179 except RuntimeError as error:
180 fail(str(error))
181 for extension, declared in declared_by_extension.items():
182 try:
183 found += [
184 Requirement.parse(text, site=f"extension {extension}")
185 for text in declared
186 ]
187 except LockError as error:
188 fail(str(error))
189 for plugin, declared in plugin_tools(root).items():
190 try:
191 found += [
192 Requirement.parse(text, site=f"plugin {plugin}") for text in declared
193 ]
194 except LockError as error:
195 fail(str(error))
196 for package in packages:
197 contract = package.directory / "workshop.toml"
198 found += _requires(tools_table(contract), site=f"{package.path}/workshop.toml")
199 found += _requires(tools_table(root / "workshop.toml"), site="workshop.toml")
200 return tuple(found)
203def _held(root: Path, packages: tuple[Package, ...]) -> tuple[str, ...]:
204 """The extensions the root's own files and the *packages* hold.
206 The root's own files, its `tasks.py` and tests, hold the root's
207 set, and are judged as a package holding it is.
208 """
209 from livery.workshop._composition import extension_set
210 from livery.workshop._extensions import root_set
212 held = (name for package in packages for name in extension_set(package))
213 return tuple(dict.fromkeys((*root_set(root), *held)))
216def plugin_tools(root: Path) -> dict[str, tuple[str, ...]]:
217 """The tools each plugin the project mounts declares, by its entry point name.
219 The plugins are the ``footman.builtin`` names the project's direct
220 dependencies offer, as footman's project rung mounts them. A plugin
221 declares the tools its verbs need in a data module that its
222 ``workshop.tools`` entry point names, under the plugin's own name:
223 ``"livery.forge" = "livery.forge._dev_tools:TOOLS"``, a tuple of
224 requirement strings, ``("docker?",)`` for one it can do without.
225 Loading it imports that module alone, never the plugin's tasks.
227 Raises:
228 Failed: when a declaration is not a tuple or list of strings,
229 naming the plugin.
230 """
231 from livery.footman import installed_entry_points, project_builtins
233 declared = {entry.name: entry for entry in installed_entry_points("workshop.tools")}
234 found: dict[str, tuple[str, ...]] = {}
235 for name in project_builtins(root):
236 entry = declared.get(name)
237 if entry is None:
238 continue
239 value: object = entry.load()
240 if not isinstance(value, (tuple, list)) or not all(
241 isinstance(text, str)
242 for text in value # pyright: ignore[reportUnknownVariableType]
243 ):
244 fail(
245 f"plugin {name}: its workshop.tools entry point ({entry.value}) is"
246 ' not a tuple of requirement strings, ("docker?",)'
247 )
248 found[name] = tuple(str(text) for text in value) # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType]
249 return found
252def host_allowed(root: Path) -> tuple[str, ...]:
253 """The tools whose copy on the machine may serve: the root contract's.
255 The root contract's `[toolroom] host-allowed`, each name once,
256 sorted. A name no site requires yet is kept and named by the lock
257 as an allowance ahead of its requirement: a workspace may allow
258 `cmake` before its first C++ package arrives, and the lock's line
259 is where a misspelling shows. A tool a step reads its verdict from
260 refuses, naming the steps that read it: a linter that varies by
261 machine makes the gate disagree with CI. `uv` refuses by name,
262 since the entry pins it. A tool an extension that writes release
263 notes requires refuses, naming the extension: a release's entries
264 must not depend on the machine. A download whose executable is
265 not named like the tool, or that has none, refuses too: the
266 allowance finds the tool on PATH by its name.
267 """
268 from livery.toolroom.store import LockError, Requirement
269 from livery.workshop._extensions import notes_tools
270 from livery.workshop._steps import step_for, tools_for
272 declared = cast(
273 "list[str]", tools_table(root / "workshop.toml").get("host-allowed", [])
274 )
275 names = tuple(sorted(set(declared)))
276 if not names:
277 return ()
278 packages = discover_packages(root) if (root / "packages").is_dir() else ()
279 verdicts: dict[str, set[str]] = {}
280 for tool, step in tools_for(_held(root, packages)):
281 if step_for(step).phase in VERDICT_PHASES:281 ↛ 280line 281 didn't jump to line 280 because the condition on line 281 was always true
282 verdicts.setdefault(tool, set()).add(step)
283 writers: dict[str, str] = {}
284 for extension, requires in notes_tools(root).items():
285 for line in requires:
286 try:
287 parsed = Requirement.parse(line, site=f"extension {extension}")
288 except LockError as error:
289 fail(str(error))
290 writers[parsed.name] = extension
291 for name in names:
292 where = f"workshop.toml: [toolroom] host-allowed names {name}"
293 if name in PINNED_TOOLS:
294 fail(f"{where}, which takes no allowance: the entry pins {name}")
295 if name in writers:
296 fail(
297 f"{where}, which takes no allowance: extension {writers[name]}"
298 " writes the release notes with it"
299 )
300 if name in verdicts:
301 readers = sorted(verdicts[name])
302 fail(
303 f"{where}, whose version is a verdict: {', '.join(readers)}"
304 f" {'reads' if len(readers) == 1 else 'read'} it"
305 )
306 return names
309def unrequired_allowances(root: Path) -> tuple[str, ...]:
310 """The host-allowed names no site requires yet, sorted: allowances ahead of need."""
311 required = {requirement.name for requirement in requirements(root)}
312 return tuple(name for name in host_allowed(root) if name not in required)
315def _host_probe_gap(listing: Catalogue, name: str, version: str, host: str) -> str:
316 """Why *name* cannot be found on PATH by its own name; empty when it can."""
317 from livery.toolroom.store import DOWNLOAD_KINDS, CatalogueError, RecordError
319 listed = listing.listed(name)
320 if listed.kind not in DOWNLOAD_KINDS:320 ↛ 321line 320 didn't jump to line 321 because the condition on line 320 was never true
321 return ""
322 try:
323 deployment = listing.deployment(name, version, host)
324 except (CatalogueError, RecordError):
325 return ""
326 if not deployment.entry_points:326 ↛ 327line 326 didn't jump to line 327 because the condition on line 326 was never true
327 return (
328 f"{name} has no executable; the allowance finds a tool on PATH by its name"
329 )
330 first = Path(deployment.entry_points[0]).name
331 if first.lower().removesuffix(".exe") != name:
332 return (
333 f"{name}'s executable is {first}; the allowance finds a tool on PATH by"
334 " its name"
335 )
336 return ""
339def tool_names(root: Path, host: str = "") -> tuple[str, ...]:
340 """The tools the sites require, each once, in the order first declared.
342 With *host*, the tools required there: a requirement scoped to other
343 hosts alone (``dotnet_coverage@windows``) requires nothing on it.
344 """
345 return tuple(
346 dict.fromkeys(
347 requirement.name
348 for requirement in requirements(root)
349 if not host or requirement.on((host,))
350 )
351 )
354def this_host() -> str:
355 """This machine's host key as the store names it: ``macos-arm``."""
356 import platform
358 from livery.toolroom.store import host_key
360 return host_key(platform.system(), platform.machine())
363def supported_hosts(root: Path) -> tuple[str, ...]:
364 """The hosts the workspace supports and locks for, in host-key order.
366 `[workspace] hosts` names them in the requirement scope tokens:
367 a platform (`macos`), a host key (`linux-arm`), and either with `!`
368 to remove it, a list of removals alone starting from every host.
369 Absent, every host key is supported.
371 Raises a refusal for a token that is neither a platform nor a host
372 key, and for a list that is empty or leaves no host.
373 """
374 from livery.toolroom.store import HOSTS, Scope, SpecError
376 every = DEFAULT_HOSTS or HOSTS
377 path = root / "workshop.toml"
378 table = load_contract(path).get(WORKSPACE) if path.is_file() else None
379 declared = cast("dict[str, object]", table or {}).get("hosts")
380 if declared is None:
381 return every
382 where = "workshop.toml: [workspace] hosts"
383 tokens = cast("list[str]", declared)
384 if not tokens:
385 fail(f"{where} is empty; name the hosts, or drop the key for every host")
386 try:
387 scope = Scope.parse(",".join(tokens), where=where, spelled=str(tokens))
388 except SpecError as error:
389 fail(str(error))
390 hosts = scope.hosts(every)
391 if not hosts:
392 fail(f"{where} {tokens} leaves no host; name at least one host")
393 return hosts
396def index_source(root: Path) -> str:
397 """Where the catalogue is read from, as `[toolroom] index` names it.
399 A URL is read as the published index; a path, relative to the root,
400 is a directory holding an index or the records that build one.
401 """
402 from livery.toolroom.store import CatalogueError
403 from livery.toolroom.store import index_source as declared_source
405 try:
406 return declared_source(tools_table(root / "workshop.toml"), root)
407 except CatalogueError as error:
408 fail(str(error))
411def has_index(root: Path) -> bool:
412 """Whether the root contract names a catalogue source at all."""
413 declared = tools_table(root / "workshop.toml").get("index")
414 return isinstance(declared, str) and bool(declared)
417def receipt_gap(tool: str, receipt: ToolReceipt, bin_dir: Path) -> str:
418 """Why what *receipt* claims does not hold here; empty when it does.
420 A receipt is the store's own record of what it installed for this
421 checkout, so it is the thing to check: looking for an executable by
422 a tool's name instead answers a different question, and answers it
423 wrongly for a tool that is not a program.
425 What a receipt claims depends on how the tool was supplied. One
426 that puts directories on PATH claims they exist and that each entry
427 point runs from one of them, the venv's own bin, or PATH, since a
428 name is not always a binary's (``git_cliff`` installs
429 ``git-cliff``). One that carries environment values claims each
430 value that is a path is there: the cmake-conan provider is a file
431 CMake reads, and no program by that name exists anywhere. One that
432 claims nothing was verified on the host rather than installed, so
433 the host must still answer for it.
435 Returns:
436 The first claim that does not hold, as a line to print after
437 the tool's name, or empty when every claim does.
438 """
439 for spelled in receipt.paths:
440 if not Path(spelled).is_dir():
441 return f"MISSING (the receipt's path {spelled} is not there)"
442 for name in receipt.entry_points:
443 if not _entry_point_runs(name, receipt, bin_dir):
444 return f"MISSING (the receipt names {name}, which does not run)"
445 for key, value in sorted(receipt.env.items()):
446 # A value that is a path is checked, and a path is judged by
447 # `is_absolute`: a Windows receipt names `C:\...`, which starts
448 # with a drive and not a slash.
449 named = Path(value)
450 if named.is_absolute() and not named.exists():
451 return f"MISSING ({key} names {value}, which is not there)"
452 # A receipt claiming nothing was verified on the host rather than
453 # installed, so the host has to answer for the tool now.
454 claims_nothing = not (receipt.paths or receipt.entry_points or receipt.env)
455 if claims_nothing and not (shutil.which(tool) or (bin_dir / tool).is_file()):
456 return "MISSING (verified on the host, and not on PATH now)"
457 return ""
460def _entry_point_runs(name: str, receipt: ToolReceipt, bin_dir: Path) -> bool:
461 """Whether *name* runs: from the receipt's paths, the venv's bin, or PATH."""
462 own = (Path(spelled) / name for spelled in receipt.paths)
463 return (
464 any(candidate.is_file() for candidate in own)
465 or (bin_dir / name).is_file()
466 or bool(shutil.which(name))
467 )
470def store_cannot_supply(root: Path) -> str:
471 """Why the store supplies nothing here; empty when it can.
473 The sites require tools whatever a contract says, because the
474 base requires uv: a workspace with no package still runs in the
475 venv uv makes. Nothing is supplied until a lock names versions,
476 and nothing can be locked until the contract names a catalogue to
477 resolve against, so the two are one question with one answer a
478 reader can act on.
480 Empty when a lock is present, and empty for a workspace that
481 requires no tool at all.
482 """
483 required = tool_names(root)
484 if not required or current_lock(root) is not None:
485 return ""
486 named = ", ".join(required[:4])
487 rest = f", and {len(required) - 4} more" if len(required) > 4 else ""
488 lines = [
489 f"the tools this workspace requires are not locked: {named}{rest}."
490 " Nothing is materialised from a store without a lock, so the"
491 " first verb that reaches for one fails with no executable found."
492 ]
493 if not has_index(root):
494 lines += [
495 "",
496 "Name the catalogue to resolve against in workshop.toml:",
497 "",
498 " [toolroom]",
499 ' index = "<the published index\'s URL, or a directory of records>"',
500 ]
501 lines += [
502 "",
503 f"Then `{prog()} toolroom.lock` writes the lock and"
504 f" `{prog()} sync` supplies what it names.",
505 ]
506 return "\n".join(lines)
509def catalogue(root: Path, *, offline: bool = False) -> Catalogue:
510 """The catalogue the repository resolves against, from `[toolroom] index`.
512 A directory of records is read as the authoring site reads it; an
513 index, by URL or directory, through the machine's store, which
514 keeps what it fetched so a second read is offline.
515 """
516 return _read_catalogue(index_source(root), offline=offline)
519#: The catalogues this process read, by source, mode and the source's
520#: identity: one read serves every caller in the process. A sync asks
521#: for the catalogue five times (the lock judged, the tools supplied,
522#: the stubs written); an index by URL answers each read over the
523#: network, and a records directory loads every record each time.
524_CATALOGUES: dict[tuple[Any, ...], Catalogue] = {}
527def _source_stamp(source: str) -> tuple[Any, ...] | None:
528 """What identifies the catalogue at *source* now.
530 None for an index by URL, whose pointer stands for the process. For
531 a directory index, the pointer file's modification time and size,
532 which the verbs that build an index move; for a records directory,
533 every record file's. A pointer that is not there is None too, and
534 the read refuses by name.
535 """
536 if "://" in source:536 ↛ 537line 536 didn't jump to line 537 because the condition on line 536 was never true
537 return None
538 from livery.toolroom.store import POINTER, records_in
540 directory = Path(source)
541 pointer = directory / POINTER
542 if pointer.is_file():
543 found = pointer.stat()
544 return (found.st_mtime_ns, found.st_size)
545 stamps = []
546 for path in records_in(directory):
547 try:
548 found = path.stat()
549 except OSError:
550 return None
551 stamps.append((path.name, found.st_mtime_ns, found.st_size))
552 return tuple(stamps) or None
555def forget_catalogues() -> None:
556 """Drop every catalogue this process read, so the next read is fresh."""
557 _CATALOGUES.clear()
560def _read_catalogue(source: str, *, offline: bool) -> Catalogue:
561 """The catalogue at *source*, records or index, with no build first.
563 A source is read once per process: a second read answers from the
564 first until the source's files change.
565 """
566 from livery.toolroom.store import Catalogue, CatalogueError
568 key = (source, offline, _source_stamp(source))
569 held = _CATALOGUES.get(key)
570 if held is not None:
571 return held
572 try:
573 listing = Catalogue.at(source, home=_home(), offline=offline)
574 except CatalogueError as error:
575 fail(str(error))
576 _CATALOGUES[key] = listing
577 return listing
580def _home() -> Home:
581 return store_home()
584def store_home() -> Home:
585 """The tool store's home on this machine, where every checkout's tools are.
587 The ``toolroom`` directory in the runner's data directory, one per
588 machine and shared by every checkout on it, as a
589 [livery.toolroom.store.Home][].
590 """
591 from livery.footman import data_dir
592 from livery.toolroom.store import Home
594 return Home.under(data_dir())
597def lock_path(root: Path) -> Path:
598 """The lock's file, `toolroom.lock` at the root."""
599 from livery.toolroom.store import LOCK_FILE
601 return root / LOCK_FILE
604def current_lock(root: Path) -> Lock | None:
605 """The lock as committed, or `None` when the repository has none yet."""
606 from livery.toolroom.store import Lock, LockError
608 path = lock_path(root)
609 if not path.is_file():
610 return None
611 try:
612 return Lock.load(path)
613 except LockError as error:
614 fail(str(error))
617def lock_is_current(root: Path, *, offline: bool = False) -> tuple[bool, str]:
618 """Whether `toolroom.lock` is what the sites resolve to now; and what moved.
620 The question `--locked` and `--check` ask, and nothing is written to
621 answer it. An entry that still satisfies every floor stands, so a
622 newer version published since the lock was written moves nothing:
623 that is what an upgrade is for. What moves the answer is a
624 requirement the lock does not hold, or holds at a version that no
625 longer satisfies.
627 The graphs are not compared: a delegated tool's graph is resolved
628 when its version enters the lock and written beside it, so a fresh
629 resolution never carries one, and reading its absence as a move
630 would move every delegated entry on every check.
632 Returns:
633 Whether the lock on disk is current, and why not when it is
634 not: no lock at all, a requirement nothing satisfies, or the
635 tools whose entries would move.
636 """
637 from livery.toolroom.store import LOCK_FILE, LockError, resolve_lock
639 held = current_lock(root)
640 if held is None:
641 return False, f"there is no {LOCK_FILE}"
642 listing = catalogue(root, offline=offline)
643 try:
644 fresh = resolve_lock(
645 listing,
646 with_runtimes(tuple(requirements(root)), listing),
647 hosts=supported_hosts(root),
648 keep=held,
649 host_allowed=host_allowed(root),
650 )
651 except LockError as error:
652 return False, str(error)
653 current, resolved = _entries(held), _entries(fresh)
654 if fresh.hosts == held.hosts and resolved == current:
655 return True, ""
656 names = set(resolved) | set(current)
657 moved = sorted(name for name in names if resolved.get(name) != current.get(name))
658 return False, f"the lock would move: {', '.join(moved)}"
661def _entries(lock: Lock) -> dict[str, dict[str, Any]]:
662 """The lock's entries without their graphs: what the current check compares."""
663 return {
664 name: {key: value for key, value in entry.to_json().items() if key != "graph"}
665 for name, entry in lock.tools.items()
666 }
669def write_lock(
670 root: Path,
671 *,
672 upgrade: tuple[str, ...] = (),
673 relock: tuple[str, ...] = (),
674 offline: bool = False,
675) -> Lock:
676 """Resolve the sites' requirements and write the lock; the lock written.
678 An entry the lock already holds stands unless it is named in
679 *upgrade* or no longer satisfies; a refusal names the tool, each
680 floor with its site, and the host at fault. A delegated tool whose
681 version entered the lock has its graph resolved with it, and one
682 named in *relock* has its graph resolved again though its version
683 stands, which is what an install the graph could not satisfy asks
684 for.
685 """
686 from livery.toolroom.store import LockError, resolve_lock
688 listing = catalogue(root, offline=offline)
689 kept = current_lock(root)
690 allowed = host_allowed(root)
691 try:
692 lock = resolve_lock(
693 listing,
694 with_runtimes(tuple(requirements(root)), listing),
695 hosts=supported_hosts(root),
696 keep=kept,
697 upgrade=upgrade,
698 host_allowed=allowed,
699 )
700 except LockError as error:
701 fail(str(error))
702 # What the resolution left out, an optional tool it could not serve.
703 for note in lock.notes:703 ↛ 704line 703 didn't jump to line 704 because the loop on line 703 never started
704 print(f" {note}")
705 for name in allowed:
706 if name not in lock.tools:
707 continue
708 for host in lock.tools[name].on or lock.hosts:
709 if gap := _host_probe_gap(listing, name, lock.tools[name].version, host):
710 fail(f"workshop.toml: [toolroom] host-allowed: {gap}")
711 lock, notes = with_graphs(root, lock, listing, kept=kept, relock=relock)
712 lock.save(lock_path(root))
713 for note in notes:
714 print(f" {note}")
715 for name in unrequired_allowances(root):
716 print(f" host-allowed: {name} is allowed and no site requires it yet")
717 for note in unreached_scopes(root, lock):
718 print(f" {note}")
719 return lock
722def unreached_scopes(root: Path, lock: Lock) -> list[str]:
723 """One line per scoped requirement no locked host matches, and so locks nothing."""
724 lines: list[str] = []
725 seen: set[str] = set()
726 for requirement in requirements(root):
727 if not requirement.hosts or requirement.name in lock.tools:
728 continue
729 spelled = str(requirement)
730 if spelled in seen or requirement.on(lock.hosts):730 ↛ 731line 730 didn't jump to line 731 because the condition on line 730 was never true
731 continue
732 seen.add(spelled)
733 lines.append(
734 f"{spelled} ({requirement.site}): no locked host is"
735 f" {', '.join(requirement.hosts)}; not locked"
736 )
737 return lines
740def resolve_graph(
741 root: Path,
742 name: str,
743 package: str,
744 version: str,
745 *,
746 kind: str,
747 runtime: str,
748) -> tuple[Graph | None, str]:
749 """Resolve *package* at *version* and write its graph; the graph or why not.
751 A `pypi` graph is uv's universal hashed requirements, resolved
752 against the workspace's Python floor, so one file covers every
753 interpreter the workspace supports. An `npm` graph is the
754 runtime's own lockfile, resolved without installing anything.
755 Neither runs unless what resolves it is here: a checkout that has
756 not materialised its tools yet locks the version and says the
757 graph waits.
758 """
759 from livery.strongroom import digest_of
760 from livery.toolroom.store import Graph
762 directory = graphs_dir(root)
763 directory.mkdir(parents=True, exist_ok=True)
764 target = directory / _graph_file(name, kind)
765 writer = _pypi_graph if kind == "pypi" else _npm_graph
766 by, why = writer(root, package, version, target, runtime=runtime)
767 if not by:
768 return None, why
769 return Graph(target.name, digest_of(target.read_bytes()), by=by), ""
772def _pypi_graph(
773 root: Path, package: str, version: str, target: Path, *, runtime: str
774) -> tuple[str, str]:
775 """Uv's universal hashed requirements for *package*; what resolved it."""
776 _ = runtime
777 import livery.toolroom.tools as toolroom
778 from livery.workshop._pythons import python_floor
780 with tempfile.TemporaryDirectory() as scratch:
781 wanted = Path(scratch) / "in.txt"
782 wanted.write_text(f"{package}=={version}\n", encoding="utf-8")
783 done = toolroom.uv.opts(nofail=True, recorded=False)(
784 "pip",
785 "compile",
786 "--generate-hashes",
787 "--universal",
788 "--quiet",
789 # The header names the command that wrote it, the output
790 # path included, and each annotation names the input file,
791 # whose directory is new on every run: either would make two
792 # resolutions of one version differ in bytes, and the lock
793 # would churn.
794 "--no-header",
795 "--no-annotate",
796 f"--python-version={python_floor(root)}",
797 f"--output-file={target}",
798 str(wanted),
799 )
800 if done.code != 0:800 ↛ 801line 800 didn't jump to line 801 because the condition on line 800 was never true
801 return "", f"uv could not resolve it (exit {done.code})"
802 return f"uv {_tool_version(root, 'uv')}", ""
805def _npm_graph(
806 root: Path, package: str, version: str, target: Path, *, runtime: str
807) -> tuple[str, str]:
808 """The runtime's own lockfile for *package*, resolved without installing."""
809 if runtime != "node":
810 return "", f"a graph of a {runtime} tool is not written yet"
811 held = tool_receipts(root).get("node")
812 if held is None:
813 return "", "node is not materialised here; the next lock writes it"
814 node = Path(held.tool_dir) / "bin" / exe("node")
815 if not node.is_file():
816 node = Path(held.tool_dir) / exe("node")
817 if not node.is_file():
818 return "", f"the node receipt names no executable under {held.tool_dir}"
819 from livery.toolroom.store import npm_cli
821 with tempfile.TemporaryDirectory() as scratch:
822 where = Path(scratch)
823 (where / "package.json").write_text(
824 json.dumps(
825 {
826 "name": "graph",
827 "version": "0.0.0",
828 "dependencies": {package: version},
829 }
830 )
831 + "\n",
832 encoding="utf-8",
833 )
834 done = run(
835 [
836 str(node),
837 str(npm_cli(node)),
838 "install",
839 "--package-lock-only",
840 "--no-audit",
841 "--no-fund",
842 ],
843 cwd=where,
844 nofail=True,
845 recorded=False,
846 capture=True,
847 )
848 written = where / "package-lock.json"
849 if done.code != 0 or not written.is_file():
850 return "", f"npm could not resolve it (exit {done.code})"
851 target.write_bytes(written.read_bytes())
852 return f"node {held.version}", ""
855def _tool_version(root: Path, name: str) -> str:
856 """The version the lock pins for *name*, or empty when it holds none."""
857 lock = current_lock(root)
858 held = lock.tools.get(name) if lock is not None else None
859 return held.version if held is not None else ""
862def exe(name: str) -> str:
863 """*name* as this platform spells an executable."""
864 return f"{name}.exe" if sys.platform == "win32" else name
867def _graph_path(root: Path, locked: Locked) -> Path | None:
868 """The graph file the lock names for this version, when it is here and whole.
870 A graph named but missing, or whose bytes are not the ones the
871 lock recorded, is not installed from: the tool installs the way
872 it did, and the next lock writes the graph again.
873 """
874 from livery.strongroom import digest_of
876 if locked.graph is None:
877 return None
878 path = graphs_dir(root) / locked.graph.file
879 if not path.is_file() or digest_of(path.read_bytes()) != locked.graph.digest:
880 return None
884def graphs_dir(root: Path) -> Path:
885 """Where the resolved graphs live, beside the lock."""
886 from livery.toolroom.store import GRAPHS
888 return root / GRAPHS
891def _graph_file(name: str, kind: str) -> str:
892 """The graph's file name for *name*: the installer's own format."""
893 return f"{name}.txt" if kind == "pypi" else f"{name}.json"
896def _kept_graph(
897 root: Path, name: str, locked: Locked, kept: Lock | None
898) -> Graph | None:
899 """The graph the lock already had for this version, when it still stands.
901 A graph is resolved once, when a version enters the lock, and kept
902 after: a lock names artefacts and their hashes, so an install
903 re-runs no resolution and a newer installer installs the same
904 graph. A version that moved, a graph never written, and a file
905 edited or gone since are each resolved again.
906 """
907 from livery.strongroom import digest_of
909 before = kept.tools.get(name) if kept is not None else None
910 if before is None or before.version != locked.version or before.graph is None:
911 return None
912 path = graphs_dir(root) / before.graph.file
913 if not path.is_file() or digest_of(path.read_bytes()) != before.graph.digest:
914 return None
915 return before.graph
918def with_graphs(
919 root: Path,
920 lock: Lock,
921 listing: Catalogue,
922 *,
923 kept: Lock | None = None,
924 relock: tuple[str, ...] = (),
925) -> tuple[Lock, list[str]]:
926 """*lock* with a graph per delegated tool, and what to say about it.
928 A `pypi` tool's graph is uv's hashed requirements, a `npm` tool's
929 is its runtime's lockfile, each written under `GRAPHS` beside the
930 lock and named there by its digest; a `dotnet` tool has none, since
931 its package carries its dependencies and the version pins the
932 whole. A graph that cannot be
933 resolved now, the runtime it needs being absent on a checkout
934 that has not materialised yet, leaves the tool as it was: it
935 installs the way it did and says so, and the next lock writes it.
937 A tool named in *relock* is resolved again though its version
938 stands. Nothing here re-resolves on its own: a resolve is a fresh
939 answer from an index, so it moves the lock, and a lock moves when
940 a person says so, never as a side effect of installing.
941 """
942 from livery.toolroom.store import CatalogueError
944 directory = graphs_dir(root)
945 tools: dict[str, Locked] = {}
946 notes: list[str] = []
947 wanted: set[str] = set()
948 for name, locked in lock.tools.items():
949 try:
950 kind = listing.listed(name).kind
951 except CatalogueError:
952 tools[name] = locked
953 continue
954 if kind not in ("pypi", "npm"):
955 tools[name] = locked
956 continue
957 standing = None if name in relock else _kept_graph(root, name, locked, kept)
958 if standing is not None:
959 wanted.add(standing.file)
960 tools[name] = replace(locked, graph=standing)
961 continue
962 listed = listing.listed(name)
963 graph, why = resolve_graph(
964 root,
965 name,
966 listed.package or name,
967 locked.version,
968 kind=kind,
969 runtime=runtime_of(listed),
970 )
971 if graph is None:
972 notes.append(f"graphs: {name} {locked.version}: {why}")
973 tools[name] = replace(locked, graph=None)
974 continue
975 wanted.add(graph.file)
976 notes.append(f"graphs: {name} {locked.version} resolved by {graph.by}")
977 tools[name] = replace(locked, graph=graph)
978 if directory.is_dir():
979 # A graph the lock no longer names is a file nothing installs
980 # from: it goes with the tool or the version it belonged to.
981 for stale in sorted(directory.iterdir()):
982 if stale.is_file() and stale.name not in wanted:
983 stale.unlink()
984 notes.append(f"graphs: removed {stale.name}")
985 return replace(lock, tools=tools), notes
988def runtime_of(listed: Listed) -> str:
989 """The runtime a tool runs on: an `npm` tool's record's, node unless it says bun.
991 A `dotnet` tool runs on dotnet, the SDK. Empty for every other
992 kind, which runs on nothing the lock supplies.
993 """
994 if listed.kind == "npm":
995 return listed.runtime or "node"
996 if listed.kind == "dotnet":
997 return "dotnet"
998 return ""
1001def with_runtimes(
1002 found: tuple[Requirement, ...], listing: Catalogue
1003) -> tuple[Requirement, ...]:
1004 """*found*, plus each runtime a tool among them runs on and nothing names.
1006 An `npm` or `dotnet` tool is installed through its runtime and
1007 runs on it, so the runtime is the tool's dependency and the lock
1008 holds it as such, on the hosts the tool is required on: the site
1009 named is the tool that needs it. A runtime some site requires on
1010 every host needs no more; otherwise one requirement per distinct
1011 scope is added, and the lock unions them. A requirement the
1012 catalogue does not list is left for the resolver to refuse by
1013 name.
1014 """
1015 from livery.toolroom.store import CatalogueError, Requirement
1017 everywhere = {requirement.name for requirement in found if not requirement.hosts}
1018 present = {(requirement.name, requirement.hosts) for requirement in found}
1019 added: list[Requirement] = []
1020 for requirement in found:
1021 try:
1022 listed = listing.listed(requirement.name)
1023 except CatalogueError:
1024 continue
1025 runtime = runtime_of(listed)
1026 if (
1027 not runtime
1028 or runtime in everywhere
1029 or (runtime, requirement.hosts) in present
1030 ):
1031 continue
1032 present.add((runtime, requirement.hosts))
1033 added.append(
1034 Requirement(
1035 runtime,
1036 site=f"{requirement.name} ({listed.kind})",
1037 hosts=requirement.hosts,
1038 )
1039 )
1040 return (*found, *added)
1043def declare(root: Path, text: str) -> bool:
1044 """Add *text* to the project's `[toolroom] requires`; whether the contract changed.
1046 A requirement already declared at the project site, in the same
1047 spelling, is left as it is. The contract is edited in place: the
1048 `[toolroom]` table gains the entry, or is added at the end with it.
1049 """
1050 from livery.toolroom.store import TABLE, LockError, Requirement
1052 try:
1053 Requirement.parse(text, site="workshop.toml")
1054 except LockError as error:
1055 fail(str(error))
1056 path = root / "workshop.toml"
1057 declared = tools_table(path).get("requires", [])
1058 if isinstance(declared, list) and text in declared:
1059 return False
1060 source = path.read_text(encoding="utf-8") if path.is_file() else ""
1061 lines = source.split("\n")
1062 header = next(
1063 (n for n, line in enumerate(lines) if line.strip() == f"[{TABLE}]"), None
1064 )
1065 if header is None:
1066 trimmed = source.rstrip("\n")
1067 path.write_text(
1068 (trimmed + "\n\n" if trimmed else "")
1069 + f'[{TABLE}]\nrequires = ["{text}"]\n',
1070 encoding="utf-8",
1071 )
1072 return True
1073 end = next(
1074 (n for n in range(header + 1, len(lines)) if lines[n].startswith("[")),
1075 len(lines),
1076 )
1077 for n in range(header + 1, end):
1078 if lines[n].split("=", 1)[0].strip() == "requires":
1079 if lines[n].rstrip().endswith("]"):
1080 body = lines[n].split("=", 1)[1].strip()[1:-1].strip()
1081 items = f"{body}, " if body else ""
1082 lines[n] = f'requires = [{items}"{text}"]'
1083 else:
1084 close = next(m for m in range(n + 1, end) if lines[m].strip() == "]")
1085 lines.insert(close, f' "{text}",')
1086 path.write_text("\n".join(lines), encoding="utf-8")
1087 return True
1088 lines.insert(header + 1, f'requires = ["{text}"]')
1089 path.write_text("\n".join(lines), encoding="utf-8")
1090 return True
1093# --- receipts and materialisation -------------------------------------------------
1095RECEIPTS = ".workshop/receipts"
1096"""Where a checkout keeps its receipts, one file per tool, gitignored."""
1098BIN = ".workshop/bin"
1099"""The checkout's bin directory: the entry points of every tool in `link` mode."""
1101RECEIPT_SCHEMA = 1
1102"""A receipt's shape. Bumped when a reader must know."""
1105@dataclass(frozen=True)
1106class ToolReceipt:
1107 """What this checkout installed of one tool, as its file records it.
1109 `fm sync` writes one receipt per tool under the checkout's
1110 ``.workshop/receipts/``; [livery.workshop.tool_receipts][] reads
1111 them all.
1113 Attributes:
1114 tool: The tool's name.
1115 version: The exact version installed, the lock's.
1116 host: The host key it was installed for, this machine's.
1117 kind: The installer kind.
1118 mode: How it reached PATH, one of
1119 [livery.toolroom.store.MODES][].
1120 deployment: The digest of the deployment installed; empty for a
1121 delegated kind, whose installer resolves the host.
1122 tool_dir: Where the tool is, under the machine's store home
1123 ([livery.workshop.store_home][]).
1124 paths: The absolute directories the tool puts on PATH in `path`
1125 mode; empty otherwise.
1126 env: The variables the tool sets, resolved against its directory.
1127 entry_points: The executables' names the tool puts on PATH:
1128 linked into the checkout's bin directory in `link` mode,
1129 found on `paths` in `path` mode; empty in `none` mode. A
1130 check resolves the tool by these, never by its name, since
1131 a name (`git_cliff`) is not always a binary (`git-cliff`).
1132 source: `store` for a tool the store supplied, `host` for a
1133 copy already on the machine that served under the
1134 allowance; `version` is the lock's either way, what the
1135 stubs render for and the drift check speaks of.
1136 answered: The version the host's copy printed when it served;
1137 empty for a store install.
1138 stamp: The host copy's identity when it served: its `path`,
1139 `size` and `mtime`, so a sync that finds the same file
1140 probes nothing; empty for a store install.
1141 """
1143 tool: str
1144 version: str
1145 host: str
1146 kind: str
1147 mode: str
1148 deployment: str = ""
1149 tool_dir: str = ""
1150 paths: tuple[str, ...] = ()
1151 env: dict[str, str] = field(default_factory=dict)
1152 entry_points: tuple[str, ...] = ()
1153 source: str = "store"
1154 answered: str = ""
1155 stamp: dict[str, str] = field(default_factory=dict)
1157 def to_json(self) -> dict[str, Any]:
1158 """The receipt as a JSON object."""
1159 return {
1160 "schema": RECEIPT_SCHEMA,
1161 "tool": self.tool,
1162 "version": self.version,
1163 "host": self.host,
1164 "kind": self.kind,
1165 "mode": self.mode,
1166 "deployment": self.deployment,
1167 "tool_dir": self.tool_dir,
1168 "paths": list(self.paths),
1169 "env": dict(self.env),
1170 "entry_points": list(self.entry_points),
1171 "source": self.source,
1172 "answered": self.answered,
1173 "stamp": dict(self.stamp),
1174 }
1176 @classmethod
1177 def load(cls, path: Path) -> ToolReceipt:
1178 """The receipt at *path*.
1180 Raises:
1181 ValueError: for a file that is not a receipt of this schema,
1182 naming the file.
1183 """
1184 try:
1185 data = json.loads(path.read_text(encoding="utf-8"))
1186 except (OSError, ValueError) as error:
1187 raise ValueError(f"{path}: not a receipt ({error})") from None
1188 if not isinstance(data, dict) or data.get("schema") != RECEIPT_SCHEMA:
1189 raise ValueError(f"{path}: not a receipt of schema {RECEIPT_SCHEMA}")
1190 try:
1191 return cls(
1192 str(data["tool"]),
1193 str(data["version"]),
1194 str(data["host"]),
1195 str(data["kind"]),
1196 str(data["mode"]),
1197 str(data.get("deployment", "")),
1198 str(data.get("tool_dir", "")),
1199 tuple(str(p) for p in data.get("paths", [])),
1200 {str(k): str(v) for k, v in data.get("env", {}).items()},
1201 tuple(str(e) for e in data.get("entry_points", [])),
1202 str(data.get("source", "store")),
1203 str(data.get("answered", "")),
1204 {str(k): str(v) for k, v in data.get("stamp", {}).items()},
1205 )
1206 except (KeyError, TypeError, AttributeError) as error:
1207 raise ValueError(f"{path}: not a receipt ({error})") from None
1210def _runtimes_first(
1211 wanted: tuple[str, ...], lock: Lock, listing: Catalogue
1212) -> tuple[str, ...]:
1213 """*wanted* with each runtime ahead of the tools that run on it.
1215 An `npm` tool is installed through its runtime's executable, so the
1216 runtime is supplied first and its path handed on; a runtime the
1217 lock holds joins a narrowed *wanted* that names such a tool without
1218 it.
1219 """
1220 needed: list[str] = []
1221 for name in wanted:
1222 runtime = runtime_of(listing.listed(name))
1223 if runtime and runtime not in needed:
1224 needed.append(runtime)
1225 if not needed:
1226 return wanted
1227 rest = tuple(name for name in wanted if name not in needed)
1228 return (*(runtime for runtime in needed if runtime in lock.tools), *rest)
1231def receipts_dir(root: Path) -> Path:
1232 """The checkout's receipts directory."""
1233 return root / RECEIPTS
1236def bin_dir(root: Path) -> Path:
1237 """The checkout's bin directory."""
1238 return root / BIN
1241def tool_receipts(root: Path) -> dict[str, ToolReceipt]:
1242 """What the checkout at *root* installed of each tool, by the tool's name.
1244 The receipts the last sync wrote, each a
1245 [livery.workshop.ToolReceipt][]: a receipt names the version the
1246 lock pins and where the tool is on this machine. Empty for a
1247 checkout no sync has supplied.
1249 Raises:
1250 ValueError: for a file under the receipts directory that is not
1251 a receipt, naming it.
1252 """
1253 directory = receipts_dir(root)
1254 if not directory.is_dir():
1255 return {}
1256 found = {}
1257 for path in sorted(directory.glob("*.json")):
1258 receipt = ToolReceipt.load(path)
1259 found[receipt.tool] = receipt
1260 return found
1263def mode_of(
1264 root: Path, name: str, kind: str, declared: str = "", *, paths: tuple[str, ...] = ()
1265) -> str:
1266 """The mode *name* materialises in: the project's say, the record's, the kind's.
1268 *paths* are the deployment's, which decide a download's default.
1269 Raises a refusal naming the override when it is not one of `MODES`.
1270 """
1271 from livery.toolroom.store import MODES, default_mode
1273 overrides = cast(
1274 "dict[str, str]", tools_table(root / "workshop.toml").get("modes", {})
1275 )
1276 chosen = overrides.get(name, declared) or default_mode(kind, paths)
1277 if chosen not in MODES:1277 ↛ 1278line 1277 didn't jump to line 1278 because the condition on line 1277 was never true
1278 fail(
1279 f"workshop.toml: [toolroom] modes names {chosen!r} for {name}; the modes"
1280 f" are {', '.join(MODES)}"
1281 )
1282 return str(chosen)
1285def sources(root: Path) -> tuple[Source, ...]:
1286 """The tiers consulted before an origin, `[toolroom] sources`: folders or URLs."""
1287 from livery.strongroom import FolderSource, HttpSource
1289 declared = cast("list[str]", tools_table(root / "workshop.toml").get("sources", []))
1290 found: list[Source] = []
1291 for entry in declared:
1292 if "://" in entry:1292 ↛ 1293line 1292 didn't jump to line 1293 because the condition on line 1292 was never true
1293 found.append(HttpSource(entry))
1294 else:
1295 found.append(FolderSource(root / entry))
1296 return tuple(found)
1299@dataclass(frozen=True)
1300class Materialised:
1301 """What one materialisation did to one tool.
1303 Attributes:
1304 receipt: The receipt written, or `None` when the tool could not
1305 be supplied and *strict* was off.
1306 installed: Whether the store installed it on this call.
1307 failure: Why the tool could not be supplied; empty when it was.
1308 note: What the sync should say beyond the receipt: why a host
1309 copy was passed over for the store's version; empty
1310 otherwise.
1311 """
1313 receipt: ToolReceipt | None
1314 installed: bool
1315 failure: str = ""
1316 note: str = ""
1319def _stamp(found: Path) -> dict[str, str]:
1320 """The identity of *found* a later sync compares before probing; empty when gone."""
1321 try:
1322 stat = found.stat()
1323 except OSError:
1324 return {}
1325 return {
1326 "path": str(found),
1327 "size": str(stat.st_size),
1328 "mtime": str(stat.st_mtime_ns),
1329 }
1332def _fresh_host_receipt(root: Path, name: str) -> ToolReceipt | None:
1333 """*name*'s receipt when a host copy served and the same file is still there."""
1334 path = receipts_dir(root) / f"{name}.json"
1335 if not path.is_file():
1336 return None
1337 try:
1338 receipt = ToolReceipt.load(path)
1339 except ValueError:
1340 return None
1341 if receipt.source != "host" or not receipt.stamp.get("path"):
1342 return None
1343 if _stamp(Path(receipt.stamp["path"])) != receipt.stamp:
1344 return None
1345 return receipt
1348def site_floors(root: Path) -> dict[str, tuple[str, str]]:
1349 """The highest floor the sites declare per tool, with the site that asked.
1351 A tool no site floors is absent. The lock already honours these
1352 floors when it chooses a version; the check of a system tool on a
1353 machine honours them here, since the machine's own copy is never
1354 the locked version.
1355 """
1356 from livery.toolroom.store import version_key
1358 highest: dict[str, tuple[str, str]] = {}
1359 for requirement in requirements(root):
1360 if not requirement.floor:
1361 continue
1362 held = highest.get(requirement.name)
1363 if held is None or version_key(requirement.floor) > version_key(held[0]):1363 ↛ 1359line 1363 didn't jump to line 1359 because the condition on line 1363 was always true
1364 highest[requirement.name] = (requirement.floor, requirement.site)
1365 return highest
1368def materialise(
1369 root: Path,
1370 names: tuple[str, ...] = (),
1371 *,
1372 offline: bool = False,
1373 strict: bool = True,
1374) -> tuple[Materialised, ...]:
1375 """Supply every locked tool, or *names* alone, and write their receipts.
1377 The bundle is what the sites require: every tool the lock holds
1378 for this host; one locked for other hosts alone is skipped, and
1379 named it refuses. A downloaded kind is supplied from the
1380 catalogue's deployment for
1381 this host through the store's sources and the origin; a delegated
1382 kind through its installer. A system tool is the machine's own,
1383 held to the highest of the record's floor and the sites' floors;
1384 the refusal names the site whose floor it is under. A tool the
1385 lock marks `allow-host` is looked for on PATH first: a copy that
1386 satisfies the floor serves, its receipt saying `host` and the
1387 version that answered, and the store installs nothing for it; a
1388 copy that is absent or below the floor is passed over with a
1389 note, and the locked version is supplied as for any other tool. A
1390 host receipt whose file is unchanged since it was written is
1391 reused without a probe. Tools in
1392 `link` mode are linked into the checkout's bin directory together,
1393 so a name two tools offer goes to the first. A receipt is written
1394 per tool supplied, and on a materialise of the whole bundle the
1395 receipt of a tool the lock no longer holds is removed, so its
1396 paths leave the environment with it.
1398 Raises a refusal naming the tool when the lock does not hold it.
1399 A tool the store cannot supply refuses too, naming the reason,
1400 unless *strict* is off: then it is reported in its outcome and the
1401 others are supplied, which is what `sync` wants, since one tool's
1402 origin being unreachable must not stop the environment from
1403 entering. A tool the lock marks optional is reported and never
1404 refused, strict or not: the verbs that need it name its absence
1405 when they run.
1406 """
1407 from livery.toolroom.store import (
1408 DOWNLOAD_KINDS,
1409 LOCK_FILE,
1410 RUNTIMES,
1411 CatalogueError,
1412 Store,
1413 StoreError,
1414 version_key,
1415 )
1417 lock = current_lock(root)
1418 if lock is None:
1419 fail(f"no {LOCK_FILE}: lock the tools first with `{prog()} toolroom.lock`")
1420 listing = catalogue(root, offline=offline)
1421 store = Store(_home(), sources=sources(root), offline=offline)
1422 host = store.host
1423 supported = supported_hosts(root)
1424 if host not in supported:
1425 # A runner's label is free text on every forge, so the host
1426 # itself is where an unsupported runner can be told apart.
1427 reason = (
1428 f"this host ({host}) is not one the workspace supports;"
1429 f" [workspace] hosts supports {', '.join(supported)}"
1430 )
1431 if strict or os.environ.get("CI"):
1432 fail(reason)
1433 print(f" {reason}")
1434 here = lock.on_host(host)
1435 wanted = names or here
1436 if not names:
1437 for name, entry in sorted(lock.tools.items()):
1438 if entry.optional and name not in here:1438 ↛ 1439line 1438 didn't jump to line 1439 because the condition on line 1438 was never true
1439 print(
1440 f" {name}: optional, and not locked for this host ({host});"
1441 " the verbs that need it say so"
1442 )
1443 for name in wanted:
1444 if name not in lock.tools:
1445 fail(
1446 f"{name} is not in {LOCK_FILE}; the lock holds"
1447 f" {', '.join(sorted(lock.tools)) or 'nothing'}"
1448 )
1449 if name not in here:
1450 fail(
1451 f"{name} is locked for {', '.join(lock.tools[name].on)} and not"
1452 f" for this host ({host}); the sites require it there alone"
1453 )
1454 floors = site_floors(root)
1455 done: list[Materialised] = []
1456 linked: list[tuple[ToolReceipt, Ensured]] = []
1457 wanted = _runtimes_first(wanted, lock, listing)
1458 runtimes: dict[str, Path] = {}
1459 for name in wanted:
1460 locked = lock.tools[name]
1461 listed = listing.listed(name)
1462 floor, site = listed.min_version, ""
1463 asked = floors.get(name) if listed.kind == "system-check" else None
1464 if asked is not None and (
1465 not floor or version_key(asked[0]) > version_key(floor)
1466 ):
1467 floor, site = asked
1468 note = ""
1469 if locked.allow_host:
1470 fresh = _fresh_host_receipt(root, name)
1471 if fresh is not None:
1472 if name in RUNTIMES:1472 ↛ 1473line 1472 didn't jump to line 1473 because the condition on line 1472 was never true
1473 runtimes[name] = Path(fresh.stamp["path"])
1474 done.append(Materialised(fresh, False))
1475 continue
1476 asked = floors.get(name)
1477 host_floor = asked[0] if asked is not None else listed.min_version
1478 try:
1479 served = store.supply(
1480 name, "system-check", locked.version, None, min_version=host_floor
1481 )
1482 except StoreError as error:
1483 note = f"{error}; the store's {locked.version} serves"
1484 else:
1485 found = served.tool_dir / exe(name)
1486 if name in RUNTIMES:1486 ↛ 1487line 1486 didn't jump to line 1487 because the condition on line 1486 was never true
1487 runtimes[name] = found
1488 receipt = ToolReceipt(
1489 name,
1490 locked.version,
1491 host,
1492 listed.kind,
1493 "none",
1494 tool_dir=str(served.tool_dir),
1495 source="host",
1496 answered=served.answered,
1497 stamp=_stamp(found),
1498 )
1499 done.append(Materialised(receipt, False))
1500 continue
1501 deployment: Deployment | None = None
1502 if listed.kind in DOWNLOAD_KINDS:
1503 try:
1504 deployment = listing.deployment(name, locked.version, host)
1505 except CatalogueError as error:
1506 # A host outside the lock's set, or one the version has
1507 # no build for: the tool is reported and the others are
1508 # supplied, as any tool the store cannot supply is.
1509 reason = f"{name} {locked.version}: not locked for {host}: {error}"
1510 if strict and not locked.optional:
1511 fail(reason)
1512 done.append(Materialised(None, False, reason))
1513 continue
1514 try:
1515 ensured = store.supply(
1516 name,
1517 listed.kind,
1518 locked.version,
1519 deployment,
1520 package=listed.package,
1521 min_version=floor,
1522 runtime=listed.runtime,
1523 runtime_exe=runtimes.get(runtime_of(listed)),
1524 graph=_graph_path(root, locked),
1525 )
1526 except StoreError as error:
1527 reason = str(error)
1528 if site and "below the floor" in reason:
1529 reason += f"; {site} requires {name}>={floor}"
1530 # An optional tool the host cannot supply is named, never
1531 # refused: the verbs that need it say so when they run.
1532 if strict and not locked.optional:
1533 fail(reason)
1534 done.append(Materialised(None, False, reason))
1535 continue
1536 if name in RUNTIMES and ensured.deployment.entry_points:
1537 runtimes[name] = ensured.tool_dir / ensured.deployment.entry_points[0]
1538 mode = mode_of(
1539 root,
1540 name,
1541 listed.kind,
1542 listed.mode,
1543 paths=deployment.paths if deployment is not None else (),
1544 )
1545 receipt = ToolReceipt(
1546 name,
1547 locked.version,
1548 host,
1549 listed.kind,
1550 mode,
1551 str(deployment.digest()) if deployment is not None else "",
1552 str(ensured.tool_dir),
1553 tuple(str(p) for p in ensured.paths) if mode == "path" else (),
1554 ensured.env,
1555 tuple(Path(e).name for e in ensured.deployment.entry_points)
1556 if mode in ("link", "path")
1557 else (),
1558 )
1559 if mode == "link":
1560 linked.append((receipt, ensured))
1561 done.append(Materialised(receipt, ensured.installed, note=note))
1562 if linked or bin_dir(root).is_dir():
1563 store.link([ensured for _, ensured in linked], bin_dir(root))
1564 receipts_dir(root).mkdir(parents=True, exist_ok=True)
1565 for made in done:
1566 if made.receipt is None:
1567 continue
1568 path = receipts_dir(root) / f"{made.receipt.tool}.json"
1569 path.write_text(
1570 json.dumps(made.receipt.to_json(), indent=2) + "\n", encoding="utf-8"
1571 )
1572 if not names:
1573 # A tool that left the lock, or is locked for other hosts
1574 # alone, leaves the environment: a receipt kept past its tool
1575 # would keep the tool's directory on PATH ahead of whatever
1576 # replaced it.
1577 for stale in receipts_dir(root).glob("*.json"):
1578 if stale.stem not in here:
1579 stale.unlink()
1580 return tuple(done)
1583def typings_dir(root: Path) -> Path:
1584 """The typings directory the stubs are written under."""
1585 return root / TYPINGS
1588def stubs_dir(root: Path) -> Path:
1589 """The `stubs` package under the typings directory's `livery.toolroom`."""
1590 return typings_dir(root).joinpath(*TYPINGS_PACKAGE) / "stubs"
1593def stubs_present(root: Path) -> int:
1594 """How many tool stubs the typings directory holds."""
1595 return sum(1 for p in stubs_dir(root).glob("*.pyi") if p.stem != "__init__")
1598@dataclass(frozen=True)
1599class Stubbed:
1600 """What `write_stubs` did.
1602 Attributes:
1603 written: The tools whose stub changed on disk, or was new.
1604 kept: The tools whose stub was already what the catalogue holds.
1605 skipped: Per tool the catalogue lists and no stub was written
1606 for, why.
1607 removed: Stubs of tools the catalogue no longer lists.
1608 """
1610 written: tuple[str, ...]
1611 kept: tuple[str, ...]
1612 skipped: dict[str, str]
1613 removed: tuple[str, ...]
1616def _no_command(listing: Catalogue, name: str, version: str) -> bool:
1617 """Whether *name* is a download with no entry point: no command, so no stub."""
1618 import platform
1620 from livery.toolroom.store import CatalogueError, RecordError, host_key
1622 if listing.listed(name).kind != "download":
1623 return False
1624 try:
1625 deployment = listing.deployment(
1626 name, version, host_key(platform.system(), platform.machine())
1627 )
1628 except (CatalogueError, RecordError):
1629 return False
1630 return not deployment.entry_points
1633def write_stubs(root: Path, *, offline: bool = False) -> Stubbed:
1634 """Write the stubs of the locked tools into the typings directory.
1636 One stub per tool `toolroom.lock` holds, rendered by the store from the
1637 locked version's own surface, from the records or from the index:
1638 a tool the workspace does not deploy gets no stub, and its handle
1639 types as a bare `Tool`. The `handles` module beside the stubs
1640 declares the handles, one import and one `name: Class[Result]` per
1641 stub written, and the installed tools package's index imports every
1642 name from it. The handles cannot live in the stubs package's own
1643 index: a package's index binds its submodules, and `uv` would name
1644 `uv.pyi` rather than the handle. Nothing is written inside the
1645 tools package's own directory, and a tree left there is removed:
1646 it shadows the package for a checker run on explicit paths. Without
1647 a lock nothing is written.
1649 A receipt under `.workshop/state/` names the lock and the source
1650 the last write rendered from, a records directory by its stat
1651 fingerprint and an index by its pointer. When both stand and every file it
1652 wrote is there, nothing is read and nothing is written, which is
1653 the steady state of every sync; otherwise a stub already on disk
1654 as the source renders it is kept, so a checker's cache stands.
1655 """
1656 from livery.toolroom.store import CatalogueError
1658 source = index_source(root)
1659 lock = current_lock(root)
1660 locked = dict(lock.tools) if lock is not None else {}
1661 shutil.rmtree(
1662 typings_dir(root).joinpath(*TYPINGS_PACKAGE, "tools"), ignore_errors=True
1663 )
1664 directory = stubs_dir(root)
1665 directory.mkdir(parents=True, exist_ok=True)
1666 seen = None
1667 if locked:
1668 try:
1669 seen = source_mark(source)
1670 except CatalogueError as error:
1671 fail(str(error))
1672 standing = _stubs_standing(root, seen, sorted(locked))
1673 if standing is not None:
1674 return standing
1675 listing = _read_catalogue(source, offline=offline) if locked else None
1676 written: list[str] = []
1677 kept: list[str] = []
1678 skipped: dict[str, str] = {}
1679 declared: list[str] = []
1680 for name in sorted(locked):
1681 version = locked[name].version
1682 assert listing is not None
1683 if _no_command(listing, name, version):
1685 try:
1686 text = listing.stub(name, version)
1687 except CatalogueError as error:
1688 skipped[name] = str(error)
1689 continue
1690 path = directory / f"{name}.pyi"
1691 if path.is_file() and path.read_text(encoding="utf-8") == text:
1692 kept.append(name)
1693 else:
1694 path.write_text(text, encoding="utf-8")
1695 written.append(name)
1696 declared.append(name)
1697 removed: list[str] = []
1698 for stale in sorted(directory.glob("*.pyi")):
1699 if stale.stem != "__init__" and stale.stem not in declared:
1700 stale.unlink()
1701 removed.append(stale.stem)
1702 _write_if_changed(directory / "__init__.pyi", "")
1703 _write_if_changed(handles_path(root), handles_index(declared))
1704 if seen is not None:
1705 _write_stubs_receipt(root, seen, sorted(locked), declared)
1706 return Stubbed(tuple(written), tuple(kept), skipped, tuple(removed))
1709def source_mark(source: str) -> str:
1710 """What the catalogue source is right now, in one digest, without reading it whole.
1712 A directory of records is its stat fingerprint; an index, by
1713 directory or URL, is the digest of its pointer document.
1714 """
1715 from livery.strongroom import canonical, digest_of
1716 from livery.toolroom.store import is_records, read_pointer, tree_fingerprint
1718 if is_records(source):
1719 return tree_fingerprint([source])
1720 return str(digest_of(canonical(read_pointer(source))))
1723STUBS_RECEIPT = ".workshop/state/stubs.json"
1724"""The receipt naming the lock and the source the last stub write rendered from.
1726Under `state/` rather than `receipts/`: that directory is the tool
1727receipts, which `tool_receipts()` reads as a set and refuses a file of
1728another shape in. Each kind of thing under `.workshop/` keeps its own
1729directory, so no walk of one kind can reach another.
1730"""
1733def _stubs_receipt_path(root: Path) -> Path:
1734 return root / STUBS_RECEIPT
1737def _lock_digest(root: Path) -> str:
1738 return "sha256:" + hashlib.sha256(lock_path(root).read_bytes()).hexdigest()
1741def _stubs_standing(root: Path, stubs: str, locked: list[str]) -> Stubbed | None:
1742 """The last write's answer when the lock, the source and the files stand."""
1743 try:
1744 held = json.loads(_stubs_receipt_path(root).read_text(encoding="utf-8"))
1745 except (OSError, ValueError):
1746 return None
1747 if not isinstance(held, dict) or held.get("schema") != 1:1747 ↛ 1748line 1747 didn't jump to line 1748 because the condition on line 1747 was never true
1748 return None
1749 if held.get("source") != stubs or held.get("lock") != _lock_digest(root):
1750 return None
1751 if held.get("locked") != locked:1751 ↛ 1752line 1751 didn't jump to line 1752 because the condition on line 1751 was never true
1752 return None
1753 written = held.get("written")
1754 if not isinstance(written, list) or not all(isinstance(n, str) for n in written):1754 ↛ 1755line 1754 didn't jump to line 1755 because the condition on line 1754 was never true
1755 return None
1756 # A locked tool with no stub is named on every write, so a write that
1757 # skipped one is never answered from the receipt.
1758 if written != locked:
1759 return None
1760 present = {p.stem for p in stubs_dir(root).glob("*.pyi")} - {"__init__"}
1761 if present != set(written) or not handles_path(root).is_file():
1762 return None
1763 return Stubbed((), tuple(written), {}, ())
1766def _write_stubs_receipt(
1767 root: Path, stubs: str, locked: list[str], written: list[str]
1768) -> None:
1769 path = _stubs_receipt_path(root)
1770 path.parent.mkdir(parents=True, exist_ok=True)
1771 document = {
1772 "schema": 1,
1773 "source": stubs,
1774 "lock": _lock_digest(root),
1775 "locked": locked,
1776 "written": written,
1777 }
1778 path.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8")
1781def handles_path(root: Path) -> Path:
1782 """The `handles.pyi` module beside the stubs, declaring the typed handles."""
1783 return typings_dir(root).joinpath(*TYPINGS_PACKAGE) / "handles.pyi"
1786def handles_index(names: list[str]) -> str:
1787 """The `handles` module declaring the handles of *names*, in order.
1789 The installed tools package's own index imports every name from
1790 this module, so the handle `uv` types as `Uv[Result]` exactly
1791 when `stubs/uv.pyi` is beside it.
1792 """
1793 from livery.toolroom.store import class_name
1795 lines = [
1796 f"# Rendered by `{prog()} toolroom.restub`: the handles this workspace",
1797 "# locks. Do not edit by hand.",
1798 "from livery.toolroom.tools import Result",
1799 ]
1800 lines += [
1801 f"from livery.toolroom.stubs.{name} import"
1802 f" {class_name(name)} as {class_name(name)}"
1803 for name in names
1804 ]
1805 lines.append("")
1806 lines += [f"{name}: {class_name(name)}[Result]" for name in names]
1807 return "\n".join(lines) + "\n"
1810def _write_if_changed(path: Path, text: str) -> None:
1811 if path.is_file() and path.read_text(encoding="utf-8") == text:
1812 return
1813 path.write_text(text, encoding="utf-8")
1816def stub_lines(root: Path, *, offline: bool = False, strict: bool = True) -> list[str]:
1817 """Write the stubs and say what happened, as `sync` and the lock verbs print it.
1819 A refusal is raised when *strict*; otherwise it is the one line
1820 returned, naming the reason, since a lock written or a bundle
1821 materialised stands whether or not the stubs could follow.
1822 """
1823 if strict:
1824 made = write_stubs(root, offline=offline)
1825 else:
1826 try:
1827 made = write_stubs(root, offline=offline)
1828 except Failed as refusal:
1829 return [f" stubs: not written: {refusal}"]
1830 held = len(made.written) + len(made.kept)
1831 line = f" stubs: {held} in {TYPINGS}/"
1832 if made.written:
1833 line += f", wrote {len(made.written)}"
1834 if made.removed:1834 ↛ 1835line 1834 didn't jump to line 1835 because the condition on line 1834 was never true
1835 line += f", removed {', '.join(made.removed)}"
1836 lines = [line]
1837 lines += [f" stubs: {name}: {why}" for name, why in made.skipped.items()]
1838 return lines
1841def emission(root: Path) -> tuple[tuple[str, ...], dict[str, str]]:
1842 """What the receipts add to an entered shell: PATH entries and variables.
1844 The checkout's bin directory first when any receipt links into it,
1845 then each `path` receipt's directories in tool order, then the
1846 variables, a later receipt's value winning.
1847 """
1848 held = tool_receipts(root)
1849 paths: list[str] = []
1850 env: dict[str, str] = {}
1851 if any(r.mode == "link" for r in held.values()):
1852 paths.append(str(bin_dir(root)))
1853 for receipt in held.values():
1854 for entry in receipt.paths:
1855 if entry not in paths:1855 ↛ 1854line 1855 didn't jump to line 1854 because the condition on line 1855 was always true
1856 paths.append(entry)
1857 env.update(receipt.env)
1858 return tuple(paths), env
1861def drift(root: Path) -> dict[str, str]:
1862 """Per tool the sites require, what `fm env.check` says of its receipt.
1864 An empty string is a tool whose receipt matches the lock; anything
1865 else names the problem: no receipt, a receipt at another version
1866 than the lock, or a deployment the lock has moved under it.
1867 """
1868 lock = current_lock(root)
1869 held = tool_receipts(root)
1870 found: dict[str, str] = {}
1871 for name in tool_names(root, this_host()):
1872 if lock is None or name not in lock.tools:
1873 found[name] = f"not locked; run `{prog()} toolroom.lock`"
1874 continue
1875 locked = lock.tools[name]
1876 receipt = held.get(name)
1877 if receipt is None:
1878 found[name] = (
1879 f"no receipt for {locked.version}; run `{prog()} sync` to materialise"
1880 )
1881 continue
1882 if receipt.version != locked.version:
1883 found[name] = (
1884 f"receipt {receipt.version}, lock {locked.version}; run `{prog()} sync`"
1885 )
1886 continue
1887 expected = locked.hosts.get(receipt.host)
1888 if receipt.source == "host":
1889 continue
1890 if expected is not None and str(expected) != receipt.deployment:
1891 found[name] = (
1892 f"DRIFT: the deployment of {locked.version} on {receipt.host} moved"
1893 f" under the receipt ({receipt.deployment[:19]}... is now"
1894 f" {str(expected)[:19]}...); run `{prog()} sync`"
1895 )
1896 continue
1897 found[name] = ""
1898 return found
1901def undeployed(name: str) -> NoReturn:
1902 """Refuse the task: *name* is a tool of the store and is not on ``PATH``.
1904 Call it where a step spawns a tool of the store and the spawn
1905 raises ``OSError``, or where ``shutil.which`` finds nothing for a
1906 tool the step spawns by its path. A toolroom handle spawns its tool
1907 by name from ``PATH``, and the store's tools are on ``PATH`` only
1908 after the machine has deployed them and the shell has entered the
1909 environment. A traceback from the spawn names nothing a person can
1910 act on. This refusal names the tool and the next action: run
1911 ``fm sync``, then the ``env.emit`` line it prints, then run the task
1912 again.
1914 It never returns.
1916 Args:
1917 name: The tool's name, as its handle spawns it.
1919 Raises:
1920 Failed: always, naming *name* and the sync that deploys it.
1921 """
1922 fail(
1923 f"{name} is not on PATH: it is a tool of the store, so enter the"
1924 f" environment (`{prog()} sync`, then the printed"
1925 " env.emit line) and re-run"
1926 )