1"""Steps in words: a tool and its arguments in a declaration, and their engine.
3A step whose verdict is its tool's exit code needs no code. Its
4declaration names the tool and the words it is called with: ``main``
5in its ``[phases.<phase>.<step>]`` table, and ``fix`` and ``safe-fix``
6for one that rewrites. The engine does the rest. It selects what the
7run reaches, as the step's scope and narrowing say; puts the paths and
8the words after ``--`` on the step's verb where the template stands
9them; splits a long path list into the fewest calls under the
10command-line limit; and runs the tool through its toolroom handle,
11which prints the tool's output. A non-zero exit refuses, and the
12refusals of every call merge into one.
14The template places both itself: ``{paths}`` stands where each batch
15of the paths goes, and a template without it runs whole, once with no
16path; ``{arguments}`` stands where the words after ``--`` go, and a
17template without it takes none. Each stands alone as one word, which
18the engine expands into many.
20A word may name a placeholder, ``{name}``, answered per call:
21``{cache}``, the step's own directory under ``.workshop/.cache/``;
22``{package}``, the directory of the package a call judges;
23``{compile-commands}``, the directory of that package's compilation
24database, without which the call does not run; and each key of
25``matrix``, which runs one call per value in parallel, each its own
26verdict. ``env`` gives every call variables over the run's own.
28A step the words cannot say, one whose verdict is not its tool's exit
29code or whose paths need reshaping first, declares a reference to its
30code instead: ``main`` as a string, ``module:function``.
31"""
33from __future__ import annotations
35import itertools
36import os
37import re
38from dataclasses import dataclass, replace
39from pathlib import Path
40from typing import TYPE_CHECKING
42from livery.footman import fail
44if TYPE_CHECKING:
45 from livery.workshop._packages import Package
46 from livery.workshop._phases import PhaseContext
47 from livery.workshop._steps import StepRecord
49#: The placeholders the engine answers, beside a step's matrix keys.
50PLACEHOLDERS = ("cache", "package", "compile-commands", "paths", "arguments")
52#: The placeholders only a call that judges one package can answer.
53PACKAGE_PLACEHOLDERS = ("package", "compile-commands")
55#: The words a step's template places itself, each one word of its own:
56#: the engine puts each batch of the run's paths where the first stands
57#: and the words after ``--`` where the second stands.
58PATHS_WORD = "{paths}"
59ARGUMENTS_WORD = "{arguments}"
60PLACED = (PATHS_WORD, ARGUMENTS_WORD)
62#: A placeholder in a word: ``{name}``, lower-case words joined by dashes.
63_PLACEHOLDER = re.compile(r"\{([a-z][a-z0-9-]*)\}")
66@dataclass(frozen=True)
67class Words:
68 """A step's words, as its declaration spells them.
70 Attributes:
71 judge: The tool's name, as the step's ``tools`` name it, and
72 the arguments that judge.
73 fix: The tool and the arguments that rewrite under ``--fix``;
74 empty for a step that only judges.
75 safe_fix: The tool and the arguments that rewrite under
76 ``--safe-fix``, removing no code; empty to rewrite with
77 ``fix``.
78 env: The variables every call gets, over the run's own, by
79 name.
80 matrix: Each placeholder the calls vary, with its values: one
81 call per combination.
82 """
84 judge: tuple[str, ...]
85 fix: tuple[str, ...] = ()
86 safe_fix: tuple[str, ...] = ()
87 env: tuple[tuple[str, str], ...] = ()
88 matrix: tuple[tuple[str, tuple[str, ...]], ...] = ()
90 @property
91 def tool(self) -> str:
92 """The tool every call runs, the first word of ``judge``."""
93 return self.judge[0]
95 def placeholders(self) -> frozenset[str]:
96 """Every placeholder the words name, in any mode or variable."""
97 texts = [*self.judge, *self.fix, *self.safe_fix]
98 texts += [value for _name, value in self.env]
99 return frozenset(name for text in texts for name in _PLACEHOLDER.findall(text))
101 def unanswered(self) -> tuple[str, ...]:
102 """The placeholders neither the engine nor the matrix answers, sorted."""
103 known = {*PLACEHOLDERS, *(key for key, _values in self.matrix)}
104 return tuple(sorted(self.placeholders() - known))
106 def unvaried(self) -> tuple[str, ...]:
107 """The matrix keys no word names, in the matrix's order."""
108 named = self.placeholders()
109 return tuple(key for key, _values in self.matrix if key not in named)
112@dataclass(frozen=True)
113class Command:
114 """One mode of a step in words, called as the step's ``run`` or ``fix``.
116 Attributes:
117 words: The step's words.
118 step: The step's name, ``lint.ruff``, which a run that names
119 no step judges by.
120 fixing: Whether this is the fix mode, which runs ``safe-fix``
121 under ``--safe-fix`` where the words declare it.
122 """
124 words: Words
125 step: str
126 fixing: bool = False
128 def argv(self, *, safe: bool = False) -> tuple[str, ...]:
129 """The words this mode runs: the tool's name, then its arguments."""
130 if not self.fixing:
131 return self.words.judge
132 return self.words.safe_fix if safe and self.words.safe_fix else self.words.fix
134 def __call__(self, ctx: PhaseContext) -> None:
135 """Run this mode over what *ctx* reaches; a non-zero exit refuses."""
136 run_words(self, ctx)
138 def __str__(self) -> str:
139 """The words this mode runs, as the declaration spells them."""
140 return " ".join(self.argv())
143def run_words(command: Command, ctx: PhaseContext) -> None:
144 """Run *command* over what the run reaches, as its step's scope and narrowing say.
146 A package step calls the tool on the package's files the run
147 reaches, from the package's directory, and calls nothing when there
148 are none. A workspace step that narrows by paths calls it on the
149 paths the run reaches, with none when the run is whole, and not at
150 all when the run reaches nothing it reads; one that narrows by
151 packages calls it once per package; any other calls it once, with no
152 path.
154 Raises:
155 Failed: when a call exits non-zero, every call's reason merged.
156 """
157 from livery.workshop._steps import (
158 PACKAGE,
159 PACKAGES,
160 PATHS,
161 WHOLE,
162 scoped_files,
163 scoped_packages,
164 scoped_paths,
165 step_for,
166 )
168 record = step_for(ctx.step or command.step)
169 ctx = replace(ctx, step=record.name)
170 argv = command.argv(safe=ctx.safe)
171 if record.scope == PACKAGE:
172 if ctx.package is None:172 ↛ 173line 172 didn't jump to line 173 because the condition on line 172 was never true
173 fail(f"{record.name} judges one package at a time, and this run names none")
174 files = tuple(str(path) for path in scoped_files(ctx, record.name))
175 if files:
176 _calls(record, command.words, argv, ctx, ctx.package, files)
177 return
178 if record.narrowing == PATHS:
179 chosen = scoped_paths(ctx, record.name)
180 if not chosen:
181 return # the run reaches nothing the step reads
182 _calls(
183 record, command.words, argv, ctx, None, () if chosen == WHOLE else chosen
184 )
185 return
186 if record.narrowing == PACKAGES:186 ↛ 187line 186 didn't jump to line 187 because the condition on line 186 was never true
187 for package in scoped_packages(ctx, record.name):
188 _calls(record, command.words, argv, ctx, package, ())
189 return
190 _calls(record, command.words, argv, ctx, None, ())
193def _calls(
194 record: StepRecord,
195 words: Words,
196 argv: tuple[str, ...],
197 ctx: PhaseContext,
198 package: Package | None,
199 paths: tuple[str, ...],
200) -> None:
201 """Run *argv* once per matrix combination, in parallel when there are several."""
202 from livery.footman import parallel, step
204 cwd = package.directory if package is not None else ctx.root
205 found = _answers(record, words, ctx.root, cwd, package)
206 if found is None:
207 return
208 combinations = _combinations(words.matrix)
210 def one(varied: dict[str, str]) -> None:
211 answers = {**found, **varied}
212 # A placed word stays as it is: the batching expands it.
213 arguments = tuple(
214 word if word in PLACED else _expand(record, word, answers)
215 for word in argv[1:]
216 )
217 variables = {name: _expand(record, value, answers) for name, value in words.env}
218 _batched(
219 argv[0], arguments, ctx.arguments, paths, cwd=cwd, ctx=ctx, env=variables
220 )
222 if len(combinations) == 1:
223 one(combinations[0])
224 return
225 parallel(
226 *(
227 step(one, title="_".join((record.step, *varied.values())))(varied)
228 for varied in combinations
229 )
230 )
233def _answers(
234 record: StepRecord,
235 words: Words,
236 root: Path,
237 cwd: Path,
238 package: Package | None,
239) -> dict[str, str] | None:
240 """The engine's answer to each placeholder the words name; None to run nothing.
242 A package with no compilation database answers nothing for
243 ``{compile-commands}``, and the call is skipped, saying so.
245 Raises:
246 Failed: when the words name a package's placeholder in a call
247 that judges no package.
248 """
249 from livery.workshop._queries import COMPILE_COMMANDS, answer
251 named = words.placeholders()
252 answers = {"cache": _relative(root / ".workshop" / ".cache" / record.step, cwd)}
253 for name in PACKAGE_PLACEHOLDERS:
254 if name in named and package is None:
255 fail(
256 f"{record.name} names {{{name}}}, which only a call that judges"
257 ' one package answers; declare the step with scope = "package"'
258 )
259 if package is not None:
260 answers["package"] = str(package.directory)
261 if "compile-commands" in named and package is not None:
262 database = answer(package, COMPILE_COMMANDS)
263 if database is None or not database.is_file():
264 print(
265 f" {record.name}: {package.path} has no compilation database; not run"
266 )
267 return None
268 answers["compile-commands"] = str(database.parent)
269 return answers
272def _relative(path: Path, cwd: Path) -> str:
273 """*path* as a call from *cwd* names it: relative where it can be."""
274 try:
275 return Path(os.path.relpath(path, cwd)).as_posix()
276 except ValueError: # another drive on Windows: no relative path exists
277 return str(path)
280def _combinations(
281 matrix: tuple[tuple[str, tuple[str, ...]], ...],
282) -> list[dict[str, str]]:
283 """Every combination of the matrix's values, by key; one empty one for no matrix."""
284 keys = [key for key, _values in matrix]
285 return [
286 dict(zip(keys, values, strict=True))
287 for values in itertools.product(*(values for _key, values in matrix))
288 ]
291def _expand(record: StepRecord, word: str, answers: dict[str, str]) -> str:
292 """*word* with each placeholder answered.
294 Raises:
295 Failed: when it names a placeholder nothing answers.
296 """
298 def answer(match: re.Match[str]) -> str:
299 name = match.group(1)
300 if name in ("paths", "arguments"):
301 fail(
302 f"{record.name} names {{{name}}} in {word!r}, which stands alone as"
303 " one word of a step's main under [phases.<phase>.<step>], where the"
304 " run's paths, or the words after --, go"
305 )
306 if name not in answers:
307 fail(
308 f"{record.name} names {{{name}}}, which neither the engine nor its"
309 f" matrix answers; the engine answers {', '.join(PLACEHOLDERS)}"
310 )
311 return answers[name]
313 return _PLACEHOLDER.sub(answer, word)
316def _batched(
317 tool: str,
318 words: tuple[str, ...],
319 arguments: tuple[str, ...],
320 paths: tuple[str, ...],
321 *,
322 cwd: Path,
323 ctx: PhaseContext,
324 env: dict[str, str],
325) -> None:
326 """Call *tool* with *words*, *arguments* and *paths*, batched under the limit.
328 The template takes each batch where ``{paths}`` stands and
329 *arguments* where ``{arguments}`` stands, and without ``{paths}``
330 runs whole, once with no path.
331 """
332 from livery.workshop._invoke import run_batched
334 def call(batch: tuple[str, ...]) -> None:
335 run_tool(tool, _placed(words, arguments, batch), cwd=cwd, ctx=ctx, env=env)
337 if paths and PATHS_WORD in words:
338 run_batched(paths, call)
339 else:
340 call(())
343def _placed(
344 words: tuple[str, ...], arguments: tuple[str, ...], batch: tuple[str, ...]
345) -> tuple[str, ...]:
346 """*words* with *batch* at ``{paths}`` and *arguments* at ``{arguments}``."""
347 found: list[str] = []
348 for word in words:
349 if word == PATHS_WORD:
350 found.extend(batch)
351 elif word == ARGUMENTS_WORD:
352 found.extend(arguments)
353 else:
354 found.append(word)
355 return tuple(found)
358def run_tool(
359 tool: str,
360 argv: tuple[str, ...],
361 *,
362 cwd: Path,
363 ctx: PhaseContext,
364 env: dict[str, str],
365) -> None:
366 """Call *tool*'s toolroom handle with *argv*, from *cwd*; a non-zero exit refuses.
368 The handle prints the tool's output. A call from the workspace root
369 runs in the run's own directory, and one with no variables in the
370 run's own environment, so neither is passed.
372 Raises:
373 Failed: when the tool exits non-zero, with the handle's reason.
374 """
375 import livery.toolroom.tools as tools
376 from livery.footman import RunFailed
378 handle = getattr(tools, tool)
379 chosen: dict[str, object] = {}
380 if cwd != ctx.root:
381 chosen["cwd"] = cwd
382 if env:
383 chosen["env"] = {**os.environ, **env}
384 try:
385 (handle.opts(**chosen) if chosen else handle)(*argv)
386 except (tools.ToolError, RunFailed) as error:
387 # A failed call refuses as a step's call does, so a batched
388 # run collects every batch's reason, the tool's output in each,
389 # before it refuses.
390 fail(str(error))
393def explained(record: StepRecord) -> list[str]:
394 """The lines ``fm explain <step>`` prints: what the step runs, and over what.
396 A step in words shows each mode's command as the tool's handle
397 spells it, the placeholders left for the engine to answer, then
398 what the engine puts where ``{paths}`` and ``{arguments}`` stand;
399 a step that runs code names its references.
400 """
401 from livery.workshop._steps import PACKAGE, PACKAGES, PATHS
403 lines = [f" {record.name}", f" extension: {record.extension}"]
404 if record.scope == PACKAGE:404 ↛ 405line 404 didn't jump to line 405 because the condition on line 404 was never true
405 lines.append(" scope: one package at a time, from its directory")
406 elif record.narrowing in (PATHS, PACKAGES):
407 lines.append(f" scope: the workspace, narrowed by {record.narrowing}")
408 else:
409 lines.append(" scope: the workspace, whole every run")
410 words = record.words
411 if words is None:
412 lines.append(f" main: {record.run} (code)")
413 if record.fix is not None:413 ↛ 414line 413 didn't jump to line 414 because the condition on line 413 was never true
414 lines.append(f" fix: {record.fix} (code)")
415 return lines
416 for mode, argv in (
417 ("main", words.judge),
418 ("fix", words.fix),
419 ("safe-fix", words.safe_fix),
420 ):
421 if argv:
422 lines.append(f" {mode}: {' '.join(_shown(argv))}")
423 for key, values in words.matrix:
424 lines.append(f" matrix: {key} = {', '.join(values)}, one call each")
425 for name, value in words.env:425 ↛ 426line 425 didn't jump to line 426 because the loop on line 425 never started
426 lines.append(f" env: {name}={value}")
427 if "cache" in words.placeholders():427 ↛ 428line 427 didn't jump to line 428 because the condition on line 427 was never true
428 lines.append(f" {{cache}}: .workshop/.cache/{record.step}")
429 if "paths" in words.placeholders():429 ↛ 432line 429 didn't jump to line 432 because the condition on line 429 was always true
430 reached = "files" if record.scope == PACKAGE else "paths"
431 lines.append(f" {{paths}}: the {reached} the run reaches, a batch per call")
432 if "arguments" in words.placeholders():432 ↛ 434line 432 didn't jump to line 434 because the condition on line 432 was always true
433 lines.append(" {arguments}: the words after --")
434 return lines
437def _shown(argv: tuple[str, ...]) -> list[str]:
438 """*argv* as the tool's handle spells its command, else as written."""
439 import livery.toolroom.tools as tools
441 handle = getattr(tools, argv[0], None)
442 if handle is None:442 ↛ 443line 442 didn't jump to line 443 because the condition on line 442 was never true
443 return list(argv)
444 return [str(word) for word in handle.argv(*argv[1:])]