1"""What the newborn's store says a run did: the rows the loop's proofs read.
3A proof asks the rows the legs wrote, never a job's log: a run's
4metrics row (each check leg's scope and the coverage section the gate
5job left), the verified series (the tree a run proved and what it
6rests on), a base's coverage record (which run measured each unit),
7the coverage marks, and a platform leg's durations series (the seconds
8per item a run's gate job recorded). A platform that ran as
9shards has one check row, the fold's, keyed by the platform leg with
10its shards listed under ``shard``; the forge's rows of the shards
11stand beside it with their times alone, and no proof reads them as
12the leg. Every reader fails naming the series, the file and the run
13when the row it needs is not there, so a proof that fails says what
14the legs did not write.
16The coverage rows are read by their ref and a few fields, since the
17loop lives in the base and the base imports no extension; their schema
18is the coverage extension's, and the loop leaves the base with phase
1915 of the destination note.
20"""
35 from pathlib import Path
37#: The coverage extension's record family and marks series, by ref alone.
44 """One shard of a folded check leg, as the fold listed it.
46 Attributes:
47 shard: The shard, ``1of2``.
48 leg: The shard's leg label (``check-ubuntu-latest-3.14-1of2``).
49 job: The shard's job name, as the forge lists it.
50 items: How many items the shard ran.
51 """
61 """The fold's record on a platform's check row: what its shards partitioned.
63 Attributes:
64 count: How many shards the platform ran as.
65 digest: The digest of the set the shards partitioned.
66 rows: The durations rows the shards partitioned by, newest
67 first; empty when no row was there to read.
68 shards: Each shard, in order.
69 """
79 """One job's half of a run's metrics row: what its gate ran.
81 Attributes:
82 job: The job's name, as the forge lists it.
83 scope: What the leg's gate ran: ``full``, ``affected``,
84 ``nothing``, ``verified``, or ``unknown`` when the leg
85 left no marker.
86 packages: The suites a narrowed gate ran.
87 base_tree: The tree a narrowed gate measured from; empty
88 otherwise.
89 leg: The leg's label, the key of its coverage record
90 (``check-ubuntu-latest-3.14``).
91 tasks: The tasks the job ran, by the names its trace recorded:
92 the top-level tasks and each task's steps.
93 shard: The shard whose own row this is (``1of2``), when the
94 gate job's fold did not fold it into its platform's row;
95 empty for a platform's row.
96 fold: What the fold recorded on a platform's row; ``None`` for
97 a leg that ran as one shard of one.
98 """
112 """A run's metrics row, read for a proof.
114 Attributes:
115 run: The run's id.
116 jobs: Each job's facts, by the job's name; the forge's rows of
117 a platform's shards are not among them.
118 coverage: The coverage section the gate job left: the percent
119 each unit was judged at, by the unit's path; empty when
120 the gate job judged nothing.
121 shard_jobs: The names of the forge's rows of a platform's
122 shards, kept beside the fold's row with their times alone.
123 """
131 """The one job whose name starts with *prefix*; fails naming the row.
133 The answer is the leg's own row: for a platform that ran as
134 shards, the fold's, never the forge's row of a shard. A row
135 that is one shard's own, or shard rows with no fold's row
136 among them, say the gate job's fold did not run, and the
137 answer fails naming them.
138 """
139 found = [facts for name, facts in self.jobs.items() if name.startswith(prefix)]
140 shards = [name for name in self.shard_jobs if name.startswith(prefix)]
141 if shards and not found:
142 fail(
143 f"run {self.run}: {_metrics.SERIES.ref}'s row names"
144 f" {', '.join(shards)} and no folded row of their platform; the"
145 " gate job's fold did not run"
146 )
147 own = [f"{facts.job} ({facts.shard})" for facts in found if facts.shard]
148 if own:
149 fail(
150 f"run {self.run}: {_metrics.SERIES.ref}'s row holds"
151 f" {', '.join(own)}, one shard's own row each, not the platform's;"
152 " the gate job's fold did not run"
153 )
154 if len(found) != 1:
155 fail(
156 f"run {self.run}: {_metrics.SERIES.ref}'s row names"
157 f" {len(found)} job(s) starting with {prefix!r}, expected one:"
158 f" {', '.join(sorted(self.jobs))}"
159 )
160 return found[0]
164 """The metrics row of *run_id*, fetched fresh from the newborn's origin.
166 Fails naming the series and the file when the row is not there:
167 the gate job's collect did not write it.
168 """
169 store = state_store(root)
170 name = _metrics.run_file(str(run_id))
171 with git.remote_snapshot(store, fetch=(_metrics.SERIES.name,)):
172 found = _metrics.SERIES.rows(store, only=lambda file: file == name)
173 if found.failed:173 ↛ 174line 173 didn't jump to line 174 because the condition on line 173 was never true
174 fail(f"run {run_id}: {_metrics.SERIES.ref} could not be read: {found.reason}")
175 row = next((item for item in found.rows if item.name == name), None)
176 if row is None:
177 fail(
178 f"run {run_id}: no row {name} on {_metrics.SERIES.ref}; the gate job's"
179 " collect did not write the run's row"
180 )
181 jobs: dict[str, LegFacts] = {}
182 shard_jobs: list[str] = []
183 for job, data in dict(row.data.get("jobs") or {}).items():
184 if forge_shard_row(str(job), data):
185 shard_jobs.append(str(job))
186 continue
187 scope = data.get("scope") or {}
188 tasks = (*data.get("tasks", {}), *data.get("task_steps", {}))
189 leg = str(scope.get("leg") or "")
190 own, fold = _shard_facts(leg, data.get("shard"))
191 jobs[str(job)] = LegFacts(
192 job=str(job),
193 scope=str(scope.get("scope", "unknown")),
194 packages=tuple(str(p) for p in scope.get("packages") or ()),
195 base_tree=str(scope.get("base_tree") or ""),
196 leg=leg,
197 tasks=tuple(str(t) for t in tasks),
198 shard=own,
199 fold=fold,
200 )
201 section = (row.data.get("sections") or {}).get("coverage") or {}
202 coverage = {
203 str(path): float(value)
204 for path, value in dict(section.get("packages") or {}).items()
205 }
206 return RunFacts(str(run_id), jobs, coverage, tuple(shard_jobs))
210 """The row's shard: its own, when it is one shard's row, or the fold's record.
212 A row whose ``shard`` lists ``shards`` is the fold's; one whose
213 ``shard`` is a shard's own record, or whose leg label ends in a
214 shard, is one shard's row the fold did not fold.
215 """
216 if isinstance(shard, dict) and "shards" in shard:
217 listed: Any = shard.get("shards")
218 entries = tuple(
219 ShardFacts(
220 str(item.get("shard") or ""),
221 str(item.get("leg") or ""),
222 str(item.get("job") or ""),
223 int(item.get("items") or 0),
224 )
225 for item in (listed if isinstance(listed, list) else [])
226 if isinstance(item, dict)
227 )
228 return "", FoldFacts(
229 int(shard.get("count") or 0),
230 str(shard.get("digest") or ""),
231 tuple(str(name) for name in shard.get("rows") or ()),
232 entries,
233 )
234 if isinstance(shard, dict):
235 return str(shard.get("shard") or ""), None
236 try:
237 own = _shard.shard_of(leg)
238 except ValueError:
239 return "", None
240 return ("" if own == _shard.ONE else str(own)), None
245 """What a platform leg's durations series holds for one run.
247 Attributes:
248 ref: The series' ref, ``durations/<platform leg>`` under the
249 store's namespace.
250 name: The row's file name, the run's.
251 items: The seconds per item the row holds; ``None`` when the
252 series holds no row of the run.
253 """
261 """The durations row *run_id*'s gate job wrote on *leg*'s platform, fetched fresh.
263 The series is the platform leg's ([livery.workshop.platform_leg][]),
264 so a shard's label reads its platform's. Fails naming the series
265 when it cannot be read; a series without the run's row answers
266 ``items`` as ``None``.
267 """
268 series = _shard.DURATIONS.series(_shard.platform_leg(leg))
269 name = _metrics.run_file(str(run_id))
270 store = state_store(root)
271 with git.remote_snapshot(store, fetch=(series.name,)):
272 found = series.rows(store, only=name.__eq__)
273 if found.failed:
274 fail(f"run {run_id}: {series.ref} could not be read: {found.reason}")
275 row = next((item for item in found.rows if item.name == name), None)
276 if row is None:
277 return DurationsRow(series.ref, name, None)
278 items = {
279 str(each): float(seconds)
280 for each, seconds in dict(row.data.get("items") or {}).items()
281 if isinstance(seconds, int | float) and not isinstance(seconds, bool)
282 }
283 return DurationsRow(series.ref, name, items)
287 """The verified row *run_id* stamped; fails naming the series when none did."""
288 rows, why = _verified.proved(root)
289 if why:289 ↛ 290line 289 didn't jump to line 290 because the condition on line 289 was never true
290 fail(f"run {run_id}: {_verified.SERIES.ref} could not be read: {why}")
291 for entry in rows.values():
292 if entry.run == str(run_id):292 ↛ 291line 292 didn't jump to line 291 because the condition on line 292 was always true
293 return entry
294 fail(
295 f"run {run_id}: no row on {_verified.SERIES.ref} names it; the gate job's"
296 " stamp did not record the tree as proved"
297 )
301 """The verified row of *tree*; fails naming the series when the record lacks it."""
302 entry, why = _verified.record(root, tree)
303 if why:303 ↛ 304line 303 didn't jump to line 304 because the condition on line 303 was never true
304 fail(f"tree {tree[:12]}: {_verified.SERIES.ref} could not be read: {why}")
305 if entry is None:
306 fail(f"tree {tree[:12]}: no row on {_verified.SERIES.ref}; nothing proved it")
307 return entry
312 """One unit of a base's coverage record: the run that measured it.
314 Attributes:
315 unit: The unit's path (``packages/loop-echo``, or ``tests``).
316 run: The run that measured it.
317 sha: The commit that run checked out.
318 closure: The closure identity the measurement is keyed by.
319 """
328 """*base*'s coverage record on *leg*: the current row of each unit, by path.
330 A current row is named by its unit's slug alone; a kept row, named
331 by its moment and closure as well, and a fixed-facts file under
332 ``facts/`` are not units of the record. The rows are read raw and
333 four fields taken, so the loop carries neither the coverage
334 extension's schema number nor its parser. Fails naming the ref when
335 it cannot be read; an absent record is empty.
336 """
337 series = COVERAGE_RECORD.series(base, leg)
338 store = state_store(root)
340 def current(name: str) -> bool:
341 return name.count(".") == 1 and "/" not in name
343 with git.remote_snapshot(store, fetch=(series.name,)):
344 found = git.read(store, series.ref, only=current)
345 if found.failed:345 ↛ 346line 345 didn't jump to line 346 because the condition on line 345 was never true
346 fail(f"{series.ref} could not be read: {found.reason}")
347 units: dict[str, UnitRow] = {}
348 for _name, text in sorted((found.files or {}).items()):
349 try:
350 data = json.loads(text)
351 except ValueError:
352 continue
353 if not isinstance(data, dict):353 ↛ 354line 353 didn't jump to line 354 because the condition on line 353 was never true
354 continue
355 unit = str(data.get("unit") or "")
356 if unit:356 ↛ 348line 356 didn't jump to line 348 because the condition on line 356 was always true
357 units[unit] = UnitRow(
358 unit,
359 str(data.get("run") or ""),
360 str(data.get("sha") or ""),
361 str(data.get("closure") or ""),
362 )
363 return units
368 """A package's current coverage mark: the newest row that set it.
370 Attributes:
371 package: The package path.
372 value: The mark, in percent.
373 kind: ``first``, ``ratchet`` or ``accept``.
374 by: The run id or the person that wrote the row.
375 """
384 """The current mark per package, the newest row each; fails when unreadable."""
385 store = state_store(root)
386 with git.remote_snapshot(store, fetch=(COVERAGE_MARKS.name,)):
387 found = COVERAGE_MARKS.rows(store)
388 if found.failed:388 ↛ 389line 388 didn't jump to line 389 because the condition on line 388 was never true
389 fail(f"{COVERAGE_MARKS.ref} could not be read: {found.reason}")
390 current: dict[str, MarkRow] = {}
391 for row in found.rows:
392 data: dict[str, Any] = row.data
393 package = str(data.get("package") or "")
394 if not package or package in current:
395 continue
396 try:
397 value = float(data["value"])
398 except (KeyError, TypeError, ValueError):
399 continue
400 current[package] = MarkRow(
401 package, value, str(data.get("kind") or ""), str(data.get("by") or "")
402 )
403 return current