1"""A check leg's shard: its spelling, its share of the items, and the durations.
3A platform's check leg may run as several shards, each on a runner of
4its own, each running a share of the items a test step partitions;
5pytest's items are test files. A shard is spelled ``<i>of<n>`` at the
6end of the leg's label, ``check-<os>-<python>-2of4``;
7the label without it is the platform leg, the key every record below
8the gate job's fold is kept by. [livery.workshop.Shard][] parses and
9spells one, [livery.workshop.platform_leg][] and
10[livery.workshop.shard_of][] split a label.
12The share is a function of its inputs: [livery.workshop.partition][]
13packs the items longest first into the shard with the least summed
14seconds, the first shard's sum starting at the head, so every shard of
15a run computes the same partition from the same set, the same
16durations and the same head, and every item lands in exactly one
17shard. The durations and the head are the medians
18[livery.workshop.read_durations][] takes over the newest rows of the
19platform leg's series in the [livery.workshop.DURATIONS][] family,
20bounded by the checkout's committer time
21([livery.workshop.checkout_time][]) so a row written while the run is
22in flight reaches no shard. A shard leaves what it partitioned in
23`SHARD_FILE`, under the checkout's ``.workshop/`` directory
24([livery.workshop.leave_shard][]), and the gate job reads it back
25([livery.workshop.read_shard][]) to prove the shards of a platform
26partitioned one set from one row.
27"""
45#: The durations rows' schema. A reader skips a row of another version
46#: and names it; a new field is the same version, a changed meaning is
47#: the next.
50#: How many rows a platform leg's durations series keeps.
53#: How many of the newest rows before the bound a read takes the
54#: median over.
57#: The file a shard leaves under the checkout's ``.workshop/``
58#: directory, which git ignores, naming the shard it ran, the digest of
59#: the set it partitioned, its own items, the rows it read and the
60#: bound it read them under.
69 """One shard of a leg: which of how many, spelled ``<i>of<n>``.
71 Attributes:
72 index: The shard's number, from 1.
73 count: How many shards the leg runs as.
75 Raises:
76 ValueError: When the count is below 1, the index below 1, or
77 the index above the count.
78 """
84 why = _unsound(self.index, self.count)
85 if why:
86 raise ValueError(f"shard {self.index} of {self.count}: {why}")
90 """The shard *text* spells, ``2of4``.
92 Raises:
93 ValueError: When *text* is not of the form, naming it.
94 """
95 matched = _SPELLING.match(text)
96 if matched is None:
97 raise ValueError(
98 f"{text!r} is not a shard; a shard is spelled <i>of<n>, such as 2of4"
99 )
100 index, count = int(matched.group(1)), int(matched.group(2))
101 why = _unsound(index, count)
102 if why:
103 raise ValueError(f"{text!r} is not a shard: {why}")
104 return cls(index, count)
108 """Whether this is the first shard, the one that runs what is not split."""
109 return self.index == 1
112 return f"{self.index}of{self.count}"
116 if count < 1:
117 return "the count is at least 1"
118 if index < 1:
119 return "the index counts from 1"
120 if index > count:
121 return "the index is at most the count"
122 return ""
125#: The one shard of a leg that is not split.
130 """The leg *label* without its shard; a label without one is returned as it is.
132 ``check-windows-latest-3.14-2of4`` gives ``check-windows-latest-3.14``.
133 """
134 matched = _TAIL.match(label)
135 return matched.group("leg") if matched else label
139 """The shard the leg *label* ends in; `ONE` when it ends in none.
141 Raises:
142 ValueError: When the label ends in a shard spelling outside the
143 form, such as ``0of2``, naming it.
144 """
145 matched = _TAIL.match(label)
146 return Shard.parse(matched.group("shard")) if matched else ONE
150 items: Iterable[str],
151 durations: Mapping[str, float],
152 count: int,
153 *,
154 head: float = 0.0,
155) -> tuple[tuple[str, ...], ...]:
156 """Pack *items* into *count* shards by their *durations*, every item in exactly one.
158 The items go longest first, ties by name, each into the shard with
159 the least summed seconds so far, ties to the lowest shard. The
160 first shard's sum starts at *head* seconds, the time of the steps
161 it runs that the other shards leave to it, so its share of the
162 items is lighter by about that much; a head above every other
163 shard's final sum leaves the first shard no item. An item
164 *durations* does not name weighs the median of the named
165 durations, or one second when none are named. The result is a
166 function of the set, the durations and the head alone: the same
167 whatever order *items* comes in, and an item given twice is one
168 item. With more shards than items the surplus shards are empty.
170 Returns:
171 One tuple of items per shard, in the order the items were
172 packed.
174 Raises:
175 ValueError: When *count* is below 1, or *head* is below 0.
176 """
177 if count < 1:
178 raise ValueError(f"count is {count}; a partition has at least one shard")
179 if head < 0:
180 raise ValueError(f"head is {head}; a head is at least 0 seconds")
181 named = [float(seconds) for seconds in durations.values()]
182 fill = statistics.median(named) if named else 1.0
184 def weight(name: str) -> float:
185 seconds = durations.get(name)
186 return float(seconds) if seconds is not None else fill
188 ordered = sorted(set(items), key=lambda name: (-weight(name), name))
189 shards: list[list[str]] = [[] for _ in range(count)]
190 sums = [0.0] * count
191 sums[0] = float(head)
192 for name in ordered:
193 at = min(range(count), key=lambda index: (sums[index], index))
194 shards[at].append(name)
195 sums[at] += weight(name)
196 return tuple(tuple(shard) for shard in shards)
200 """A hex digest of the sorted names in *files*.
202 Two shards that partitioned one set have one digest, so the gate
203 job can prove they did.
204 """
205 text = "\n".join(sorted(set(files)))
206 return hashlib.sha256(text.encode("utf-8")).hexdigest()
210 """The keys the matrix produces, one per check leg; ``None`` when unknown.
212 The contract or the matrix cannot be read, and then the janitor
213 drops nothing.
214 """
215 from livery.workshop import check_legs
217 try:
218 legs = check_legs(root)
219 except (Exception, SystemExit):
220 return None
221 return {(git.slug(leg),) for leg in legs}
224#: The family: one series per platform leg, ``durations/<leg>``, one row
225#: per run, ``{"run": "<id>", "items": {"<item>": <seconds>}, "head":
226#: <seconds>}``, written by the run's gate job from its shards' rows.
227#: ``head`` is the item seconds the first shard's unsplit steps stand
228#: for on the run ([livery.workshop._metrics.first_shard_head][]); a row
229#: without it reads as 0. A series of a leg the matrix no longer
230#: produces is the janitor's to drop.
232 "durations", ("leg",), window=WINDOW, schema=SCHEMA, current=_current_legs
233)
237 """The time *stamp* names, timezone-aware; ``None`` when it names none."""
238 if not stamp:
239 return None
240 try:
241 moment = datetime.fromisoformat(stamp)
242 except ValueError:
243 return None
244 return moment if moment.tzinfo is not None else moment.replace(tzinfo=UTC)
248 root: Path, leg: str, *, before: datetime
249) -> tuple[dict[str, float], float, tuple[str, ...], str]:
250 """The median seconds per item, and the head, of *leg*'s rows before *before*.
252 The rows are the newest `NEWEST` of the series ``durations/<platform
253 leg>`` whose store stamp is before *before*; a row stamped at or
254 after it, or not stamped, is not read, so every shard of a run
255 reads the same rows whatever lands while the run is in flight. The
256 median of an item is over the rows that name it with a number
257 under ``items``. The head is the median of every row read's
258 ``head``, a row without a number there counting as 0, and is what
259 [livery.workshop.partition][] starts the first shard's sum at. A
260 naive *before* is read as UTC.
262 Returns:
263 The seconds per item, the head in seconds, the names of the
264 rows read, newest first, and why the series could not be read.
265 A series that cannot be read is ``({}, 0.0, (), reason)``,
266 never a raise: the caller runs as one shard of one and says
267 so. An absent series is ``({}, 0.0, (), "")``.
268 """
269 bound = before if before.tzinfo is not None else before.replace(tzinfo=UTC)
270 series = DURATIONS.series(platform_leg(leg))
271 found = series.rows(state_store(root))
272 if found.failed:
273 return {}, 0.0, (), found.reason
274 chosen: list[git.Row] = []
275 for row in found.rows:
276 when = _moment(row.when)
277 if when is None or when >= bound:
278 continue
279 chosen.append(row)
280 if len(chosen) == NEWEST:
281 break
282 seen: dict[str, list[float]] = defaultdict(list)
283 for row in chosen:
284 held: Any = row.data.get("items")
285 if not isinstance(held, dict):
286 continue
287 for name, seconds in held.items():
288 if isinstance(seconds, int | float) and not isinstance(seconds, bool):
289 seen[str(name)].append(float(seconds))
290 medians = {
291 name: round(statistics.median(values), 1)
292 for name, values in sorted(seen.items())
293 }
294 heads = [_seconds(row.data.get("head")) for row in chosen]
295 head = round(statistics.median(heads), 1) if heads else 0.0
296 return medians, head, tuple(row.name for row in chosen), ""
300 """*value* as seconds when it is a number of at least 0; else 0."""
301 if isinstance(value, bool) or not isinstance(value, int | float):
302 return 0.0
303 return max(0.0, float(value))
307 """The committer time of HEAD's commit in the checkout at *root*, timezone-aware.
309 Every shard of a run checks out the same commit, so this bounds
310 the durations rows every shard reads without a call to the forge.
312 Raises:
313 GitError: When HEAD names no commit.
314 """
315 spelled = git.run(root, "log", "-1", "--format=%cI", "HEAD").strip()
316 return datetime.fromisoformat(spelled)
321 """What a shard left in `SHARD_FILE`: what it ran and what it partitioned from.
323 Attributes:
324 shard: The shard the leg ran as.
325 digest: The digest of the set it partitioned,
326 [livery.workshop.files_digest][] of every item of every
327 shard.
328 items: The items this shard ran, in packing order; pytest's
329 are test files.
330 rows: The durations rows it read, newest first.
331 before: The bound it read them under, the checkout's committer
332 time.
333 suites: The units whose items were in the set, by path. A
334 unit the record does not name ran whole on the first shard,
335 with every step that does not read the shard, so the first
336 shard leaves its parts. A record written without the key
337 names none.
338 attempt: The attempt of the run the shard ran in, as the runner
339 counts them ([livery.workshop.RunContext][]'s ``attempt``);
340 empty when the runner did not say, and for a record written
341 without the key. Two shards of one platform from two
342 attempts partitioned from the record as it was each time,
343 which the fold names when their partitions differ.
344 """
356 """*marker* as `SHARD_FILE` holds it, and a leg's row under ``shard``."""
357 return {
358 "shard": str(marker.shard),
359 "digest": marker.digest,
360 "items": list(marker.items),
361 "rows": list(marker.rows),
362 "before": marker.before.isoformat(),
363 "suites": list(marker.suites),
364 "attempt": marker.attempt,
365 }
369 """The record *loaded* holds, as `marker_fields` wrote it; or why it is none.
371 *source* names where it came from, for the reason.
373 Returns:
374 ``(record, "")`` when it reads; ``(None, reason)`` when *loaded*
375 is not a record.
376 """
377 if not isinstance(loaded, dict):
378 return None, f"{source} is not a JSON object"
379 try:
380 shard = Shard.parse(str(loaded.get("shard", "")))
381 except ValueError as error:
382 return None, f"{source}: {error}"
383 stamp = str(loaded.get("before", "") or "")
384 before = _moment(stamp)
385 if before is None:
386 return None, f"{source}: {stamp!r} is not a time"
387 return ShardMarker(
388 shard=shard,
389 digest=str(loaded.get("digest", "") or ""),
390 items=tuple(str(name) for name in loaded.get("items", []) or []),
391 rows=tuple(str(name) for name in loaded.get("rows", []) or []),
392 before=before,
393 suites=tuple(str(name) for name in loaded.get("suites", []) or []),
394 attempt=str(loaded.get("attempt", "") or ""),
395 ), ""
399 root: Path,
400 *,
401 shard: Shard,
402 digest: str,
403 items: Iterable[str],
404 rows: Iterable[str],
405 before: datetime,
406 suites: Iterable[str] = (),
407 attempt: str = "",
408) -> Path:
409 """Leave the shard's record in `SHARD_FILE` at *root*, for the gate job's proof.
411 *suites* names the units whose items are in the set; a unit
412 left out ran whole on the first shard. *attempt* is the run's
413 attempt the shard ran in, the run context's; empty when unknown.
414 The leg's one write copies the record into its timing row under
415 ``shard``, where the gate job's fold reads it.
417 Returns:
418 The path of the file left.
419 """
420 path = root / SHARD_FILE
421 path.parent.mkdir(parents=True, exist_ok=True)
422 marker = ShardMarker(
423 shard, digest, tuple(items), tuple(rows), before, tuple(suites), attempt
424 )
425 path.write_text(json.dumps(marker_fields(marker), sort_keys=True), encoding="utf-8")
426 return path
430 """Read the record a shard left in `SHARD_FILE` at *root*.
432 Returns:
433 ``(record, "")`` when it reads; ``(None, "")`` when there is no
434 file; ``(None, reason)`` when the file is not a record.
435 """
436 path = root / SHARD_FILE
437 if not path.is_file():
438 return None, ""
439 try:
440 loaded: Any = json.loads(path.read_text("utf-8"))
441 except (OSError, ValueError) as error:
442 return None, f"{SHARD_FILE} does not parse ({error})"
443 return parse_marker(loaded, source=SHARD_FILE)