1r"""The one dotenv implementation: parsing, substitution, the cascade.
3Every ``.repo.env`` in the wild is written against these rules:
5- **Precedence** (highest first, kind-major): the pre-existing
6 process environment, ``.repo.env.local`` files (nearest to
7 farthest), ``.repo.shared.env`` in the runner's config directory,
8 committed ``.repo.env`` files (nearest to farthest).
9- **Values**: ``~`` expands at the start of a raw value; double
10 quotes strip and allow substitution; single quotes strip and are
11 literal; an unquoted value loses inline ``#`` comments and
12 trailing whitespace.
13- **Substitution**: ``$VAR`` and ``${VAR}`` resolve from the stack
14 first, then the pre-existing environment, else empty; ``\$``
15 stays a literal dollar; iterated until stable.
16- **Provenance** is tracked per key (which extension supplied the
17 value); the display and the agent emission select on it.
18"""
20from __future__ import annotations
22import os
23import re
24from dataclasses import dataclass, field
25from enum import IntEnum
26from pathlib import Path
28#: Name suffixes whose values are masked in display output.
29SECRET_SUFFIXES = ("_TOKEN", "_SECRET", "_PASSWORD", "_KEY", "_API_KEY")
31#: The prefix under which the emitted entry step carries each declared
32#: key's CI secret; the emission strips it when it writes the shared slot.
33RUNG_PREFIX = "RUNG_"
35#: Matches a ``KEY=value`` assignment line in an env file.
36KEY_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)")
38_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
40_LITERAL = "\x00literal\x00"
41_ESCAPED_DOLLAR = "\x00dollar\x00"
44class Source(IntEnum):
45 """Where a stack value came from (lower is higher precedence)."""
47 environment = 0
48 """The pre-existing process environment: always wins."""
49 local = 1
50 """``.repo.env.local`` (per machine, gitignored)."""
51 shared = 2
52 """``.repo.shared.env`` in the runner's config directory."""
53 repo = 3
54 """``.repo.env`` (committed repo config)."""
57def is_secret_name(key: str) -> bool:
58 """Whether *key* names a secret by the suffix convention."""
59 return any(key.endswith(suffix) for suffix in SECRET_SUFFIXES)
62def _expand_tilde(value: str) -> str:
63 if value.startswith("~"):63 ↛ 64line 63 didn't jump to line 64 because the condition on line 63 was never true
64 return os.path.expanduser("~") + value[1:]
65 return value
68def _parse_raw(raw: str) -> str:
69 """Quote and comment handling for one raw value."""
70 double = re.match(r'^"(.*)"\s*(#.*)?$', raw)
71 if double:
72 return double.group(1)
73 single = re.match(r"^'(.*)'\s*(#.*)?$", raw)
74 if single:
75 return _LITERAL + single.group(1)
76 value = raw.split("#", 1)[0]
77 return value.rstrip()
80def parse_env_file(path: Path) -> dict[str, str]:
81 """Parse an env file into ``key -> raw value`` pairs.
83 Comments and blank lines are skipped, surrounding quotes
84 stripped, ``${VAR}`` left raw. The membership-only readers use
85 this; full resolution goes through
86 ``livery.workshop._envfile.load_cascade``. A missing file
87 answers an empty dict.
88 """
89 result: dict[str, str] = {}
90 if not path.is_file():
91 return result
92 for raw in path.read_text(encoding="utf-8").splitlines():
93 line = raw.strip()
94 if not line or line.startswith("#"):
95 continue
96 match = KEY_RE.match(line)
98 key = match.group(1)
99 value = match.group(2).strip()
100 if (value.startswith('"') and value.endswith('"')) or (
101 value.startswith("'") and value.endswith("'")
102 ):
103 value = value[1:-1]
104 result[key] = value
105 return result
108@dataclass
109class EnvStack:
110 """The resolved env-file stack: values plus per-key provenance."""
112 values: dict[str, str] = field(default_factory=dict)
113 sources: dict[str, Source] = field(default_factory=dict)
115 def managed(self) -> dict[str, str]:
116 """The keys the files supplied (everything not pre-existing)."""
117 return {
118 key: value
119 for key, value in self.values.items()
120 if self.sources[key] is not Source.environment
121 }
124def _substitute(value: str, lookup: dict[str, str], environ: dict[str, str]) -> str:
125 """One substitution pass over *value* (``$VAR`` and ``${VAR}``)."""
126 out: list[str] = []
127 i = 0
128 while i < len(value):
129 ch = value[i]
130 if ch == "\\" and i + 1 < len(value) and value[i + 1] == "$":
131 out.append(_ESCAPED_DOLLAR)
132 i += 2
133 continue
134 if ch != "$":
135 out.append(ch)
136 i += 1
137 continue
138 rest = value[i + 1 :]
139 if rest.startswith("{"):139 ↛ 140line 139 didn't jump to line 140 because the condition on line 139 was never true
140 match = _NAME_RE.match(rest, 1)
141 if match and rest[match.end() : match.end() + 1] == "}":
142 name = match.group(0)
143 out.append(lookup.get(name, environ.get(name, "")))
144 i += 2 + len(name) + 1
145 continue
146 out.append("${")
147 i += 2
148 continue
149 match = _NAME_RE.match(rest)
150 if match:150 ↛ 155line 150 didn't jump to line 155 because the condition on line 150 was always true
151 name = match.group(0)
152 out.append(lookup.get(name, environ.get(name, "")))
153 i += 1 + len(name)
154 continue
155 out.append("$")
156 i += 1
157 return "".join(out)
160def load_extension(
161 path: Path,
162 source: Source,
163 stack: EnvStack,
164 environ: dict[str, str],
165) -> None:
166 """Fold one file into *stack*; pre-existing environment wins."""
167 if not path.is_file():
168 return
169 for line in path.read_text(encoding="utf-8").splitlines():
170 stripped = line.strip()
171 if not stripped or stripped.startswith("#"):171 ↛ 172line 171 didn't jump to line 172 because the condition on line 171 was never true
172 continue
173 match = KEY_RE.match(stripped)
174 if match is None:174 ↛ 175line 174 didn't jump to line 175 because the condition on line 174 was never true
175 continue
176 key, raw = match.group(1), match.group(2)
177 env_value = environ.get(key, "")
178 if env_value:
179 stack.values[key] = _expand_tilde(env_value)
180 stack.sources[key] = Source.environment
181 continue
182 # Later loads override earlier ones; the caller feeds files
183 # from lowest precedence (repo) to highest (local).
184 stack.values[key] = _parse_raw(_expand_tilde(raw))
185 stack.sources[key] = source
188MAX_SUBSTITUTION_PASSES = 16
189"""How many times one value may be re-substituted before it is given up.
191Each pass resolves one level of reference, so a legitimate chain
192settles in a handful. Two keys referring to each other (``A=$B``
193beside ``B=$A``) never settle: the value alternates, so it never
194equals its own one-step predecessor and an unbounded loop runs
195forever. That loop is inside the cascade every ``fm`` command loads,
196so the bound is what keeps a mistyped pair from hanging the whole
197toolchain.
198"""
201def resolve_all(stack: EnvStack, environ: dict[str, str]) -> None:
202 """Substitute every value until stable; honour literals and escapes.
204 A value that has not settled within
205 ``livery.workshop._envfile.MAX_SUBSTITUTION_PASSES`` keeps
206 its raw text, unexpanded: showing ``$B`` is the answer a reader
207 can act on, and half an expansion would look like a resolved
208 value.
209 """
210 for key, value in list(stack.values.items()):
211 if value.startswith(_LITERAL):
212 stack.values[key] = value[len(_LITERAL) :]
213 continue
214 raw = value
215 previous = None
216 for _ in range(MAX_SUBSTITUTION_PASSES):
217 if value == previous:
218 break
219 previous = value
220 value = _substitute(value, stack.values, environ)
221 else:
222 value = raw.replace("\\$", "$")
223 stack.values[key] = value.replace(_ESCAPED_DOLLAR, "$")
226def cascade_dirs(repo_root: Path, cwd: Path) -> list[Path]:
227 """Directories from the repo ceiling down to *cwd*, farthest first.
229 Farthest first, so folding with later-wins gives the nearest
230 file precedence within its kind.
231 """
232 root = repo_root.resolve()
233 cur = cwd.resolve()
234 if root != cur and root not in cur.parents:
235 return [root]
236 dirs: list[Path] = []
237 while True:
238 dirs.append(cur)
239 if cur == root:
240 break
241 cur = cur.parent
242 dirs.reverse()
243 return dirs
246def load_cascade(
247 repo_root: Path,
248 cwd: Path,
249 *,
250 shared_dir: Path | None = None,
251 environ: dict[str, str] | None = None,
252) -> EnvStack:
253 """Load the multi-level env-file cascade, the one config mechanism.
255 ``.repo.env`` (committed) and ``.repo.env.local`` (machine,
256 secrets, gitignored) at every level from the repo ceiling down
257 to *cwd*. Precedence is kind-major: real environment, then
258 ``.local`` files nearest to farthest, then the shared file, then
259 committed files nearest to farthest, so a machine override beats
260 committed config at any depth.
262 *shared_dir* is where ``.repo.shared.env`` lives; None means the
263 runner's own config directory, asked of footman. Under CI the
264 entry's emission points ``WORKSHOP_SHARED_ENV_FILE`` at a
265 runner-local file it wrote from the job's secrets, and that file
266 IS the shared slot: same precedence, same masking, gone with the
267 runner. *environ*
268 defaults to ``os.environ`` and is injectable for tests.
269 """
270 env = dict(os.environ) if environ is None else environ
271 dirs = cascade_dirs(repo_root, cwd)
272 committed = [d / ".repo.env" for d in dirs]
273 local = [d / ".repo.env.local" for d in dirs]
275 stack = EnvStack()
276 for path in committed:
277 load_extension(path, Source.repo, stack, env)
278 load_extension(shared_file(shared_dir, env), Source.shared, stack, env)
279 for path in local:
280 load_extension(path, Source.local, stack, env)
281 resolve_all(stack, env)
282 return stack
285def quote_value(value: str) -> str:
286 """*value* spelled so the parser reads it back unchanged.
288 An unquoted value loses everything from its first ``#`` onward
289 and all trailing whitespace, so a value carrying either is
290 written inside double quotes, which the parser strips again. A
291 line break cannot be spelled on one line at all, the value would
292 read back as several lines with the second silently discarded,
293 so it is refused rather than written.
295 Raises:
296 ValueError: *value* contains a line break.
297 """
298 if "\n" in value or "\r" in value:
299 raise ValueError("a line break cannot be stored in an env file")
300 if "#" in value or value != value.strip():
301 return f'"{value}"'
302 return value
305def set_value(path: Path, key: str, value: str) -> None:
306 """Set ``key=value`` in *path*, preserving the file's other content.
308 The stack's one write primitive: creates the file and parents
309 when missing, replaces the key's line in place when present,
310 appends with a blank separator otherwise. The value is spelled
311 by ``livery.workshop._envfile.quote_value``, so what is read
312 back is what was set.
313 """
314 lines = path.read_text(encoding="utf-8").splitlines() if path.is_file() else []
315 new_line = f"{key}={quote_value(value)}"
316 for index, line in enumerate(lines):
317 match = KEY_RE.match(line)
318 if match and match.group(1) == key:
319 lines[index] = new_line
320 break
321 else:
322 if lines and lines[-1].strip():
323 lines.append("")
324 lines.append(new_line)
325 path.parent.mkdir(parents=True, exist_ok=True)
326 path.write_text("\n".join(lines) + "\n", encoding="utf-8")
329def shared_file(shared_dir: Path | None, environ: dict[str, str]) -> Path:
330 """The shared rung's file: the CI rung when the runner set one.
332 ``WORKSHOP_SHARED_ENV_FILE`` names a runner-local file the
333 entry's emission materialised from the repository's secrets; when
334 it is unset the rung is ``.repo.shared.env`` in *shared_dir*
335 (the runner's config directory when None).
336 """
337 override = environ.get("WORKSHOP_SHARED_ENV_FILE", "")
338 if override:
339 return Path(override)
340 if shared_dir is None:
341 import livery.footman as footman
343 shared_dir = footman.config_dir()
344 return shared_dir / ".repo.shared.env"
347def member_keys(repo_root: Path, cwd: Path, shared_dir: Path) -> set[str]:
348 """Every key the cascade's files define (membership, no values).
350 The one definition of which files make up the stack, for the
351 membership consumers (the agent emission, ``env.show``): the
352 same files ``livery.workshop._envfile.load_cascade`` folds
353 for a run from *cwd*.
354 """
355 keys: set[str] = set()
356 for directory in cascade_dirs(repo_root, cwd):
357 keys |= set(parse_env_file(directory / ".repo.env"))
358 keys |= set(parse_env_file(directory / ".repo.env.local"))
359 keys |= set(parse_env_file(shared_file(shared_dir, dict(os.environ))))
360 return keys