fuzzyworld-tools / files / server / blog_watch.py
blog_watch.py
| 1 | #!/usr/bin/env python3 |
| 2 | """`fw blog watch`: mirror a blog's source folder into a local git repo and deploy edits to published posts. |
| 3 | |
| 4 | blog_watch.py <config> [--once] [--dry-run] one tick (--once, what launchd runs every 60 s), or a loop |
| 5 | blog_watch.py install|uninstall <config> [--print] the launchd agent (--print only shows the plist) |
| 6 | |
| 7 | Run it on ONE machine. Per tick, for one blog config: take a lock, fingerprint the source folder, wait until the |
| 8 | folder has been quiet for `autodeploy.quiet_seconds` (default 300), then (1) mirror it into `mirror.repo` as a |
| 9 | commit on a separate branch, (2) build the site and deploy it only when the set of output files is the same as in |
| 10 | the last deploy and some file's content differs. Anything else (a new or removed page, no baseline, or a changed |
| 11 | generator, config or deploy target) is held back with a notification: run `fw blog deploy <config>` yourself. State: ~/.fw-blog-watch/<name>-<hash>/ (see blog.watch_name): |
| 12 | state.json, watch.log, deployed.json, hashes.json, lock, launchd.log. Rules and config keys are in docs/blog.md. |
| 13 | |
| 14 | Everything with side effects (clock, notification, running commands, the deploy) is injected through `Env`, so |
| 15 | the tests use fakes. Stdlib only. |
| 16 | """ |
| 17 | |
| 18 | import argparse |
| 19 | import dataclasses |
| 20 | import datetime |
| 21 | import hashlib |
| 22 | import json |
| 23 | import os |
| 24 | import plistlib |
| 25 | import shutil |
| 26 | import subprocess |
| 27 | import sys |
| 28 | import tempfile |
| 29 | import time |
| 30 | import traceback |
| 31 | |
| 32 | HERE = os.path.dirname(os.path.abspath(__file__)) |
| 33 | sys.path.insert(0, HERE) |
| 34 | import blog # noqa: E402 |
| 35 | |
| 36 | JUNK_NAMES = {".DS_Store", ".dropbox", ".dropbox.cache", ".git"} |
| 37 | RSYNC_JUNK = [".DS_Store", ".dropbox", ".dropbox.cache", "*conflicted copy*", ".git"] |
| 38 | DEFAULT_QUIET = 300 |
| 39 | TICK_SECONDS = 60 |
| 40 | GIT_TIMEOUT, RSYNC_TIMEOUT, NOTIFY_TIMEOUT = 120, 600, 10 |
| 41 | STUCK_AFTER = 1800 # a lock held this long (seconds) is reported as a stuck tick |
| 42 | LABEL = "net.fuzzyworld.fw-blog-watch-" |
| 43 | LAUNCH_PATH = "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin" |
| 44 | |
| 45 | |
| 46 | def is_conflict(name): |
| 47 | return "conflicted copy" in name.lower() |
| 48 | |
| 49 | |
| 50 | def is_junk(name): |
| 51 | return name in JUNK_NAMES or is_conflict(name) |
| 52 | |
| 53 | |
| 54 | # ---------------------------------------------------------------- injected effects |
| 55 | |
| 56 | def notify_macos(message, title="fw blog watch"): |
| 57 | """A macOS notification; any failure is ignored. FW_BLOG_WATCH_NOTIFY=0 turns it off (the tests do).""" |
| 58 | if os.environ.get("FW_BLOG_WATCH_NOTIFY") == "0": |
| 59 | return |
| 60 | def quote(text): |
| 61 | return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"' |
| 62 | try: |
| 63 | subprocess.run(["osascript", "-e", f"display notification {quote(message[:230])} with title {quote(title)}"], |
| 64 | capture_output=True, timeout=NOTIFY_TIMEOUT) |
| 65 | except Exception: |
| 66 | pass |
| 67 | |
| 68 | |
| 69 | def run_command(cmd, cwd=None, timeout=GIT_TIMEOUT): |
| 70 | """subprocess.run with captured text output. git never asks anything on the terminal. A command that runs |
| 71 | too long is killed and reported as a failure (exit 124), like any other.""" |
| 72 | try: |
| 73 | return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="surrogateescape", |
| 74 | stdin=subprocess.DEVNULL, timeout=timeout, |
| 75 | env={**os.environ, "GIT_TERMINAL_PROMPT": "0"}) |
| 76 | except subprocess.TimeoutExpired: |
| 77 | return subprocess.CompletedProcess(cmd, 124, "", f"{cmd[0]} timed out after {timeout} s") |
| 78 | |
| 79 | |
| 80 | def build_site(cfg, out): |
| 81 | return blog.build(cfg, out) |
| 82 | |
| 83 | |
| 84 | def deploy_built(cfg, out): |
| 85 | """The same function `fw blog deploy` uses, without the question; it audits `out` and runs rsync.""" |
| 86 | return blog.deploy_site(cfg, out, prebuilt=True, locked=True, timeout=RSYNC_TIMEOUT) # the tick holds the lock |
| 87 | |
| 88 | |
| 89 | @dataclasses.dataclass |
| 90 | class Env: |
| 91 | now: object = time.time # () -> seconds since the epoch |
| 92 | notify: object = notify_macos # (message) -> None |
| 93 | run: object = run_command # (argv, cwd=None, timeout=...) -> object with returncode, stdout, stderr |
| 94 | build: object = build_site # (cfg, out) -> None |
| 95 | deploy: object = deploy_built # (cfg, out) -> None, raises when the upload fails |
| 96 | generator: object = lambda: blog.GENERATOR_HASH # () -> hash of the code loaded now (fixed at import) |
| 97 | echo: object = print # where --dry-run output goes |
| 98 | |
| 99 | |
| 100 | def iso(seconds): |
| 101 | return datetime.datetime.fromtimestamp(seconds, datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") |
| 102 | |
| 103 | |
| 104 | # ---------------------------------------------------------------- fingerprint |
| 105 | |
| 106 | def file_hash(path): |
| 107 | h = hashlib.sha256() |
| 108 | with open(path, "rb") as f: |
| 109 | for chunk in iter(lambda: f.read(1 << 20), b""): |
| 110 | h.update(chunk) |
| 111 | return h.hexdigest() |
| 112 | |
| 113 | |
| 114 | def scan(src, cache=None): |
| 115 | """(fingerprint, conflict paths) of a source folder. With a `cache` dict ({path: [size, mtime_ns, inode, ctime_ns, |
| 116 | sha256]}, updated in place) a file whose first four values are unchanged is not read again. The fingerprint is a sha256 over the sorted (relative path, |
| 117 | content sha256) pairs of every file except junk (.DS_Store, .dropbox*, .git, Dropbox "conflicted copy" files). |
| 118 | Symbolic links are not followed: a link counts by its target text.""" |
| 119 | entries, conflicts, fresh = [], [], {} |
| 120 | for dirpath, dirs, files in os.walk(src): |
| 121 | keep = [] |
| 122 | for name in sorted(dirs): |
| 123 | path = os.path.join(dirpath, name) |
| 124 | rel = os.path.relpath(path, src).replace(os.sep, "/") |
| 125 | if is_conflict(name): |
| 126 | conflicts.append(rel) |
| 127 | elif name in JUNK_NAMES: |
| 128 | pass |
| 129 | elif os.path.islink(path): |
| 130 | entries.append((rel, "link:" + os.readlink(path))) |
| 131 | else: |
| 132 | keep.append(name) |
| 133 | dirs[:] = keep |
| 134 | for name in sorted(files): |
| 135 | path = os.path.join(dirpath, name) |
| 136 | rel = os.path.relpath(path, src).replace(os.sep, "/") |
| 137 | if is_conflict(name): |
| 138 | conflicts.append(rel) |
| 139 | elif is_junk(name): |
| 140 | continue |
| 141 | elif os.path.islink(path): |
| 142 | entries.append((rel, "link:" + os.readlink(path))) |
| 143 | else: |
| 144 | try: |
| 145 | st = os.stat(path) |
| 146 | stamp = [st.st_size, st.st_mtime_ns, st.st_ino, st.st_ctime_ns] |
| 147 | old = cache.get(rel) if isinstance(cache, dict) else None |
| 148 | if isinstance(old, list) and len(old) == 5 and old[:4] == stamp and isinstance(old[4], str): |
| 149 | digest = old[4] |
| 150 | else: |
| 151 | digest = file_hash(path) |
| 152 | fresh[rel] = stamp + [digest] |
| 153 | entries.append((rel, digest)) |
| 154 | except FileNotFoundError: # Dropbox moved it while we looked: the next tick sees the result |
| 155 | continue |
| 156 | except OSError as e: |
| 157 | entries.append((rel, f"unreadable:{e.errno}")) |
| 158 | if cache is not None: |
| 159 | cache.clear() |
| 160 | cache.update(fresh) |
| 161 | text = "\n".join(f"{rel}\0{digest}" for rel, digest in sorted(entries)) |
| 162 | return hashlib.sha256(text.encode("utf-8", "surrogateescape")).hexdigest(), sorted(conflicts) |
| 163 | |
| 164 | |
| 165 | # ---------------------------------------------------------------- state and log |
| 166 | |
| 167 | def read_json(path, default): |
| 168 | try: |
| 169 | with open(path, encoding="utf-8") as f: |
| 170 | data = json.load(f) |
| 171 | return data if isinstance(data, type(default)) else default |
| 172 | except (OSError, ValueError): |
| 173 | return default |
| 174 | |
| 175 | |
| 176 | def save_json(sd, name, data): |
| 177 | tmp = os.path.join(sd, name + ".tmp") |
| 178 | with open(tmp, "w", encoding="utf-8") as f: |
| 179 | json.dump(data, f, indent=1) |
| 180 | os.replace(tmp, os.path.join(sd, name)) |
| 181 | |
| 182 | |
| 183 | def save_state(sd, state): |
| 184 | save_json(sd, "state.json", state) |
| 185 | |
| 186 | |
| 187 | def current(sd, cfg): |
| 188 | """(fingerprint, conflicts) of the source now, using (and refreshing) the hash cache in the state folder.""" |
| 189 | cache = read_json(os.path.join(sd, "hashes.json"), {}) # a damaged cache file is just an empty one |
| 190 | result = scan(cfg["_source"], cache) |
| 191 | save_json(sd, "hashes.json", cache) |
| 192 | return result |
| 193 | |
| 194 | |
| 195 | def changed_since(sd, cfg, state): |
| 196 | return current(sd, cfg) != (state.get("fingerprint"), state.get("conflicts")) |
| 197 | |
| 198 | |
| 199 | def log(sd, env, message): |
| 200 | with open(os.path.join(sd, "watch.log"), "a", encoding="utf-8") as f: |
| 201 | f.write(f"{iso(env.now())} {message}\n") |
| 202 | |
| 203 | |
| 204 | def once(sd, env, state, key, message, notify=True): |
| 205 | """Log (and notify) a message the first time `key` happens after a change of the source, not on every tick.""" |
| 206 | seen = state.setdefault("notified", []) |
| 207 | if key in seen: |
| 208 | return |
| 209 | seen.append(key) |
| 210 | log(sd, env, message) |
| 211 | if notify: |
| 212 | env.notify(message) |
| 213 | |
| 214 | |
| 215 | # ---------------------------------------------------------------- mirror |
| 216 | |
| 217 | def git(env, repo, *args): |
| 218 | return env.run(["git", "--no-optional-locks", "-C", repo, *args], timeout=GIT_TIMEOUT) |
| 219 | |
| 220 | |
| 221 | def problem(result, what): |
| 222 | lines = (result.stderr or result.stdout or "").strip().splitlines() |
| 223 | return f"{what}: {lines[-1] if lines else 'failed'}" |
| 224 | |
| 225 | |
| 226 | def mirror_settings(cfg): |
| 227 | """(repo folder, branch, remote, extra excludes). A relative `repo` is relative to the config's folder.""" |
| 228 | m = cfg["mirror"] |
| 229 | repo = os.path.abspath(os.path.join(cfg["_dir"], os.path.expanduser(m["repo"]))) |
| 230 | return repo, m.get("branch", "dropbox-mirror"), m.get("remote"), list(m.get("exclude", [])) |
| 231 | |
| 232 | |
| 233 | def rsync_command(src, repo, extra, dry_run=False): |
| 234 | """The user's excludes come before the protective /.git (rsync uses the first pattern that matches).""" |
| 235 | cmd = ["rsync", "-a", "--delete"] + (["-n", "-v"] if dry_run else []) |
| 236 | for pattern in RSYNC_JUNK + extra + ["/.git"]: |
| 237 | cmd.append("--exclude=" + pattern) |
| 238 | return cmd + [src.rstrip("/") + "/", repo.rstrip("/") + "/"] |
| 239 | |
| 240 | |
| 241 | def head_sha(env, repo): |
| 242 | result = git(env, repo, "rev-parse", "-q", "--verify", "HEAD") |
| 243 | return result.stdout.strip() if result.returncode == 0 and result.stdout.strip() else "unborn" |
| 244 | |
| 245 | |
| 246 | def current_branch(env, repo): |
| 247 | result = git(env, repo, "symbolic-ref", "--short", "-q", "HEAD") |
| 248 | return result.stdout.strip() if result.returncode == 0 else "" |
| 249 | |
| 250 | |
| 251 | def status_lines(env, repo): |
| 252 | """(exit code, sorted lines) of `git status --porcelain=v1 -z --ignored`: "XY path" with the real (unquoted) path; |
| 253 | a rename or copy adds a second line "-> origin path" so both names are recorded.""" |
| 254 | result = git(env, repo, "status", "--porcelain=v1", "-z", "--ignored") |
| 255 | fields = result.stdout.split("\0") |
| 256 | lines, i = [], 0 |
| 257 | while i < len(fields): |
| 258 | entry = fields[i] |
| 259 | i += 1 |
| 260 | if len(entry) < 4: |
| 261 | continue |
| 262 | lines.append(entry) |
| 263 | if entry[0] in "RC" or entry[1] in "RC": # the next field is the origin path |
| 264 | if i < len(fields) and fields[i]: |
| 265 | lines.append("-> " + fields[i]) |
| 266 | i += 1 |
| 267 | return result.returncode, sorted(lines) |
| 268 | |
| 269 | |
| 270 | def content_digest(repo, lines): |
| 271 | """sha256 over the contents of the paths a status listing names (folders: every file in them), so an edit to a file |
| 272 | that was already listed is noticed too.""" |
| 273 | h = hashlib.sha256() |
| 274 | for line in lines: |
| 275 | rel = line[3:] |
| 276 | top = os.path.join(repo, rel) |
| 277 | paths = ([os.path.join(d, n) for d, _, names in os.walk(top) for n in sorted(names)] |
| 278 | if os.path.isdir(top) else [top]) |
| 279 | for path in sorted(paths): |
| 280 | try: |
| 281 | h.update(path.encode() + b"\0" + file_hash(path).encode()) |
| 282 | except OSError: |
| 283 | h.update(path.encode() + b"\0missing") |
| 284 | return h.hexdigest() |
| 285 | |
| 286 | |
| 287 | def record_leftover(env, repo, state): |
| 288 | """Remember exactly what our own run left in the repo (HEAD, the status listing, the files' content): a later tick |
| 289 | accepts a dirty tree only if it is still exactly that. Nothing left: no record.""" |
| 290 | code, lines = status_lines(env, repo) |
| 291 | if code == 0 and lines: |
| 292 | state["mirror_dirty"] = {"repo": os.path.realpath(repo), "head": head_sha(env, repo), "status": lines, |
| 293 | "content": content_digest(repo, lines)} |
| 294 | else: |
| 295 | state.pop("mirror_dirty", None) |
| 296 | |
| 297 | |
| 298 | def mirror_blocker(env, cfg, repo, branch, dirty_ok=None): |
| 299 | """Why the mirror must not run now (a sentence), or None when the repo is ready (or about to be created). |
| 300 | Git-ignored files count as changes (rsync --delete could remove them). A dirty tree is accepted only when |
| 301 | `dirty_ok` (what our own run recorded with record_leftover) is exactly what is there now: same repo, same HEAD, |
| 302 | same status listing, same file contents. Anything else is somebody else's work.""" |
| 303 | src = os.path.realpath(cfg["_source"]) |
| 304 | real = os.path.realpath(repo) |
| 305 | if real == src or real.startswith(src + os.sep) or src.startswith(real + os.sep): |
| 306 | return f"mirror.repo {repo} and the source folder contain each other" |
| 307 | if not os.path.exists(repo) or (os.path.isdir(repo) and not os.listdir(repo)): |
| 308 | return None # will be created with `git init` |
| 309 | if not os.path.isdir(repo): |
| 310 | return f"mirror.repo {repo} is not a folder" |
| 311 | top = git(env, repo, "rev-parse", "--show-toplevel") |
| 312 | if top.returncode or os.path.realpath(top.stdout.strip()) != real: |
| 313 | return f"mirror.repo {repo} is not a git repository (its own top folder)" |
| 314 | here = current_branch(env, repo) |
| 315 | if here != branch: |
| 316 | return f"mirror.repo {repo} has {here or 'no branch'} checked out, not {branch}" |
| 317 | gitdir = git(env, repo, "rev-parse", "--absolute-git-dir").stdout.strip() |
| 318 | for name in ("MERGE_HEAD", "rebase-merge", "rebase-apply", "CHERRY_PICK_HEAD", "REVERT_HEAD", "BISECT_LOG"): |
| 319 | if gitdir and os.path.exists(os.path.join(gitdir, name)): |
| 320 | return f"mirror.repo {repo} is in the middle of a merge, rebase or similar ({name})" |
| 321 | code, lines = status_lines(env, repo) |
| 322 | if code == 0 and not lines: |
| 323 | return None |
| 324 | if (code == 0 and isinstance(dirty_ok, dict) and dirty_ok.get("repo") == real and dirty_ok.get("head") == head_sha(env, repo) |
| 325 | and dirty_ok.get("status") == lines and dirty_ok.get("content") == content_digest(repo, lines)): |
| 326 | return None |
| 327 | return (f"mirror.repo {repo} has uncommitted or git-ignored files (or git status failed); " |
| 328 | "the mirror will not overwrite them") |
| 329 | |
| 330 | |
| 331 | def mirror_step(sd, env, state, cfg): |
| 332 | """Mirror the source into the repo. 'ok' or 'blocked' (try again next tick; nothing in the repo is overwritten |
| 333 | that the mirror did not write itself).""" |
| 334 | repo, branch, remote, extra = mirror_settings(cfg) |
| 335 | blocked = mirror_blocker(env, cfg, repo, branch, state.get("mirror_dirty")) |
| 336 | if blocked: |
| 337 | once(sd, env, state, "mirror:" + blocked, "mirror skipped: " + blocked) |
| 338 | return "blocked" |
| 339 | if not os.path.isdir(repo) or not os.listdir(repo): |
| 340 | os.makedirs(repo, exist_ok=True) |
| 341 | init = env.run(["git", "init", "-b", branch, repo], timeout=GIT_TIMEOUT) |
| 342 | if init.returncode: |
| 343 | once(sd, env, state, "mirror:init", "mirror skipped: " + problem(init, "git init")) |
| 344 | return "blocked" |
| 345 | log(sd, env, f"mirror: created git repo {repo} on branch {branch}") |
| 346 | for key in ("user.name", "user.email"): |
| 347 | if not git(env, repo, "config", key).stdout.strip(): |
| 348 | once(sd, env, state, "mirror:identity", |
| 349 | f"mirror skipped: git has no {key}; run: git config --global {key} ...") |
| 350 | return "blocked" |
| 351 | |
| 352 | def failed(key, message): |
| 353 | record_leftover(env, repo, state) # the dirt in the repo is ours: allow a retry over it |
| 354 | once(sd, env, state, key, message) |
| 355 | return "blocked" |
| 356 | |
| 357 | sync = env.run(rsync_command(cfg["_source"], repo, extra), timeout=RSYNC_TIMEOUT) |
| 358 | if sync.returncode == 24: # files vanished while Dropbox was syncing: normal |
| 359 | record_leftover(env, repo, state) |
| 360 | log(sd, env, "mirror: files vanished while copying (rsync 24); trying again next tick") |
| 361 | return "blocked" |
| 362 | if sync.returncode: |
| 363 | return failed("mirror:rsync", "mirror failed: " + problem(sync, "rsync") |
| 364 | + f" ({repo} may be partly updated; the next tick tries again)") |
| 365 | added = git(env, repo, "add", "-A") |
| 366 | if added.returncode: |
| 367 | return failed("mirror:add", "mirror failed: " + problem(added, "git add")) |
| 368 | status = git(env, repo, "status", "--porcelain") |
| 369 | if status.returncode: |
| 370 | return failed("mirror:status", "mirror failed: " + problem(status, "git status")) |
| 371 | if status.stdout.strip(): |
| 372 | if current_branch(env, repo) != branch: # someone switched branches since the check |
| 373 | return failed("mirror:branch", f"mirror skipped: {repo} no longer has {branch} checked out") |
| 374 | stamp = iso(env.now()) |
| 375 | message = f"Mirror of {os.path.basename(os.path.normpath(cfg['_source']))} at {stamp}" |
| 376 | commit = git(env, repo, "commit", "--no-verify", "-m", message) |
| 377 | if commit.returncode: |
| 378 | return failed("mirror:commit", "mirror failed: " + problem(commit, "git commit")) |
| 379 | log(sd, env, f"mirror: committed ({len(status.stdout.splitlines())} paths changed)") |
| 380 | else: |
| 381 | log(sd, env, "mirror: nothing new to commit") |
| 382 | record_leftover(env, repo, state) |
| 383 | if state.get("mirror_dirty"): |
| 384 | # git-ignored files copied from the source (it has a .gitignore?): ours, recorded exactly |
| 385 | once(sd, env, state, "mirror:ignored", "mirror: the repo holds git-ignored files copied from the source") |
| 386 | if remote: |
| 387 | ref = f"refs/heads/{branch}:refs/heads/{branch}" |
| 388 | push = git(env, repo, "push", remote, ref) |
| 389 | if push.returncode: |
| 390 | once(sd, env, state, "mirror:push", f"mirror: push to {remote} failed: " + problem(push, "git push")) |
| 391 | else: |
| 392 | log(sd, env, f"mirror: pushed {branch} to {remote}") |
| 393 | return "ok" |
| 394 | |
| 395 | |
| 396 | # ---------------------------------------------------------------- deploy guard |
| 397 | |
| 398 | def compare(built, baseline, generator=None, config=None, target=None): |
| 399 | """('no-baseline'|'generator'|'target'|'config', {}) | ('paths', {added, removed}) | ('same', {}) | |
| 400 | ('changed', {changed}). A baseline made by another version of the generator (its code, templates, CSS), with another |
| 401 | deploy target or another config holds back, so an automatic deploy never carries a code, template or config change.""" |
| 402 | if baseline is None: |
| 403 | return "no-baseline", {} |
| 404 | if generator is not None and baseline.get("generator") != generator: |
| 405 | return "generator", {} |
| 406 | if target is not None and baseline.get("target") != target: |
| 407 | return "target", {} |
| 408 | if config is not None and baseline.get("config") != config: |
| 409 | return "config", {} |
| 410 | old = baseline["files"] |
| 411 | added, removed = sorted(set(built) - set(old)), sorted(set(old) - set(built)) |
| 412 | if added or removed: |
| 413 | return "paths", {"added": added, "removed": removed} |
| 414 | changed = sorted(p for p in built if built[p] != old[p]) |
| 415 | return ("changed", {"changed": changed}) if changed else ("same", {}) |
| 416 | |
| 417 | |
| 418 | def load_baseline(sd): |
| 419 | data = read_json(os.path.join(sd, "deployed.json"), {}) |
| 420 | return data if isinstance(data.get("files"), dict) else None |
| 421 | |
| 422 | |
| 423 | def build_to_temp(env, cfg): |
| 424 | """(temp folder, built folder, manifest) or raises. The caller removes the temp folder.""" |
| 425 | tmp = tempfile.mkdtemp(prefix="fw-blog-watch-") |
| 426 | out = os.path.join(tmp, "site") |
| 427 | try: |
| 428 | env.build(cfg, out) |
| 429 | return tmp, out, blog.manifest(out) |
| 430 | except BaseException: |
| 431 | shutil.rmtree(tmp, ignore_errors=True) |
| 432 | raise |
| 433 | |
| 434 | |
| 435 | def few(paths, n=5): |
| 436 | return ", ".join(paths[:n]) + (f" (+{len(paths) - n} more)" if len(paths) > n else "") |
| 437 | |
| 438 | |
| 439 | def guard_step(sd, env, state, cfg, config_path): |
| 440 | """Build, compare with deployed.json, deploy when it is safe. Ends the change (the caller marks it handled), |
| 441 | except 'changed-during-build': the source moved, so the new quiet period applies.""" |
| 442 | try: |
| 443 | tmp, out, built = build_to_temp(env, cfg) |
| 444 | except Exception as e: |
| 445 | once(sd, env, state, "build", f"build failed, nothing deployed: {e}") |
| 446 | return "build-error" |
| 447 | try: |
| 448 | if changed_since(sd, cfg, state): |
| 449 | log(sd, env, "changed during build; nothing deployed") |
| 450 | return "changed-during-build" |
| 451 | generator = env.generator() |
| 452 | kind, info = compare(built, load_baseline(sd), generator, blog.config_hash(cfg), blog.deploy_target(cfg)) |
| 453 | if kind == "no-baseline": |
| 454 | once(sd, env, state, "hold", "held back: no baseline: run fw blog deploy once") |
| 455 | return "held" |
| 456 | if kind in ("generator", "target", "config"): |
| 457 | what = {"generator": "generator changed", "target": "deploy target changed", "config": "config changed"}[kind] |
| 458 | once(sd, env, state, "hold", f"held back: {what}: run `fw blog deploy {config_path}`") |
| 459 | return "held" |
| 460 | if kind == "paths": |
| 461 | a, r = info["added"], info["removed"] |
| 462 | once(sd, env, state, "hold", f"held back: {len(a)} added, {len(r)} removed: run `fw blog deploy {config_path}`") |
| 463 | log(sd, env, "held paths: " + few(["+" + p for p in a] + ["-" + p for p in r])) |
| 464 | return "held" |
| 465 | if kind == "same": |
| 466 | log(sd, env, "no visible change") |
| 467 | return "unchanged" |
| 468 | try: |
| 469 | env.deploy(cfg, out) |
| 470 | except Exception as e: |
| 471 | once(sd, env, state, "deploy", f"deploy failed: {e}") |
| 472 | return "deploy-error" |
| 473 | blog.write_deployed(sd, built, iso(env.now()), generator, cfg) |
| 474 | log(sd, env, f"deployed: {len(info['changed'])} files changed") |
| 475 | return "deployed" |
| 476 | finally: |
| 477 | shutil.rmtree(tmp, ignore_errors=True) |
| 478 | |
| 479 | |
| 480 | # ---------------------------------------------------------------- one tick |
| 481 | |
| 482 | def tick(config_path, env=None): |
| 483 | """One pass. Returns a word saying what happened (for the tests and --dry-run): locked, changed, idle, waiting, |
| 484 | conflict, blocked, deployed, held, unchanged, build-error, deploy-error, changed-during-build, mirrored, |
| 485 | config-error, noop, error (an unexpected exception: logged, notified once, never raised).""" |
| 486 | env = env or Env() |
| 487 | config_path = os.path.abspath(config_path) |
| 488 | sd = blog.state_dir(config_path) |
| 489 | os.makedirs(sd, exist_ok=True) |
| 490 | lock = blog.acquire_lock(sd, env.now) |
| 491 | if lock is None: |
| 492 | since = blog.lock_since(sd) |
| 493 | marker = os.path.join(sd, "stuck-notified") |
| 494 | if since and env.now() - since > STUCK_AFTER and read_text(marker) != repr(since): |
| 495 | with open(marker, "w") as f: |
| 496 | f.write(repr(since)) |
| 497 | text = "watch tick stuck: the lock has been held for over 30 minutes" |
| 498 | log(sd, env, text) |
| 499 | env.notify(text) |
| 500 | return "locked" |
| 501 | try: |
| 502 | result = locked_tick(sd, env, config_path) |
| 503 | state = read_json(os.path.join(sd, "state.json"), {}) |
| 504 | if state.pop("last_error", None): # a clean tick: the next unexpected error notifies again |
| 505 | save_state(sd, state) |
| 506 | return result |
| 507 | except Exception as e: |
| 508 | frame = traceback.extract_tb(e.__traceback__)[-1] |
| 509 | text = f"unexpected error: {type(e).__name__}: {e} ({os.path.basename(frame.filename)}:{frame.lineno})" |
| 510 | state = read_json(os.path.join(sd, "state.json"), {}) |
| 511 | if state.get("last_error") != text: |
| 512 | state["last_error"] = text |
| 513 | save_state(sd, state) |
| 514 | log(sd, env, text) |
| 515 | env.notify(text) |
| 516 | return "error" |
| 517 | finally: |
| 518 | lock.close() |
| 519 | |
| 520 | |
| 521 | def read_text(path): |
| 522 | try: |
| 523 | with open(path) as f: |
| 524 | return f.read() |
| 525 | except OSError: |
| 526 | return None |
| 527 | |
| 528 | |
| 529 | def locked_tick(sd, env, config_path): |
| 530 | state = read_json(os.path.join(sd, "state.json"), {}) |
| 531 | earlier = state.pop("config_error", None) |
| 532 | |
| 533 | def config_problem(text): |
| 534 | if earlier != text: |
| 535 | log(sd, env, text) |
| 536 | env.notify(text) |
| 537 | state["config_error"] = text |
| 538 | save_state(sd, state) |
| 539 | return "config-error" |
| 540 | |
| 541 | try: |
| 542 | cfg = blog.load_config(config_path) |
| 543 | except blog.BlogError as e: |
| 544 | return config_problem(f"config error: {e}") |
| 545 | if cfg["mirror"] is None and cfg["autodeploy"] is None: |
| 546 | if earlier: |
| 547 | save_state(sd, state) |
| 548 | return "noop" |
| 549 | if not os.path.isdir(cfg["_source"]): |
| 550 | return config_problem(f"source folder {cfg['_source']} not found") |
| 551 | if earlier: |
| 552 | save_state(sd, state) # recovered: the next problem notifies again |
| 553 | now = env.now() |
| 554 | fingerprint, conflicts = current(sd, cfg) |
| 555 | if state.get("fingerprint") != fingerprint or state.get("conflicts") != conflicts: |
| 556 | save_state(sd, {"fingerprint": fingerprint, "conflicts": conflicts, "changed_at": now, "handled": False, |
| 557 | "notified": [], **{k: state[k] for k in ("mirror_dirty",) if k in state}}) |
| 558 | log(sd, env, "changed") |
| 559 | return "changed" |
| 560 | if state.get("handled"): |
| 561 | return "idle" |
| 562 | quiet = (cfg["autodeploy"] or {}).get("quiet_seconds", DEFAULT_QUIET) |
| 563 | if now - state.get("changed_at", now) < quiet: |
| 564 | if "waiting" not in state.setdefault("notified", []): |
| 565 | state["notified"].append("waiting") |
| 566 | log(sd, env, f"waiting: {quiet} s without changes needed") |
| 567 | save_state(sd, state) |
| 568 | return "waiting" |
| 569 | try: |
| 570 | return decide(sd, env, state, cfg, config_path, conflicts) |
| 571 | finally: |
| 572 | save_state(sd, state) |
| 573 | |
| 574 | |
| 575 | def decide(sd, env, state, cfg, config_path, conflicts): |
| 576 | if conflicts: |
| 577 | once(sd, env, state, "conflict", "Dropbox conflict file(s) in the source, nothing mirrored or deployed: " |
| 578 | + few(conflicts, 3)) |
| 579 | return "conflict" |
| 580 | result = "mirrored" |
| 581 | if cfg["mirror"] is not None: |
| 582 | if mirror_step(sd, env, state, cfg) == "blocked": |
| 583 | return "blocked" |
| 584 | if changed_since(sd, cfg, state): |
| 585 | log(sd, env, "changed during mirror; nothing built") |
| 586 | return "changed-during-build" |
| 587 | if cfg["autodeploy"] is not None: |
| 588 | result = guard_step(sd, env, state, cfg, config_path) |
| 589 | if result == "changed-during-build": |
| 590 | return result |
| 591 | state["handled"] = True |
| 592 | return result |
| 593 | |
| 594 | |
| 595 | # ---------------------------------------------------------------- --dry-run |
| 596 | |
| 597 | def dry_run(config_path, env): |
| 598 | """Shows what a tick would do right now (ignoring the quiet period); writes nothing.""" |
| 599 | cfg = blog.load_config(config_path) |
| 600 | say = env.echo |
| 601 | if cfg["mirror"] is None and cfg["autodeploy"] is None: |
| 602 | say("the config has neither mirror nor autodeploy: nothing to do") |
| 603 | return |
| 604 | if not os.path.isdir(cfg["_source"]): |
| 605 | say(f"source folder {cfg['_source']} not found") |
| 606 | return |
| 607 | _, conflicts = scan(cfg["_source"]) |
| 608 | if conflicts: |
| 609 | say("would stop: Dropbox conflict file(s): " + few(conflicts)) |
| 610 | return |
| 611 | if cfg["mirror"] is not None: |
| 612 | repo, branch, remote, extra = mirror_settings(cfg) |
| 613 | dirty_ok = read_json(os.path.join(blog.state_dir(cfg["_path"]), "state.json"), {}).get("mirror_dirty") |
| 614 | blocked = mirror_blocker(env, cfg, repo, branch, dirty_ok) |
| 615 | if blocked: |
| 616 | say("mirror would be skipped: " + blocked) |
| 617 | return |
| 618 | if not os.path.isdir(repo) or not os.listdir(repo): |
| 619 | say(f"mirror: would run git init -b {branch} {repo}") |
| 620 | else: |
| 621 | say(f"mirror: rsync would change (source -> {repo}):") |
| 622 | result = env.run(rsync_command(cfg["_source"], repo, extra, dry_run=True), timeout=RSYNC_TIMEOUT) |
| 623 | say(result.stdout.rstrip() or " (nothing)") |
| 624 | say("mirror: would git add -A, commit if anything changed" + (f", and push to {remote}" if remote else "")) |
| 625 | if cfg["autodeploy"] is not None: |
| 626 | try: |
| 627 | tmp, _, built = build_to_temp(env, cfg) |
| 628 | except Exception as e: |
| 629 | say(f"deploy: the build fails, nothing would be deployed: {e}") |
| 630 | return |
| 631 | shutil.rmtree(tmp, ignore_errors=True) |
| 632 | kind, info = compare(built, load_baseline(blog.state_dir(cfg["_path"])), env.generator(), |
| 633 | blog.config_hash(cfg), blog.deploy_target(cfg)) |
| 634 | say({"no-baseline": "deploy: would hold back (no baseline: run fw blog deploy once)", |
| 635 | "generator": "deploy: would hold back (generator changed: run fw blog deploy)", |
| 636 | "target": "deploy: would hold back (deploy target changed: run fw blog deploy)", |
| 637 | "config": "deploy: would hold back (config changed: run fw blog deploy)", |
| 638 | "paths": f"deploy: would hold back ({len(info.get('added', []))} added, {len(info.get('removed', []))} " |
| 639 | "removed paths)", |
| 640 | "same": "deploy: no visible change, nothing to deploy", |
| 641 | "changed": f"deploy: would upload ({len(info.get('changed', []))} files changed): " |
| 642 | + few(info.get("changed", []))}[kind]) |
| 643 | |
| 644 | |
| 645 | # ---------------------------------------------------------------- launchd agent |
| 646 | |
| 647 | def plist_data(config_path, name, sd, fw=None): |
| 648 | fw = fw or os.path.realpath(os.path.join(HERE, "..", "bin", "fw")) |
| 649 | return {"Label": LABEL + name, |
| 650 | "ProgramArguments": [fw, "blog", "watch", os.path.abspath(config_path), "--once"], |
| 651 | "StartInterval": TICK_SECONDS, "RunAtLoad": True, |
| 652 | "EnvironmentVariables": {"PATH": LAUNCH_PATH}, |
| 653 | "StandardOutPath": os.path.join(sd, "launchd.log"), "StandardErrorPath": os.path.join(sd, "launchd.log"), |
| 654 | "ProcessType": "Background", "Nice": 10} |
| 655 | |
| 656 | |
| 657 | def plist_text(config_path, fw=None): |
| 658 | config_path = os.path.abspath(config_path) |
| 659 | return plistlib.dumps(plist_data(config_path, blog.watch_name(config_path), blog.state_dir(config_path), fw)).decode("utf-8") |
| 660 | |
| 661 | |
| 662 | def agent_path(config_path, agents_dir=None): |
| 663 | return os.path.join(agents_dir or os.path.expanduser("~/Library/LaunchAgents"), |
| 664 | LABEL + blog.watch_name(config_path) + ".plist") |
| 665 | |
| 666 | |
| 667 | def install(config_path, run=run_command, agents_dir=None, uid=None, fw=None, sleep=time.sleep): |
| 668 | """Writes the plist and loads it. Reinstalling replaces a loaded agent.""" |
| 669 | blog.load_config(config_path) # a typo in the config is better found now than every minute |
| 670 | path = agent_path(config_path, agents_dir) |
| 671 | os.makedirs(os.path.dirname(path), exist_ok=True) |
| 672 | os.makedirs(blog.state_dir(config_path), exist_ok=True) |
| 673 | domain = f"gui/{os.getuid() if uid is None else uid}" |
| 674 | with open(path, "w", encoding="utf-8") as f: |
| 675 | f.write(plist_text(config_path, fw)) |
| 676 | run(["launchctl", "bootout", domain, path]) # fails when it was not loaded: fine |
| 677 | for attempt in range(5): # bootstrap can fail for a moment right after a bootout |
| 678 | result = run(["launchctl", "bootstrap", domain, path]) |
| 679 | if not result.returncode: |
| 680 | break |
| 681 | if attempt < 4: |
| 682 | sleep(1) |
| 683 | if result.returncode: |
| 684 | raise blog.BlogError(f"launchctl bootstrap failed: {(result.stderr or '').strip()}") |
| 685 | return path |
| 686 | |
| 687 | |
| 688 | def uninstall(config_path, run=run_command, agents_dir=None, uid=None): |
| 689 | path = agent_path(config_path, agents_dir) |
| 690 | run(["launchctl", "bootout", f"gui/{os.getuid() if uid is None else uid}", path]) |
| 691 | if os.path.exists(path): |
| 692 | os.remove(path) |
| 693 | return path |
| 694 | |
| 695 | |
| 696 | # ---------------------------------------------------------------- command line |
| 697 | |
| 698 | def main(argv): |
| 699 | ap = argparse.ArgumentParser(prog="blog_watch.py", description=__doc__.split("\n")[0]) |
| 700 | ap.add_argument("words", nargs="+", metavar="[install|uninstall] config") |
| 701 | ap.add_argument("--once", action="store_true", help="one tick, then exit (what launchd runs)") |
| 702 | ap.add_argument("--dry-run", action="store_true", help="show what a tick would do now; write nothing") |
| 703 | ap.add_argument("--print", action="store_true", dest="show", help="with install: only print the launchd plist") |
| 704 | args = ap.parse_args(argv[1:]) |
| 705 | action = args.words[0] if args.words[0] in ("install", "uninstall") else None |
| 706 | rest = args.words[1:] if action else args.words |
| 707 | if len(rest) != 1: |
| 708 | ap.error("needs exactly one config file") |
| 709 | config = os.path.abspath(os.path.expanduser(rest[0])) |
| 710 | try: |
| 711 | if action == "install": |
| 712 | if args.show: |
| 713 | print(plist_text(config), end="") |
| 714 | else: |
| 715 | print("installed " + install(config)) |
| 716 | elif action == "uninstall": |
| 717 | print("removed " + uninstall(config)) |
| 718 | elif args.dry_run: |
| 719 | dry_run(config, Env()) |
| 720 | elif args.once: |
| 721 | result = tick(config) |
| 722 | if result in ("config-error", "build-error", "deploy-error", "blocked", "error"): # the exit code stays 0: launchd must not throttle the agent |
| 723 | print(f"blog_watch: {result} (see {blog.state_dir(config)}/watch.log)", file=sys.stderr) |
| 724 | else: |
| 725 | while True: |
| 726 | tick(config) |
| 727 | time.sleep(TICK_SECONDS) |
| 728 | except blog.BlogError as e: |
| 729 | raise SystemExit(f"blog_watch.py: {e}") |
| 730 | except KeyboardInterrupt: |
| 731 | pass |
| 732 | |
| 733 | |
| 734 | if __name__ == "__main__": |
| 735 | main(sys.argv) |