fuzzyworld-tools / files / server / blog_watch.py
blog_watch.py
9bb22e48 2026-11-03 Pavel Safronov | 1 | #!/usr/bin/env python3 |
| 2 | """`fw blog watch`: mirror a blog's source folder into a local git repo and deploy edits to published posts. | |
| 3 | ||
| 4 | blog_watch.py <config> [--once] [--dry-run] one tick (--once, what launchd runs every 60 s), or a loop | |
| 5 | blog_watch.py install|uninstall <config> [--print] the launchd agent (--print only shows the plist) | |
| 6 | ||
| 7 | Run it on ONE machine. Per tick, for one blog config: take a lock, fingerprint the source folder, wait until the | |
| 8 | folder has been quiet for `autodeploy.quiet_seconds` (default 300), then (1) mirror it into `mirror.repo` as a | |
| 9 | commit on a separate branch, (2) build the site and deploy it only when the set of output files is the same as in | |
| 10 | the last deploy and some file's content differs. Anything else (a new or removed page, no baseline, or a changed | |
| 11 | generator, config or deploy target) is held back with a notification: run `fw blog deploy <config>` yourself. State: ~/.fw-blog-watch/<name>-<hash>/ (see blog.watch_name): | |
bb7e1f57 2026-11-06 Pavel Safronov | 12 | state.json, watch.log, deployed.json, hashes.json, lock, launchd.log. Rules and config keys are in docs/blog.md. |
9bb22e48 2026-11-03 Pavel Safronov | 13 | |
| 14 | Everything with side effects (clock, notification, running commands, the deploy) is injected through `Env`, so | |
| 15 | the tests use fakes. Stdlib only. | |
| 16 | """ | |
| 17 | ||
| 18 | import argparse | |
| 19 | import dataclasses | |
| 20 | import datetime | |
| 21 | import hashlib | |
| 22 | import json | |
| 23 | import os | |
| 24 | import plistlib | |
| 25 | import shutil | |
| 26 | import subprocess | |
| 27 | import sys | |
| 28 | import tempfile | |
| 29 | import time | |
| 30 | import traceback | |
| 31 | ||
| 32 | HERE = os.path.dirname(os.path.abspath(__file__)) | |
| 33 | sys.path.insert(0, HERE) | |
| 34 | import blog # noqa: E402 | |
| 35 | ||
| 36 | JUNK_NAMES = {".DS_Store", ".dropbox", ".dropbox.cache", ".git"} | |
| 37 | RSYNC_JUNK = [".DS_Store", ".dropbox", ".dropbox.cache", "*conflicted copy*", ".git"] | |
| 38 | DEFAULT_QUIET = 300 | |
| 39 | TICK_SECONDS = 60 | |
| 40 | GIT_TIMEOUT, RSYNC_TIMEOUT, NOTIFY_TIMEOUT = 120, 600, 10 | |
| 41 | STUCK_AFTER = 1800 # a lock held this long (seconds) is reported as a stuck tick | |
| 42 | LABEL = "net.fuzzyworld.fw-blog-watch-" | |
| 43 | LAUNCH_PATH = "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin" | |
| 44 | ||
| 45 | ||
| 46 | def is_conflict(name): | |
| 47 | return "conflicted copy" in name.lower() | |
| 48 | ||
| 49 | ||
| 50 | def is_junk(name): | |
| 51 | return name in JUNK_NAMES or is_conflict(name) | |
| 52 | ||
| 53 | ||
| 54 | # ---------------------------------------------------------------- injected effects | |
| 55 | ||
| 56 | def notify_macos(message, title="fw blog watch"): | |
| 57 | """A macOS notification; any failure is ignored. FW_BLOG_WATCH_NOTIFY=0 turns it off (the tests do).""" | |
| 58 | if os.environ.get("FW_BLOG_WATCH_NOTIFY") == "0": | |
| 59 | return | |
| 60 | def quote(text): | |
| 61 | return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"' | |
| 62 | try: | |
| 63 | subprocess.run(["osascript", "-e", f"display notification {quote(message[:230])} with title {quote(title)}"], | |
| 64 | capture_output=True, timeout=NOTIFY_TIMEOUT) | |
| 65 | except Exception: | |
| 66 | pass | |
| 67 | ||
| 68 | ||
| 69 | def run_command(cmd, cwd=None, timeout=GIT_TIMEOUT): | |
| 70 | """subprocess.run with captured text output. git never asks anything on the terminal. A command that runs | |
| 71 | too long is killed and reported as a failure (exit 124), like any other.""" | |
| 72 | try: | |
| 73 | return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="surrogateescape", | |
| 74 | stdin=subprocess.DEVNULL, timeout=timeout, | |
| 75 | env={**os.environ, "GIT_TERMINAL_PROMPT": "0"}) | |
| 76 | except subprocess.TimeoutExpired: | |
| 77 | return subprocess.CompletedProcess(cmd, 124, "", f"{cmd[0]} timed out after {timeout} s") | |
| 78 | ||
| 79 | ||
| 80 | def build_site(cfg, out): | |
| 81 | return blog.build(cfg, out) | |
| 82 | ||
| 83 | ||
| 84 | def deploy_built(cfg, out): | |
| 85 | """The same function `fw blog deploy` uses, without the question; it audits `out` and runs rsync.""" | |
| 86 | return blog.deploy_site(cfg, out, prebuilt=True, locked=True, timeout=RSYNC_TIMEOUT) # the tick holds the lock | |
| 87 | ||
| 88 | ||
| 89 | @dataclasses.dataclass | |
| 90 | class Env: | |
| 91 | now: object = time.time # () -> seconds since the epoch | |
| 92 | notify: object = notify_macos # (message) -> None | |
| 93 | run: object = run_command # (argv, cwd=None, timeout=...) -> object with returncode, stdout, stderr | |
| 94 | build: object = build_site # (cfg, out) -> None | |
| 95 | deploy: object = deploy_built # (cfg, out) -> None, raises when the upload fails | |
| 96 | generator: object = lambda: blog.GENERATOR_HASH # () -> hash of the code loaded now (fixed at import) | |
| 97 | echo: object = print # where --dry-run output goes | |
| 98 | ||
| 99 | ||
| 100 | def iso(seconds): | |
| 101 | return datetime.datetime.fromtimestamp(seconds, datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | |
| 102 | ||
| 103 | ||
| 104 | # ---------------------------------------------------------------- fingerprint | |
| 105 | ||
| 106 | def file_hash(path): | |
| 107 | h = hashlib.sha256() | |
| 108 | with open(path, "rb") as f: | |
| 109 | for chunk in iter(lambda: f.read(1 << 20), b""): | |
| 110 | h.update(chunk) | |
| 111 | return h.hexdigest() | |
| 112 | ||
| 113 | ||
| 114 | def scan(src, cache=None): | |
| 115 | """(fingerprint, conflict paths) of a source folder. With a `cache` dict ({path: [size, mtime_ns, inode, ctime_ns, | |
| 116 | sha256]}, updated in place) a file whose first four values are unchanged is not read again. The fingerprint is a sha256 over the sorted (relative path, | |
| 117 | content sha256) pairs of every file except junk (.DS_Store, .dropbox*, .git, Dropbox "conflicted copy" files). | |
| 118 | Symbolic links are not followed: a link counts by its target text.""" | |
| 119 | entries, conflicts, fresh = [], [], {} | |
| 120 | for dirpath, dirs, files in os.walk(src): | |
| 121 | keep = [] | |
| 122 | for name in sorted(dirs): | |
| 123 | path = os.path.join(dirpath, name) | |
| 124 | rel = os.path.relpath(path, src).replace(os.sep, "/") | |
| 125 | if is_conflict(name): | |
| 126 | conflicts.append(rel) | |
| 127 | elif name in JUNK_NAMES: | |
| 128 | pass | |
| 129 | elif os.path.islink(path): | |
| 130 | entries.append((rel, "link:" + os.readlink(path))) | |
| 131 | else: | |
| 132 | keep.append(name) | |
| 133 | dirs[:] = keep | |
| 134 | for name in sorted(files): | |
| 135 | path = os.path.join(dirpath, name) | |
| 136 | rel = os.path.relpath(path, src).replace(os.sep, "/") | |
| 137 | if is_conflict(name): | |
| 138 | conflicts.append(rel) | |
| 139 | elif is_junk(name): | |
| 140 | continue | |
| 141 | elif os.path.islink(path): | |
| 142 | entries.append((rel, "link:" + os.readlink(path))) | |
| 143 | else: | |
| 144 | try: | |
| 145 | st = os.stat(path) | |
| 146 | stamp = [st.st_size, st.st_mtime_ns, st.st_ino, st.st_ctime_ns] | |
| 147 | old = cache.get(rel) if isinstance(cache, dict) else None | |
| 148 | if isinstance(old, list) and len(old) == 5 and old[:4] == stamp and isinstance(old[4], str): | |
| 149 | digest = old[4] | |
| 150 | else: | |
| 151 | digest = file_hash(path) | |
| 152 | fresh[rel] = stamp + [digest] | |
| 153 | entries.append((rel, digest)) | |
| 154 | except FileNotFoundError: # Dropbox moved it while we looked: the next tick sees the result | |
| 155 | continue | |
| 156 | except OSError as e: | |
| 157 | entries.append((rel, f"unreadable:{e.errno}")) | |
| 158 | if cache is not None: | |
| 159 | cache.clear() | |
| 160 | cache.update(fresh) | |
| 161 | text = "\n".join(f"{rel}\0{digest}" for rel, digest in sorted(entries)) | |
| 162 | return hashlib.sha256(text.encode("utf-8", "surrogateescape")).hexdigest(), sorted(conflicts) | |
| 163 | ||
| 164 | ||
| 165 | # ---------------------------------------------------------------- state and log | |
| 166 | ||
| 167 | def read_json(path, default): | |
| 168 | try: | |
| 169 | with open(path, encoding="utf-8") as f: | |
| 170 | data = json.load(f) | |
| 171 | return data if isinstance(data, type(default)) else default | |
| 172 | except (OSError, ValueError): | |
| 173 | return default | |
| 174 | ||
| 175 | ||
| 176 | def save_json(sd, name, data): | |
| 177 | tmp = os.path.join(sd, name + ".tmp") | |
| 178 | with open(tmp, "w", encoding="utf-8") as f: | |
| 179 | json.dump(data, f, indent=1) | |
| 180 | os.replace(tmp, os.path.join(sd, name)) | |
| 181 | ||
| 182 | ||
| 183 | def save_state(sd, state): | |
| 184 | save_json(sd, "state.json", state) | |
| 185 | ||
| 186 | ||
| 187 | def current(sd, cfg): | |
| 188 | """(fingerprint, conflicts) of the source now, using (and refreshing) the hash cache in the state folder.""" | |
| 189 | cache = read_json(os.path.join(sd, "hashes.json"), {}) # a damaged cache file is just an empty one | |
| 190 | result = scan(cfg["_source"], cache) | |
| 191 | save_json(sd, "hashes.json", cache) | |
| 192 | return result | |
| 193 | ||
| 194 | ||
| 195 | def changed_since(sd, cfg, state): | |
| 196 | return current(sd, cfg) != (state.get("fingerprint"), state.get("conflicts")) | |
| 197 | ||
| 198 | ||
| 199 | def log(sd, env, message): | |
| 200 | with open(os.path.join(sd, "watch.log"), "a", encoding="utf-8") as f: | |
| 201 | f.write(f"{iso(env.now())} {message}\n") | |
| 202 | ||
| 203 | ||
| 204 | def once(sd, env, state, key, message, notify=True): | |
| 205 | """Log (and notify) a message the first time `key` happens after a change of the source, not on every tick.""" | |
| 206 | seen = state.setdefault("notified", []) | |
| 207 | if key in seen: | |
| 208 | return | |
| 209 | seen.append(key) | |
| 210 | log(sd, env, message) | |
| 211 | if notify: | |
| 212 | env.notify(message) | |
| 213 | ||
| 214 | ||
| 215 | # ---------------------------------------------------------------- mirror | |
| 216 | ||
| 217 | def git(env, repo, *args): | |
| 218 | return env.run(["git", "--no-optional-locks", "-C", repo, *args], timeout=GIT_TIMEOUT) | |
| 219 | ||
| 220 | ||
| 221 | def problem(result, what): | |
| 222 | lines = (result.stderr or result.stdout or "").strip().splitlines() | |
| 223 | return f"{what}: {lines[-1] if lines else 'failed'}" | |
| 224 | ||
| 225 | ||
| 226 | def mirror_settings(cfg): | |
| 227 | """(repo folder, branch, remote, extra excludes). A relative `repo` is relative to the config's folder.""" | |
| 228 | m = cfg["mirror"] | |
| 229 | repo = os.path.abspath(os.path.join(cfg["_dir"], os.path.expanduser(m["repo"]))) | |
| 230 | return repo, m.get("branch", "dropbox-mirror"), m.get("remote"), list(m.get("exclude", [])) | |
| 231 | ||
| 232 | ||
| 233 | def rsync_command(src, repo, extra, dry_run=False): | |
| 234 | """The user's excludes come before the protective /.git (rsync uses the first pattern that matches).""" | |
| 235 | cmd = ["rsync", "-a", "--delete"] + (["-n", "-v"] if dry_run else []) | |
| 236 | for pattern in RSYNC_JUNK + extra + ["/.git"]: | |
| 237 | cmd.append("--exclude=" + pattern) | |
| 238 | return cmd + [src.rstrip("/") + "/", repo.rstrip("/") + "/"] | |
| 239 | ||
| 240 | ||
| 241 | def head_sha(env, repo): | |
| 242 | result = git(env, repo, "rev-parse", "-q", "--verify", "HEAD") | |
| 243 | return result.stdout.strip() if result.returncode == 0 and result.stdout.strip() else "unborn" | |
| 244 | ||
| 245 | ||
| 246 | def current_branch(env, repo): | |
| 247 | result = git(env, repo, "symbolic-ref", "--short", "-q", "HEAD") | |
| 248 | return result.stdout.strip() if result.returncode == 0 else "" | |
| 249 | ||
| 250 | ||
| 251 | def status_lines(env, repo): | |
| 252 | """(exit code, sorted lines) of `git status --porcelain=v1 -z --ignored`: "XY path" with the real (unquoted) path; | |
| 253 | a rename or copy adds a second line "-> origin path" so both names are recorded.""" | |
| 254 | result = git(env, repo, "status", "--porcelain=v1", "-z", "--ignored") | |
| 255 | fields = result.stdout.split("\0") | |
| 256 | lines, i = [], 0 | |
| 257 | while i < len(fields): | |
| 258 | entry = fields[i] | |
| 259 | i += 1 | |
| 260 | if len(entry) < 4: | |
| 261 | continue | |
| 262 | lines.append(entry) | |
| 263 | if entry[0] in "RC" or entry[1] in "RC": # the next field is the origin path | |
| 264 | if i < len(fields) and fields[i]: | |
| 265 | lines.append("-> " + fields[i]) | |
| 266 | i += 1 | |
| 267 | return result.returncode, sorted(lines) | |
| 268 | ||
| 269 | ||
| 270 | def content_digest(repo, lines): | |
| 271 | """sha256 over the contents of the paths a status listing names (folders: every file in them), so an edit to a file | |
| 272 | that was already listed is noticed too.""" | |
| 273 | h = hashlib.sha256() | |
| 274 | for line in lines: | |
| 275 | rel = line[3:] | |
| 276 | top = os.path.join(repo, rel) | |
| 277 | paths = ([os.path.join(d, n) for d, _, names in os.walk(top) for n in sorted(names)] | |
| 278 | if os.path.isdir(top) else [top]) | |
| 279 | for path in sorted(paths): | |
| 280 | try: | |
| 281 | h.update(path.encode() + b"\0" + file_hash(path).encode()) | |
| 282 | except OSError: | |
| 283 | h.update(path.encode() + b"\0missing") | |
| 284 | return h.hexdigest() | |
| 285 | ||
| 286 | ||
| 287 | def record_leftover(env, repo, state): | |
| 288 | """Remember exactly what our own run left in the repo (HEAD, the status listing, the files' content): a later tick | |
| 289 | accepts a dirty tree only if it is still exactly that. Nothing left: no record.""" | |
| 290 | code, lines = status_lines(env, repo) | |
| 291 | if code == 0 and lines: | |
| 292 | state["mirror_dirty"] = {"repo": os.path.realpath(repo), "head": head_sha(env, repo), "status": lines, | |
| 293 | "content": content_digest(repo, lines)} | |
| 294 | else: | |
| 295 | state.pop("mirror_dirty", None) | |
| 296 | ||
| 297 | ||
| 298 | def mirror_blocker(env, cfg, repo, branch, dirty_ok=None): | |
| 299 | """Why the mirror must not run now (a sentence), or None when the repo is ready (or about to be created). | |
| 300 | Git-ignored files count as changes (rsync --delete could remove them). A dirty tree is accepted only when | |
| 301 | `dirty_ok` (what our own run recorded with record_leftover) is exactly what is there now: same repo, same HEAD, | |
| 302 | same status listing, same file contents. Anything else is somebody else's work.""" | |
| 303 | src = os.path.realpath(cfg["_source"]) | |
| 304 | real = os.path.realpath(repo) | |
| 305 | if real == src or real.startswith(src + os.sep) or src.startswith(real + os.sep): | |
| 306 | return f"mirror.repo {repo} and the source folder contain each other" | |
| 307 | if not os.path.exists(repo) or (os.path.isdir(repo) and not os.listdir(repo)): | |
| 308 | return None # will be created with `git init` | |
| 309 | if not os.path.isdir(repo): | |
| 310 | return f"mirror.repo {repo} is not a folder" | |
| 311 | top = git(env, repo, "rev-parse", "--show-toplevel") | |
| 312 | if top.returncode or os.path.realpath(top.stdout.strip()) != real: | |
| 313 | return f"mirror.repo {repo} is not a git repository (its own top folder)" | |
| 314 | here = current_branch(env, repo) | |
| 315 | if here != branch: | |
| 316 | return f"mirror.repo {repo} has {here or 'no branch'} checked out, not {branch}" | |
| 317 | gitdir = git(env, repo, "rev-parse", "--absolute-git-dir").stdout.strip() | |
| 318 | for name in ("MERGE_HEAD", "rebase-merge", "rebase-apply", "CHERRY_PICK_HEAD", "REVERT_HEAD", "BISECT_LOG"): | |
| 319 | if gitdir and os.path.exists(os.path.join(gitdir, name)): | |
| 320 | return f"mirror.repo {repo} is in the middle of a merge, rebase or similar ({name})" | |
| 321 | code, lines = status_lines(env, repo) | |
| 322 | if code == 0 and not lines: | |
| 323 | return None | |
| 324 | if (code == 0 and isinstance(dirty_ok, dict) and dirty_ok.get("repo") == real and dirty_ok.get("head") == head_sha(env, repo) | |
| 325 | and dirty_ok.get("status") == lines and dirty_ok.get("content") == content_digest(repo, lines)): | |
| 326 | return None | |
| 327 | return (f"mirror.repo {repo} has uncommitted or git-ignored files (or git status failed); " | |
| 328 | "the mirror will not overwrite them") | |
| 329 | ||
| 330 | ||
| 331 | def mirror_step(sd, env, state, cfg): | |
| 332 | """Mirror the source into the repo. 'ok' or 'blocked' (try again next tick; nothing in the repo is overwritten | |
| 333 | that the mirror did not write itself).""" | |
| 334 | repo, branch, remote, extra = mirror_settings(cfg) | |
| 335 | blocked = mirror_blocker(env, cfg, repo, branch, state.get("mirror_dirty")) | |
| 336 | if blocked: | |
| 337 | once(sd, env, state, "mirror:" + blocked, "mirror skipped: " + blocked) | |
| 338 | return "blocked" | |
| 339 | if not os.path.isdir(repo) or not os.listdir(repo): | |
| 340 | os.makedirs(repo, exist_ok=True) | |
| 341 | init = env.run(["git", "init", "-b", branch, repo], timeout=GIT_TIMEOUT) | |
| 342 | if init.returncode: | |
| 343 | once(sd, env, state, "mirror:init", "mirror skipped: " + problem(init, "git init")) | |
| 344 | return "blocked" | |
| 345 | log(sd, env, f"mirror: created git repo {repo} on branch {branch}") | |
| 346 | for key in ("user.name", "user.email"): | |
| 347 | if not git(env, repo, "config", key).stdout.strip(): | |
| 348 | once(sd, env, state, "mirror:identity", | |
| 349 | f"mirror skipped: git has no {key}; run: git config --global {key} ...") | |
| 350 | return "blocked" | |
| 351 | ||
| 352 | def failed(key, message): | |
| 353 | record_leftover(env, repo, state) # the dirt in the repo is ours: allow a retry over it | |
| 354 | once(sd, env, state, key, message) | |
| 355 | return "blocked" | |
| 356 | ||
| 357 | sync = env.run(rsync_command(cfg["_source"], repo, extra), timeout=RSYNC_TIMEOUT) | |
| 358 | if sync.returncode == 24: # files vanished while Dropbox was syncing: normal | |
| 359 | record_leftover(env, repo, state) | |
| 360 | log(sd, env, "mirror: files vanished while copying (rsync 24); trying again next tick") | |
| 361 | return "blocked" | |
| 362 | if sync.returncode: | |
| 363 | return failed("mirror:rsync", "mirror failed: " + problem(sync, "rsync") | |
| 364 | + f" ({repo} may be partly updated; the next tick tries again)") | |
| 365 | added = git(env, repo, "add", "-A") | |
| 366 | if added.returncode: | |
| 367 | return failed("mirror:add", "mirror failed: " + problem(added, "git add")) | |
| 368 | status = git(env, repo, "status", "--porcelain") | |
| 369 | if status.returncode: | |
| 370 | return failed("mirror:status", "mirror failed: " + problem(status, "git status")) | |
| 371 | if status.stdout.strip(): | |
| 372 | if current_branch(env, repo) != branch: # someone switched branches since the check | |
| 373 | return failed("mirror:branch", f"mirror skipped: {repo} no longer has {branch} checked out") | |
| 374 | stamp = iso(env.now()) | |
| 375 | message = f"Mirror of {os.path.basename(os.path.normpath(cfg['_source']))} at {stamp}" | |
| 376 | commit = git(env, repo, "commit", "--no-verify", "-m", message) | |
| 377 | if commit.returncode: | |
| 378 | return failed("mirror:commit", "mirror failed: " + problem(commit, "git commit")) | |
| 379 | log(sd, env, f"mirror: committed ({len(status.stdout.splitlines())} paths changed)") | |
| 380 | else: | |
| 381 | log(sd, env, "mirror: nothing new to commit") | |
| 382 | record_leftover(env, repo, state) | |
| 383 | if state.get("mirror_dirty"): | |
| 384 | # git-ignored files copied from the source (it has a .gitignore?): ours, recorded exactly | |
| 385 | once(sd, env, state, "mirror:ignored", "mirror: the repo holds git-ignored files copied from the source") | |
| 386 | if remote: | |
| 387 | ref = f"refs/heads/{branch}:refs/heads/{branch}" | |
| 388 | push = git(env, repo, "push", remote, ref) | |
| 389 | if push.returncode: | |
| 390 | once(sd, env, state, "mirror:push", f"mirror: push to {remote} failed: " + problem(push, "git push")) | |
| 391 | else: | |
| 392 | log(sd, env, f"mirror: pushed {branch} to {remote}") | |
| 393 | return "ok" | |
| 394 | ||
| 395 | ||
| 396 | # ---------------------------------------------------------------- deploy guard | |
| 397 | ||
| 398 | def compare(built, baseline, generator=None, config=None, target=None): | |
| 399 | """('no-baseline'|'generator'|'target'|'config', {}) | ('paths', {added, removed}) | ('same', {}) | | |
| 400 | ('changed', {changed}). A baseline made by another version of the generator (its code, templates, CSS), with another | |
| 401 | deploy target or another config holds back, so an automatic deploy never carries a code, template or config change.""" | |
| 402 | if baseline is None: | |
| 403 | return "no-baseline", {} | |
| 404 | if generator is not None and baseline.get("generator") != generator: | |
| 405 | return "generator", {} | |
| 406 | if target is not None and baseline.get("target") != target: | |
| 407 | return "target", {} | |
| 408 | if config is not None and baseline.get("config") != config: | |
| 409 | return "config", {} | |
| 410 | old = baseline["files"] | |
| 411 | added, removed = sorted(set(built) - set(old)), sorted(set(old) - set(built)) | |
| 412 | if added or removed: | |
| 413 | return "paths", {"added": added, "removed": removed} | |
| 414 | changed = sorted(p for p in built if built[p] != old[p]) | |
| 415 | return ("changed", {"changed": changed}) if changed else ("same", {}) | |
| 416 | ||
| 417 | ||
| 418 | def load_baseline(sd): | |
| 419 | data = read_json(os.path.join(sd, "deployed.json"), {}) | |
| 420 | return data if isinstance(data.get("files"), dict) else None | |
| 421 | ||
| 422 | ||
| 423 | def build_to_temp(env, cfg): | |
| 424 | """(temp folder, built folder, manifest) or raises. The caller removes the temp folder.""" | |
| 425 | tmp = tempfile.mkdtemp(prefix="fw-blog-watch-") | |
| 426 | out = os.path.join(tmp, "site") | |
| 427 | try: | |
| 428 | env.build(cfg, out) | |
| 429 | return tmp, out, blog.manifest(out) | |
| 430 | except BaseException: | |
| 431 | shutil.rmtree(tmp, ignore_errors=True) | |
| 432 | raise | |
| 433 | ||
| 434 | ||
| 435 | def few(paths, n=5): | |
| 436 | return ", ".join(paths[:n]) + (f" (+{len(paths) - n} more)" if len(paths) > n else "") | |
| 437 | ||
| 438 | ||
| 439 | def guard_step(sd, env, state, cfg, config_path): | |
| 440 | """Build, compare with deployed.json, deploy when it is safe. Ends the change (the caller marks it handled), | |
| 441 | except 'changed-during-build': the source moved, so the new quiet period applies.""" | |
| 442 | try: | |
| 443 | tmp, out, built = build_to_temp(env, cfg) | |
| 444 | except Exception as e: | |
| 445 | once(sd, env, state, "build", f"build failed, nothing deployed: {e}") | |
| 446 | return "build-error" | |
| 447 | try: | |
| 448 | if changed_since(sd, cfg, state): | |
| 449 | log(sd, env, "changed during build; nothing deployed") | |
| 450 | return "changed-during-build" | |
| 451 | generator = env.generator() | |
| 452 | kind, info = compare(built, load_baseline(sd), generator, blog.config_hash(cfg), blog.deploy_target(cfg)) | |
| 453 | if kind == "no-baseline": | |
| 454 | once(sd, env, state, "hold", "held back: no baseline: run fw blog deploy once") | |
| 455 | return "held" | |
| 456 | if kind in ("generator", "target", "config"): | |
| 457 | what = {"generator": "generator changed", "target": "deploy target changed", "config": "config changed"}[kind] | |
| 458 | once(sd, env, state, "hold", f"held back: {what}: run `fw blog deploy {config_path}`") | |
| 459 | return "held" | |
| 460 | if kind == "paths": | |
| 461 | a, r = info["added"], info["removed"] | |
| 462 | once(sd, env, state, "hold", f"held back: {len(a)} added, {len(r)} removed: run `fw blog deploy {config_path}`") | |
| 463 | log(sd, env, "held paths: " + few(["+" + p for p in a] + ["-" + p for p in r])) | |
| 464 | return "held" | |
| 465 | if kind == "same": | |
| 466 | log(sd, env, "no visible change") | |
| 467 | return "unchanged" | |
| 468 | try: | |
| 469 | env.deploy(cfg, out) | |
| 470 | except Exception as e: | |
| 471 | once(sd, env, state, "deploy", f"deploy failed: {e}") | |
| 472 | return "deploy-error" | |
| 473 | blog.write_deployed(sd, built, iso(env.now()), generator, cfg) | |
| 474 | log(sd, env, f"deployed: {len(info['changed'])} files changed") | |
| 475 | return "deployed" | |
| 476 | finally: | |
| 477 | shutil.rmtree(tmp, ignore_errors=True) | |
| 478 | ||
| 479 | ||
| 480 | # ---------------------------------------------------------------- one tick | |
| 481 | ||
| 482 | def tick(config_path, env=None): | |
| 483 | """One pass. Returns a word saying what happened (for the tests and --dry-run): locked, changed, idle, waiting, | |
| 484 | conflict, blocked, deployed, held, unchanged, build-error, deploy-error, changed-during-build, mirrored, | |
| 485 | config-error, noop, error (an unexpected exception: logged, notified once, never raised).""" | |
| 486 | env = env or Env() | |
| 487 | config_path = os.path.abspath(config_path) | |
| 488 | sd = blog.state_dir(config_path) | |
| 489 | os.makedirs(sd, exist_ok=True) | |
| 490 | lock = blog.acquire_lock(sd, env.now) | |
| 491 | if lock is None: | |
| 492 | since = blog.lock_since(sd) | |
| 493 | marker = os.path.join(sd, "stuck-notified") | |
| 494 | if since and env.now() - since > STUCK_AFTER and read_text(marker) != repr(since): | |
| 495 | with open(marker, "w") as f: | |
| 496 | f.write(repr(since)) | |
| 497 | text = "watch tick stuck: the lock has been held for over 30 minutes" | |
| 498 | log(sd, env, text) | |
| 499 | env.notify(text) | |
| 500 | return "locked" | |
| 501 | try: | |
| 502 | result = locked_tick(sd, env, config_path) | |
| 503 | state = read_json(os.path.join(sd, "state.json"), {}) | |
| 504 | if state.pop("last_error", None): # a clean tick: the next unexpected error notifies again | |
| 505 | save_state(sd, state) | |
| 506 | return result | |
| 507 | except Exception as e: | |
| 508 | frame = traceback.extract_tb(e.__traceback__)[-1] | |
| 509 | text = f"unexpected error: {type(e).__name__}: {e} ({os.path.basename(frame.filename)}:{frame.lineno})" | |
| 510 | state = read_json(os.path.join(sd, "state.json"), {}) | |
| 511 | if state.get("last_error") != text: | |
| 512 | state["last_error"] = text | |
| 513 | save_state(sd, state) | |
| 514 | log(sd, env, text) | |
| 515 | env.notify(text) | |
| 516 | return "error" | |
| 517 | finally: | |
| 518 | lock.close() | |
| 519 | ||
| 520 | ||
| 521 | def read_text(path): | |
| 522 | try: | |
| 523 | with open(path) as f: | |
| 524 | return f.read() | |
| 525 | except OSError: | |
| 526 | return None | |
| 527 | ||
| 528 | ||
| 529 | def locked_tick(sd, env, config_path): | |
| 530 | state = read_json(os.path.join(sd, "state.json"), {}) | |
| 531 | earlier = state.pop("config_error", None) | |
| 532 | ||
| 533 | def config_problem(text): | |
| 534 | if earlier != text: | |
| 535 | log(sd, env, text) | |
| 536 | env.notify(text) | |
| 537 | state["config_error"] = text | |
| 538 | save_state(sd, state) | |
| 539 | return "config-error" | |
| 540 | ||
| 541 | try: | |
| 542 | cfg = blog.load_config(config_path) | |
| 543 | except blog.BlogError as e: | |
| 544 | return config_problem(f"config error: {e}") | |
| 545 | if cfg["mirror"] is None and cfg["autodeploy"] is None: | |
| 546 | if earlier: | |
| 547 | save_state(sd, state) | |
| 548 | return "noop" | |
| 549 | if not os.path.isdir(cfg["_source"]): | |
| 550 | return config_problem(f"source folder {cfg['_source']} not found") | |
| 551 | if earlier: | |
| 552 | save_state(sd, state) # recovered: the next problem notifies again | |
| 553 | now = env.now() | |
| 554 | fingerprint, conflicts = current(sd, cfg) | |
| 555 | if state.get("fingerprint") != fingerprint or state.get("conflicts") != conflicts: | |
| 556 | save_state(sd, {"fingerprint": fingerprint, "conflicts": conflicts, "changed_at": now, "handled": False, | |
| 557 | "notified": [], **{k: state[k] for k in ("mirror_dirty",) if k in state}}) | |
| 558 | log(sd, env, "changed") | |
| 559 | return "changed" | |
| 560 | if state.get("handled"): | |
| 561 | return "idle" | |
| 562 | quiet = (cfg["autodeploy"] or {}).get("quiet_seconds", DEFAULT_QUIET) | |
| 563 | if now - state.get("changed_at", now) < quiet: | |
| 564 | if "waiting" not in state.setdefault("notified", []): | |
| 565 | state["notified"].append("waiting") | |
| 566 | log(sd, env, f"waiting: {quiet} s without changes needed") | |
| 567 | save_state(sd, state) | |
| 568 | return "waiting" | |
| 569 | try: | |
| 570 | return decide(sd, env, state, cfg, config_path, conflicts) | |
| 571 | finally: | |
| 572 | save_state(sd, state) | |
| 573 | ||
| 574 | ||
| 575 | def decide(sd, env, state, cfg, config_path, conflicts): | |
| 576 | if conflicts: | |
| 577 | once(sd, env, state, "conflict", "Dropbox conflict file(s) in the source, nothing mirrored or deployed: " | |
| 578 | + few(conflicts, 3)) | |
| 579 | return "conflict" | |
| 580 | result = "mirrored" | |
| 581 | if cfg["mirror"] is not None: | |
| 582 | if mirror_step(sd, env, state, cfg) == "blocked": | |
| 583 | return "blocked" | |
| 584 | if changed_since(sd, cfg, state): | |
| 585 | log(sd, env, "changed during mirror; nothing built") | |
| 586 | return "changed-during-build" | |
| 587 | if cfg["autodeploy"] is not None: | |
| 588 | result = guard_step(sd, env, state, cfg, config_path) | |
| 589 | if result == "changed-during-build": | |
| 590 | return result | |
| 591 | state["handled"] = True | |
| 592 | return result | |
| 593 | ||
| 594 | ||
| 595 | # ---------------------------------------------------------------- --dry-run | |
| 596 | ||
| 597 | def dry_run(config_path, env): | |
| 598 | """Shows what a tick would do right now (ignoring the quiet period); writes nothing.""" | |
| 599 | cfg = blog.load_config(config_path) | |
| 600 | say = env.echo | |
| 601 | if cfg["mirror"] is None and cfg["autodeploy"] is None: | |
| 602 | say("the config has neither mirror nor autodeploy: nothing to do") | |
| 603 | return | |
| 604 | if not os.path.isdir(cfg["_source"]): | |
| 605 | say(f"source folder {cfg['_source']} not found") | |
| 606 | return | |
| 607 | _, conflicts = scan(cfg["_source"]) | |
| 608 | if conflicts: | |
| 609 | say("would stop: Dropbox conflict file(s): " + few(conflicts)) | |
| 610 | return | |
| 611 | if cfg["mirror"] is not None: | |
| 612 | repo, branch, remote, extra = mirror_settings(cfg) | |
| 613 | dirty_ok = read_json(os.path.join(blog.state_dir(cfg["_path"]), "state.json"), {}).get("mirror_dirty") | |
| 614 | blocked = mirror_blocker(env, cfg, repo, branch, dirty_ok) | |
| 615 | if blocked: | |
| 616 | say("mirror would be skipped: " + blocked) | |
| 617 | return | |
| 618 | if not os.path.isdir(repo) or not os.listdir(repo): | |
| 619 | say(f"mirror: would run git init -b {branch} {repo}") | |
| 620 | else: | |
| 621 | say(f"mirror: rsync would change (source -> {repo}):") | |
| 622 | result = env.run(rsync_command(cfg["_source"], repo, extra, dry_run=True), timeout=RSYNC_TIMEOUT) | |
| 623 | say(result.stdout.rstrip() or " (nothing)") | |
| 624 | say("mirror: would git add -A, commit if anything changed" + (f", and push to {remote}" if remote else "")) | |
| 625 | if cfg["autodeploy"] is not None: | |
| 626 | try: | |
| 627 | tmp, _, built = build_to_temp(env, cfg) | |
| 628 | except Exception as e: | |
| 629 | say(f"deploy: the build fails, nothing would be deployed: {e}") | |
| 630 | return | |
| 631 | shutil.rmtree(tmp, ignore_errors=True) | |
| 632 | kind, info = compare(built, load_baseline(blog.state_dir(cfg["_path"])), env.generator(), | |
| 633 | blog.config_hash(cfg), blog.deploy_target(cfg)) | |
| 634 | say({"no-baseline": "deploy: would hold back (no baseline: run fw blog deploy once)", | |
| 635 | "generator": "deploy: would hold back (generator changed: run fw blog deploy)", | |
| 636 | "target": "deploy: would hold back (deploy target changed: run fw blog deploy)", | |
| 637 | "config": "deploy: would hold back (config changed: run fw blog deploy)", | |
| 638 | "paths": f"deploy: would hold back ({len(info.get('added', []))} added, {len(info.get('removed', []))} " | |
| 639 | "removed paths)", | |
| 640 | "same": "deploy: no visible change, nothing to deploy", | |
| 641 | "changed": f"deploy: would upload ({len(info.get('changed', []))} files changed): " | |
| 642 | + few(info.get("changed", []))}[kind]) | |
| 643 | ||
| 644 | ||
| 645 | # ---------------------------------------------------------------- launchd agent | |
| 646 | ||
| 647 | def plist_data(config_path, name, sd, fw=None): | |
| 648 | fw = fw or os.path.realpath(os.path.join(HERE, "..", "bin", "fw")) | |
| 649 | return {"Label": LABEL + name, | |
| 650 | "ProgramArguments": [fw, "blog", "watch", os.path.abspath(config_path), "--once"], | |
| 651 | "StartInterval": TICK_SECONDS, "RunAtLoad": True, | |
| 652 | "EnvironmentVariables": {"PATH": LAUNCH_PATH}, | |
| 653 | "StandardOutPath": os.path.join(sd, "launchd.log"), "StandardErrorPath": os.path.join(sd, "launchd.log"), | |
| 654 | "ProcessType": "Background", "Nice": 10} | |
| 655 | ||
| 656 | ||
| 657 | def plist_text(config_path, fw=None): | |
| 658 | config_path = os.path.abspath(config_path) | |
| 659 | return plistlib.dumps(plist_data(config_path, blog.watch_name(config_path), blog.state_dir(config_path), fw)).decode("utf-8") | |
| 660 | ||
| 661 | ||
| 662 | def agent_path(config_path, agents_dir=None): | |
| 663 | return os.path.join(agents_dir or os.path.expanduser("~/Library/LaunchAgents"), | |
| 664 | LABEL + blog.watch_name(config_path) + ".plist") | |
| 665 | ||
| 666 | ||
| 667 | def install(config_path, run=run_command, agents_dir=None, uid=None, fw=None, sleep=time.sleep): | |
| 668 | """Writes the plist and loads it. Reinstalling replaces a loaded agent.""" | |
| 669 | blog.load_config(config_path) # a typo in the config is better found now than every minute | |
| 670 | path = agent_path(config_path, agents_dir) | |
| 671 | os.makedirs(os.path.dirname(path), exist_ok=True) | |
| 672 | os.makedirs(blog.state_dir(config_path), exist_ok=True) | |
| 673 | domain = f"gui/{os.getuid() if uid is None else uid}" | |
| 674 | with open(path, "w", encoding="utf-8") as f: | |
| 675 | f.write(plist_text(config_path, fw)) | |
| 676 | run(["launchctl", "bootout", domain, path]) # fails when it was not loaded: fine | |
| 677 | for attempt in range(5): # bootstrap can fail for a moment right after a bootout | |
| 678 | result = run(["launchctl", "bootstrap", domain, path]) | |
| 679 | if not result.returncode: | |
| 680 | break | |
| 681 | if attempt < 4: | |
| 682 | sleep(1) | |
| 683 | if result.returncode: | |
| 684 | raise blog.BlogError(f"launchctl bootstrap failed: {(result.stderr or '').strip()}") | |
| 685 | return path | |
| 686 | ||
| 687 | ||
| 688 | def uninstall(config_path, run=run_command, agents_dir=None, uid=None): | |
| 689 | path = agent_path(config_path, agents_dir) | |
| 690 | run(["launchctl", "bootout", f"gui/{os.getuid() if uid is None else uid}", path]) | |
| 691 | if os.path.exists(path): | |
| 692 | os.remove(path) | |
| 693 | return path | |
| 694 | ||
| 695 | ||
| 696 | # ---------------------------------------------------------------- command line | |
| 697 | ||
| 698 | def main(argv): | |
| 699 | ap = argparse.ArgumentParser(prog="blog_watch.py", description=__doc__.split("\n")[0]) | |
| 700 | ap.add_argument("words", nargs="+", metavar="[install|uninstall] config") | |
| 701 | ap.add_argument("--once", action="store_true", help="one tick, then exit (what launchd runs)") | |
| 702 | ap.add_argument("--dry-run", action="store_true", help="show what a tick would do now; write nothing") | |
| 703 | ap.add_argument("--print", action="store_true", dest="show", help="with install: only print the launchd plist") | |
| 704 | args = ap.parse_args(argv[1:]) | |
| 705 | action = args.words[0] if args.words[0] in ("install", "uninstall") else None | |
| 706 | rest = args.words[1:] if action else args.words | |
| 707 | if len(rest) != 1: | |
| 708 | ap.error("needs exactly one config file") | |
| 709 | config = os.path.abspath(os.path.expanduser(rest[0])) | |
| 710 | try: | |
| 711 | if action == "install": | |
| 712 | if args.show: | |
| 713 | print(plist_text(config), end="") | |
| 714 | else: | |
| 715 | print("installed " + install(config)) | |
| 716 | elif action == "uninstall": | |
| 717 | print("removed " + uninstall(config)) | |
| 718 | elif args.dry_run: | |
| 719 | dry_run(config, Env()) | |
| 720 | elif args.once: | |
| 721 | result = tick(config) | |
| 722 | if result in ("config-error", "build-error", "deploy-error", "blocked", "error"): # the exit code stays 0: launchd must not throttle the agent | |
| 723 | print(f"blog_watch: {result} (see {blog.state_dir(config)}/watch.log)", file=sys.stderr) | |
| 724 | else: | |
| 725 | while True: | |
| 726 | tick(config) | |
| 727 | time.sleep(TICK_SECONDS) | |
| 728 | except blog.BlogError as e: | |
| 729 | raise SystemExit(f"blog_watch.py: {e}") | |
| 730 | except KeyboardInterrupt: | |
| 731 | pass | |
| 732 | ||
| 733 | ||
| 734 | if __name__ == "__main__": | |
| 735 | main(sys.argv) |