fuzzyworld-tools / commit
deploy-tools: npm run deploy with a ledger on the server (Q110)
README.md | 14 ++--- bin/deploy-tools | 122 +++++++++++++++++++++++++++++++++++++ package.json | 8 +-- tests/test_deploy_tools.py | 146 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 276 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index b4d6b83..fa46259 100644 --- a/README.md +++ b/README.md @@ -290,7 +290,7 @@ the file still runs that repo's other jobs; tag a deploy job `secrets` to keep i Without any, jobs that deploy elsewhere (e.g. `dreamhost`'s rsync, engye's store uploads) fail. Anything on the laptop is as safe as the laptop (FileVault on); if it is lost, rotate the secrets. -**The laptop as a runner:** `runner/install-laptop.sh` (`npm run update-laptop-runner`) installs a launchd job (every 2 minutes) +**The laptop as a runner:** `runner/install-laptop.sh` (`bin/deploy-tools laptop-runner`) installs a launchd job (every 2 minutes) with `FW_RUNNER_POLITE=1`, so the runner claims a job only when the Mac is on AC power, at least `FW_RUNNER_MIN_IDLE` (50) percent of the CPU is idle, and Docker is running; it checks before each job, never during one, and its containers get a low CPU share (`FW_RUNNER_CPUS`, default 2 there, is the real limit). Settings live in `~/.fw-runner/runner.env`, @@ -502,17 +502,15 @@ runner/install-laptop.sh # the laptop as a polite second runner (launchd, Do vscode-web/install.sh # VS Code in the browser at apps.fuzzyworld.net/vscode/ ``` -The same actions as npm scripts (`npm run` lists them): +npm scripts (`npm run` lists them): | Script | Runs | |---|---| -| `npm run update-server` | `server/install.sh` | -| `npm run update-server-and-republish` | `server/install.sh --publish` | -| `npm run republish-all` | `bin/fw publish-all` | +| `npm run deploy` | `bin/deploy-tools`: install the server and republish every repo, but only if `server/` changed since the last deploy | +| `npm run deploy-status` | `bin/deploy-tools status`: what is deployed where, and what is pending | | `npm run list-repos` | `bin/fw list` | -| `npm run update-runner` | `runner/install.sh` | -| `npm run update-laptop-runner` | `runner/install-laptop.sh` | -| `npm run update-vscode` | `vscode-web/install.sh` | + +`bin/deploy-tools [server|runner|laptop-runner|vscode|all] [--force]` wraps the install scripts (`server/install.sh`, `runner/install.sh`, `runner/install-laptop.sh`, `vscode-web/install.sh`) with a ledger kept on the server, `~/fw/state/deployed.json`. For each target it records the commit and a content hash of the folders that target ships (`server/`, `runner/`, `vscode-web/`). A target is redeployed only when that hash differs, so uncommitted edits count, unrelated commits (README, tests) do not, and committing the files you already deployed does not trigger a second deploy. The ledger is written only after the script succeeds, and any laptop sees the same state, except `laptop-runner`, which is installed per machine and so is recorded per hostname. Gitignored files in those folders are not hashed. `server` includes the republish, since pages are generated by the installed `gen.py`. For flags `deploy-tools` does not pass (`--authorize`, `--remove`, `--out`) run the install script directly; to regenerate pages without installing, run `bin/fw publish-all`. `install.sh` copies `server/` from the working copy, not from a commit, so uncommitted changes there go live too. diff --git a/bin/deploy-tools b/bin/deploy-tools new file mode 100755 index 0000000..761e3eb --- /dev/null +++ b/bin/deploy-tools @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Deploy fuzzyworld-tools itself, only when what a target ships has changed. + + bin/deploy-tools [server] deploy the server (install + republish all) if server/ changed + bin/deploy-tools all every target below + bin/deploy-tools runner vscode named targets + bin/deploy-tools status what is deployed, and what is pending + --force deploy even if unchanged + +A ledger on the server (~/fw/state/deployed.json on $FW_HOST, default `dh`) records, per target, the +commit and the content hash of the folders it ships. "Changed" means that content hash differs, so it covers +uncommitted edits too, and committing the very files you already deployed does not redeploy. +The ledger is written only after a target's script succeeds. `laptop-runner` is per machine, so its +ledger key includes this Mac's hostname. Gitignored files inside a shipped folder are not hashed (and +are never "changes" here) even though rsync may upload them. +""" +import json +import os +import socket +import subprocess +import sys +import tempfile +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HOST = os.environ.get("FW_HOST", "dh") +LEDGER = "fw/state/deployed.json" # on HOST, relative to its home +# target: (what it ships, command). `server` also republishes every repo, because the pages are +# generated by the installed gen.py: new generator code only shows once they are regenerated. +TARGETS = { + "server": (["server"], ["server/install.sh", "--publish"]), + "runner": (["runner"], ["runner/install.sh"]), + "laptop-runner": (["runner"], ["runner/install-laptop.sh"]), + "vscode": (["vscode-web"], ["vscode-web/install.sh"]), +} + + +def git(*args, env=None): + return subprocess.run(["git", "-C", ROOT, *args], check=True, capture_output=True, text=True, env=env).stdout.strip() + + +def content_hash(paths): + """Hash of the working-tree content of `paths` (untracked files included, ignored ones not). + Equals the committed tree hash once the same content is committed.""" + index = os.path.join(git("rev-parse", "--absolute-git-dir"), "index") + with tempfile.TemporaryDirectory() as tmp: + copy = os.path.join(tmp, "index") + if os.path.exists(index): + with open(index, "rb") as src, open(copy, "wb") as dst: + dst.write(src.read()) + env = {**os.environ, "GIT_INDEX_FILE": copy} + try: + git("add", "-A", "--", *paths, env=env) + tree = git("write-tree", env=env) + return ",".join(git("rev-parse", f"{tree}:{p}", env=env) for p in paths) + except subprocess.CalledProcessError: + raise SystemExit(f"nothing to ship: {', '.join(paths)} is missing or holds only ignored files") + + +def load(): + """The ledger from the server; {} if it does not exist yet. An unreachable server is an error.""" + r = subprocess.run(["ssh", HOST, f"cat {LEDGER} 2>/dev/null || true"], capture_output=True, text=True) + if r.returncode != 0: + raise SystemExit(f"cannot reach {HOST}: {r.stderr.strip()}") + if not r.stdout.strip(): + return {} + try: + return json.loads(r.stdout) + except ValueError: + raise SystemExit(f"{HOST}:{LEDGER} is not valid JSON; fix or delete it (saving now would wipe the other targets' records)") + + +def save(ledger): + cmd = f"mkdir -p ~/fw/state && cat > {LEDGER}.tmp && mv {LEDGER}.tmp {LEDGER}" + r = subprocess.run(["ssh", HOST, cmd], input=json.dumps(ledger, indent=2) + "\n", text=True) + if r.returncode != 0: + raise SystemExit("the deploy succeeded but the ledger could not be written; the next run will redeploy") + + +def key(name): + """Ledger key: the laptop runner is installed per machine, the rest once for the server.""" + return f"{name}@{socket.gethostname()}" if name == "laptop-runner" else name + + +def main(argv): + force = "--force" in argv + args = [a for a in argv[1:] if a != "--force"] or ["server"] + names = list(TARGETS) if args == ["all"] else args + if args != ["status"]: + for n in names: + if n not in TARGETS: + raise SystemExit(f"unknown target {n!r}; targets: {', '.join(TARGETS)}, all, status") + ledger = load() + if args == ["status"]: + head = git("rev-parse", "--short", "HEAD") + for name, (paths, _) in TARGETS.items(): + rec = ledger.get(key(name)) + state = ("never deployed" if not rec else + "up to date" if rec["hash"] == content_hash(paths) else "PENDING (changed since deploy)") + when = f' commit {rec["commit"]}, {rec["at"]}' if rec else "" + print(f"{name:14} {state}{when}") + print(f"(HEAD is {head}; uncommitted edits count as changes)") + return 0 + for name in names: + paths, cmd = TARGETS[name] + h = content_hash(paths) + if not force and ledger.get(key(name), {}).get("hash") == h: + print(f"{name}: already deployed (commit {ledger[key(name)]['commit']}), nothing to do. --force to redo.") + continue + print(f"{name}: deploying ({' '.join(cmd)})") + if subprocess.run([os.path.join(ROOT, cmd[0]), *cmd[1:]], cwd=ROOT).returncode != 0: + print(f"{name}: FAILED, ledger not updated", file=sys.stderr) + return 1 + dirty = bool(git("status", "--porcelain", "--", *paths)) + ledger[key(name)] = {"hash": h, "commit": git("rev-parse", "--short", "HEAD") + ("+uncommitted" if dirty else ""), + "at": time.strftime("%Y-%m-%d %H:%M")} + save(ledger) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/package.json b/package.json index 604d771..6db56ec 100644 --- a/package.json +++ b/package.json @@ -5,13 +5,9 @@ "type": "module", "scripts": { "dev": "server/preview.sh", - "update-server": "server/install.sh", - "update-server-and-republish": "server/install.sh --publish", - "republish-all": "bin/fw publish-all", + "deploy": "bin/deploy-tools", + "deploy-status": "bin/deploy-tools status", "list-repos": "bin/fw list", - "update-runner": "runner/install.sh", - "update-laptop-runner": "runner/install-laptop.sh", - "update-vscode": "vscode-web/install.sh", "prepare-usage-for-pr": "python3 lib/usage.py --trailers" }, "bin": { diff --git a/tests/test_deploy_tools.py b/tests/test_deploy_tools.py new file mode 100644 index 0000000..d3a4d64 --- /dev/null +++ b/tests/test_deploy_tools.py @@ -0,0 +1,146 @@ +import importlib.machinery +import importlib.util +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from unittest import mock + +PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "bin", "deploy-tools") +loader = importlib.machinery.SourceFileLoader("deploy_tools", PATH) +dt = importlib.util.module_from_spec(importlib.util.spec_from_loader("deploy_tools", loader)) +loader.exec_module(dt) + + +class DeployTools(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root, True) + self.git = lambda *a: subprocess.run(["git", "-C", self.root, *a], check=True, capture_output=True, text=True, + env={**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@x", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@x"}).stdout.strip() + self.git("init", "-q") + os.makedirs(os.path.join(self.root, "server")) + self.write("server/a.txt", "one") + self.write("other.txt", "x") + self.write(".gitignore", "*.log\n") + self.git("add", "-A") + self.git("commit", "-qm", "c") + patcher = mock.patch.object(dt, "ROOT", self.root) + patcher.start() + self.addCleanup(patcher.stop) + self.ledger = {} # stands in for the server's ledger + for name, fn in [("load", lambda: json.loads(json.dumps(self.ledger))), + ("save", lambda ledger: self.ledger.update(json.loads(json.dumps(ledger))))]: + patcher = mock.patch.object(dt, name, fn) + patcher.start() + self.addCleanup(patcher.stop) + + def write(self, rel, text): + with open(os.path.join(self.root, rel), "w") as f: + f.write(text) + + def test_hash_follows_content_not_commits_or_unrelated_files(self): + h = dt.content_hash(["server"]) + self.assertEqual(h, self.git("rev-parse", "HEAD:server")) + self.write("other.txt", "changed") # outside the target + self.write("server/ignored.log", "noise") # gitignored + self.assertEqual(dt.content_hash(["server"]), h) + self.write("server/new.txt", "untracked") # untracked counts + dirty = dt.content_hash(["server"]) + self.assertNotEqual(dirty, h) + self.git("add", "-A") + self.git("commit", "-qm", "d") # committing the same content gives the same hash + self.assertEqual(dt.content_hash(["server"]), dirty) + self.assertEqual(dirty, self.git("rev-parse", "HEAD:server")) + + def test_deploys_only_when_changed_and_records_ledger(self): + calls = [] + ok = mock.Mock(returncode=0) + targets = {"server": (["server"], ["server/install.sh"])} + real_run = subprocess.run + + def fake(cmd, **kw): + if cmd[0] == "git": + return real_run(cmd, **kw) + calls.append(cmd) + return ok + with mock.patch.object(dt, "TARGETS", targets): + with mock.patch.object(dt.subprocess, "run", side_effect=fake): + self.assertEqual(dt.main(["x", "server"]), 0) + self.assertEqual(len(calls), 1) + self.assertEqual(dt.main(["x", "server"]), 0) # unchanged: skipped + self.assertEqual(len(calls), 1) + self.write("server/a.txt", "two") + dt.main(["x", "server"]) + self.assertEqual(len(calls), 2) + dt.main(["x", "server", "--force"]) + self.assertEqual(len(calls), 3) + rec = self.ledger["server"] + self.assertTrue(rec["commit"].endswith("+uncommitted")) + + def test_failed_deploy_leaves_ledger_alone(self): + real_run = subprocess.run + + def fake(cmd, **kw): + return real_run(cmd, **kw) if cmd[0] == "git" else mock.Mock(returncode=1) + with mock.patch.object(dt, "TARGETS", {"server": (["server"], ["server/install.sh"])}), \ + mock.patch.object(dt.subprocess, "run", side_effect=fake): + self.assertEqual(dt.main(["x", "server"]), 1) + self.assertEqual(self.ledger, {}) + + def test_content_hash_leaves_the_real_index_alone(self): + index = os.path.join(self.root, ".git", "index") + self.write("server/staged.txt", "s") + self.git("add", "server/staged.txt") + self.write("server/a.txt", "edited, not staged") + with open(index, "rb") as f: + before = f.read() + cached = self.git("diff", "--cached") + dt.content_hash(["server"]) + with open(index, "rb") as f: + self.assertEqual(f.read(), before) + self.assertEqual(self.git("diff", "--cached"), cached) + + def test_content_hash_without_an_index_file_and_with_a_missing_folder(self): + os.remove(os.path.join(self.root, ".git", "index")) + self.assertEqual(dt.content_hash(["server"]), self.git("rev-parse", "HEAD:server")) + with self.assertRaises(SystemExit): + dt.content_hash(["vscode-web"]) + + def test_clean_deploy_has_no_uncommitted_suffix_and_laptop_runner_is_per_machine(self): + real_run = subprocess.run + fake = lambda cmd, **kw: real_run(cmd, **kw) if cmd[0] == "git" else mock.Mock(returncode=0) + targets = {"laptop-runner": (["server"], ["server/x.sh"])} + with mock.patch.object(dt, "TARGETS", targets), mock.patch.object(dt.subprocess, "run", side_effect=fake), \ + mock.patch.object(dt.socket, "gethostname", return_value="mac-a"): + dt.main(["x", "laptop-runner"]) + self.assertEqual(list(self.ledger), ["laptop-runner@mac-a"]) + self.assertFalse(self.ledger["laptop-runner@mac-a"]["commit"].endswith("+uncommitted")) + calls = [] + fake2 = lambda cmd, **kw: real_run(cmd, **kw) if cmd[0] == "git" else calls.append((cmd, kw)) or mock.Mock(returncode=0) + with mock.patch.object(dt, "TARGETS", targets), mock.patch.object(dt.subprocess, "run", side_effect=fake2), \ + mock.patch.object(dt.socket, "gethostname", return_value="mac-b"): + dt.main(["x", "laptop-runner"]) # another Mac: not deployed there yet + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][1]["cwd"], self.root) + self.assertTrue(os.path.isabs(calls[0][0][0])) + + def test_unknown_target_fails_before_any_server_access(self): + with mock.patch.object(dt, "load", side_effect=AssertionError("must not reach the server")): + with self.assertRaises(SystemExit): + dt.main(["x", "typo"]) + + def test_status_lists_targets(self): + self.ledger["server"] = {"hash": "stale", "commit": "abc1234", "at": "2026-11-04 10:00"} + with mock.patch("builtins.print") as pr: + self.assertEqual(dt.main(["x", "status"]), 0) + text = "\n".join(c.args[0] for c in pr.call_args_list) + self.assertIn("PENDING", text) + self.assertIn("never deployed", text) + + +if __name__ == "__main__": + unittest.main()