fuzzyworld

fuzzyworld-tools / files / bin / deploy-tools

deploy-tools

8.1 KB · history · blame

1#!/usr/bin/env python3
2"""Deploy this repo's targets, only when what a target ships has changed.
3
4 bin/deploy-tools [server] deploy the server (install + republish all) if server/ changed
5 bin/deploy-tools all every target
6 bin/deploy-tools runner vscode named targets
7 bin/deploy-tools status what is deployed, and what is pending
8 --force deploy even if unchanged
9
10The targets are declared in package.json, under "fuzzyworld": {"deploy": {<target>: {"paths": [folders it ships],
11"command": [script, args...], "description": "...", "perMachine": true?}}}.
12
13A ledger on the server (~/fw/state/deployed.json on $FW_HOST, default `dh`) records, per "<repo>/<target>", the
14commit and the content hash of the folders it ships. "Changed" means that content hash differs, so it covers
15uncommitted edits too, and committing the very files you already deployed does not redeploy. The ledger is
16written only after a target's script succeeds. A `perMachine` target (the laptop runner) is installed per Mac,
17so its ledger key ends in @<hostname>. Gitignored files inside a shipped folder are not hashed (and are never
18"changes" here) even though rsync may upload them. `fw deploys --json` on the server reads the same ledger
19for `fw admin`.
20"""
21import json
22import os
23import posixpath
24import re
25import socket
26import subprocess
27import sys
28import tempfile
29import time
30
31ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
32HOST = os.environ.get("FW_HOST", "dh")
33LEDGER = "fw/state/deployed.json" # on HOST, relative to its home
34NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*\Z") # as server/deploys.py: it drops targets that fail this
35PER_MACHINE = set() # targets whose ledger key carries the hostname
36
37
38def clean_path(p):
39 """A folder inside the repo, written plainly; as server/deploys.py clean_path (the two must agree)."""
40 return (isinstance(p, str) and p != "" and posixpath.normpath(p) == p and not p.startswith(("/", "../")) and p != ".."
41 and not any(c in p for c in ":*?[]\\"))
42
43
44def declared_targets():
45 """(repo name on the server, {target: (paths, command)}) from package.json. `server` should include the republish,
46 because pages are generated by the installed gen.py: new generator code only shows once they are regenerated."""
47 with open(os.path.join(ROOT, "package.json")) as f:
48 pkg = json.load(f)
49 found = {}
50 for name, t in ((pkg.get("fuzzyworld") or {}).get("deploy") or {}).items():
51 paths, cmd = (t.get("paths"), t.get("command")) if isinstance(t, dict) else (None, None)
52 ok = (NAME_RE.match(name) and isinstance(paths, list) and paths and all(clean_path(p) for p in paths)
53 and isinstance(cmd, list) and cmd and all(isinstance(c, str) for c in cmd) and clean_path(cmd[0]))
54 if not ok:
55 raise SystemExit(f'package.json: fuzzyworld.deploy.{name} needs a plain name, "paths" (folders inside the repo, '
56 'written plainly: no "./", trailing "/" or "..") and "command" (a list)')
57 found[name] = (paths, cmd)
58 if t.get("perMachine"):
59 PER_MACHINE.add(name)
60 repo = (pkg.get("fuzzyworld") or {}).get("repo") or pkg.get("name") or os.path.basename(ROOT)
61 if not NAME_RE.match(str(repo)):
62 raise SystemExit('package.json: set "fuzzyworld": {"repo": "<name of the repo on the server>"} (the package name is not usable)')
63 return repo, found
64
65
66REPO, TARGETS = declared_targets()
67
68
69def git(*args, env=None):
70 return subprocess.run(["git", "-C", ROOT, *args], check=True, capture_output=True, text=True, env=env).stdout.strip()
71
72
73def content_hash(paths):
74 """Hash of the working-tree content of `paths` (untracked files included, ignored ones not).
75 Equals the committed tree hash once the same content is committed."""
76 index = os.path.join(git("rev-parse", "--absolute-git-dir"), "index")
77 with tempfile.TemporaryDirectory() as tmp:
78 copy = os.path.join(tmp, "index")
79 if os.path.exists(index):
80 with open(index, "rb") as src, open(copy, "wb") as dst:
81 dst.write(src.read())
82 env = {**os.environ, "GIT_INDEX_FILE": copy}
83 try:
84 git("add", "-A", "--", *paths, env=env)
85 tree = git("write-tree", env=env)
86 return ",".join(git("rev-parse", f"{tree}:{p}", env=env) for p in paths)
87 except subprocess.CalledProcessError:
88 raise SystemExit(f"nothing to ship: {', '.join(paths)} is missing or holds only ignored files")
89
90
91def load():
92 """The ledger from the server; {} if it does not exist yet. An unreachable server is an error."""
93 r = subprocess.run(["ssh", HOST, f"cat {LEDGER} 2>/dev/null || true"], capture_output=True, text=True)
94 if r.returncode != 0:
95 raise SystemExit(f"cannot reach {HOST}: {r.stderr.strip()}")
96 if not r.stdout.strip():
97 return {}
98 try:
99 ledger = json.loads(r.stdout)
100 except ValueError:
101 raise SystemExit(f"{HOST}:{LEDGER} is not valid JSON; fix or delete it (saving now would wipe the other targets' records)")
102 if not isinstance(ledger, dict):
103 raise SystemExit(f"{HOST}:{LEDGER} is not a JSON object; fix or delete it (saving now would wipe the other targets' records)")
104 return ledger
105
106
107def migrate(ledger):
108 """Before Q111 the keys were "<target>" (and "laptop-runner@<host>"); now "<repo>/<target>". Rename each legacy
109 key of a declared target in memory, unless the new key exists; the next save writes the result."""
110 for name in TARGETS:
111 old = name + (f"@{socket.gethostname()}" if name in PER_MACHINE else "")
112 if old in ledger and key(name) not in ledger:
113 ledger[key(name)] = ledger.pop(old)
114 return ledger
115
116
117def save(ledger):
118 cmd = f"mkdir -p ~/fw/state && cat > {LEDGER}.tmp && mv {LEDGER}.tmp {LEDGER}"
119 r = subprocess.run(["ssh", HOST, cmd], input=json.dumps(ledger, indent=2) + "\n", text=True)
120 if r.returncode != 0:
121 raise SystemExit("the deploy succeeded but the ledger could not be written; the next run will redeploy")
122
123
124def key(name):
125 """Ledger key "<repo>/<target>"; a per-machine target (the laptop runner) also carries this Mac's hostname."""
126 return f"{REPO}/{name}" + (f"@{socket.gethostname()}" if name in PER_MACHINE else "")
127
128
129def main(argv):
130 force = "--force" in argv
131 args = [a for a in argv[1:] if a != "--force"] or ["server"]
132 names = list(TARGETS) if args == ["all"] else args
133 if args != ["status"]:
134 for n in names:
135 if n not in TARGETS:
136 raise SystemExit(f"unknown target {n!r}; targets: {', '.join(TARGETS)}, all, status")
137 ledger = migrate(load())
138 if args == ["status"]:
139 head = git("rev-parse", "--short", "HEAD")
140 for name, (paths, _) in TARGETS.items():
141 rec = ledger.get(key(name))
142 rec = rec if isinstance(rec, dict) else None
143 state = ("never deployed" if not rec else
144 "up to date" if rec.get("hash") == content_hash(paths) else "PENDING (changed since deploy)")
145 when = f' commit {rec.get("commit", "?")}, {rec.get("at", "?")}' if rec else ""
146 print(f"{name:14} {state}{when}")
147 print(f"(HEAD is {head}; uncommitted edits count as changes)")
148 return 0
149 for name in names:
150 paths, cmd = TARGETS[name]
151 h = content_hash(paths)
152 rec = ledger.get(key(name))
153 rec = rec if isinstance(rec, dict) else {}
154 if not force and rec.get("hash") == h:
155 print(f"{name}: already deployed (commit {rec.get('commit', '?')}), nothing to do. --force to redo.")
156 continue
157 print(f"{name}: deploying ({' '.join(cmd)})")
158 if subprocess.run([os.path.join(ROOT, cmd[0]), *cmd[1:]], cwd=ROOT).returncode != 0:
159 print(f"{name}: FAILED, ledger not updated", file=sys.stderr)
160 return 1
161 dirty = bool(git("status", "--porcelain", "--", *paths))
162 ledger[key(name)] = {"hash": h, "commit": git("rev-parse", "--short", "HEAD") + ("+uncommitted" if dirty else ""),
163 "at": time.strftime("%Y-%m-%d %H:%M")} # laptop time
164 save(ledger)
165 return 0
166
167
168if __name__ == "__main__":
169 sys.exit(main(sys.argv))