fuzzyworld-tools / commit
Deploy tab in fw admin; deploy targets declared in package.json (Q111 stage 1)
bin/deploy-tools | 64 ++++++++++++----- lib/admin.py | 61 +++++++++++++++- package.json | 31 +++++++-- server/deploys.py | 170 +++++++++++++++++++++++++++++++++++++++++++++ server/fw | 4 +- tests/test_admin.py | 43 +++++++++++- tests/test_deploy_tools.py | 37 ++++++++-- tests/test_deploys.py | 164 +++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 541 insertions(+), 33 deletions(-) diff --git a/bin/deploy-tools b/bin/deploy-tools index 761e3eb..907d5d8 100755 --- a/bin/deploy-tools +++ b/bin/deploy-tools @@ -1,21 +1,27 @@ #!/usr/bin/env python3 -"""Deploy fuzzyworld-tools itself, only when what a target ships has changed. +"""Deploy this repo's targets, only when what a target ships has changed. bin/deploy-tools [server] deploy the server (install + republish all) if server/ changed - bin/deploy-tools all every target below + bin/deploy-tools all every target bin/deploy-tools runner vscode named targets bin/deploy-tools status what is deployed, and what is pending --force deploy even if unchanged -A ledger on the server (~/fw/state/deployed.json on $FW_HOST, default `dh`) records, per target, the +The targets are declared in package.json, under "fuzzyworld": {"deploy": {<target>: {"paths": [folders it ships], +"command": [script, args...], "description": "...", "perMachine": true?}}}. + +A ledger on the server (~/fw/state/deployed.json on $FW_HOST, default `dh`) records, per "<repo>/<target>", the commit and the content hash of the folders it ships. "Changed" means that content hash differs, so it covers -uncommitted edits too, and committing the very files you already deployed does not redeploy. -The ledger is written only after a target's script succeeds. `laptop-runner` is per machine, so its -ledger key includes this Mac's hostname. Gitignored files inside a shipped folder are not hashed (and -are never "changes" here) even though rsync may upload them. +uncommitted edits too, and committing the very files you already deployed does not redeploy. The ledger is +written only after a target's script succeeds. A `perMachine` target (the laptop runner) is installed per Mac, +so its ledger key ends in @<hostname>. Gitignored files inside a shipped folder are not hashed (and are never +"changes" here) even though rsync may upload them. `fw deploys --json` on the server reads the same ledger +for `fw admin`. """ import json import os +import posixpath +import re import socket import subprocess import sys @@ -25,14 +31,38 @@ import time ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) HOST = os.environ.get("FW_HOST", "dh") LEDGER = "fw/state/deployed.json" # on HOST, relative to its home -# target: (what it ships, command). `server` also republishes every repo, because the pages are -# generated by the installed gen.py: new generator code only shows once they are regenerated. -TARGETS = { - "server": (["server"], ["server/install.sh", "--publish"]), - "runner": (["runner"], ["runner/install.sh"]), - "laptop-runner": (["runner"], ["runner/install-laptop.sh"]), - "vscode": (["vscode-web"], ["vscode-web/install.sh"]), -} +NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*\Z") # as server/deploys.py: it drops targets that fail this +PER_MACHINE = set() # targets whose ledger key carries the hostname + + +def clean_path(p): + """A folder inside the repo, written plainly; as server/deploys.py clean_path (the two must agree).""" + return isinstance(p, str) and p != "" and posixpath.normpath(p) == p and not p.startswith(("/", "../")) and p != ".." + + +def declared_targets(): + """(repo name on the server, {target: (paths, command)}) from package.json. `server` should include the republish, + because pages are generated by the installed gen.py: new generator code only shows once they are regenerated.""" + with open(os.path.join(ROOT, "package.json")) as f: + pkg = json.load(f) + found = {} + for name, t in ((pkg.get("fuzzyworld") or {}).get("deploy") or {}).items(): + paths, cmd = t.get("paths"), t.get("command") + ok = (NAME_RE.match(name) and isinstance(paths, list) and paths and all(clean_path(p) for p in paths) + and isinstance(cmd, list) and cmd and all(isinstance(c, str) for c in cmd)) + if not ok: + raise SystemExit(f'package.json: fuzzyworld.deploy.{name} needs a plain name, "paths" (folders inside the repo, ' + 'written plainly: no "./", trailing "/" or "..") and "command" (a list)') + found[name] = (paths, cmd) + if t.get("perMachine"): + PER_MACHINE.add(name) + repo = (pkg.get("fuzzyworld") or {}).get("repo") or pkg.get("name") or os.path.basename(ROOT) + if not NAME_RE.match(str(repo)): + raise SystemExit('package.json: set "fuzzyworld": {"repo": "<name of the repo on the server>"} (the package name is not usable)') + return repo, found + + +REPO, TARGETS = declared_targets() def git(*args, env=None): @@ -78,8 +108,8 @@ def save(ledger): def key(name): - """Ledger key: the laptop runner is installed per machine, the rest once for the server.""" - return f"{name}@{socket.gethostname()}" if name == "laptop-runner" else name + """Ledger key "<repo>/<target>"; a per-machine target (the laptop runner) also carries this Mac's hostname.""" + return f"{REPO}/{name}" + (f"@{socket.gethostname()}" if name in PER_MACHINE else "") def main(argv): diff --git a/lib/admin.py b/lib/admin.py index 2c07008..639c59e 100644 --- a/lib/admin.py +++ b/lib/admin.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""fw admin: a LOCAL web page for the server's reports (bug-report issues) and blog comments. Python standard library only. +"""fw admin: a LOCAL web page for the server's reports (bug-report issues), blog comments and deploy state. Python standard library only. fw admin [--port N] [--no-open] [--code-dir DIR] @@ -192,6 +192,33 @@ def yes(flag): return "yes" if flag else "no" +STATE_BADGES = {"current": ("up to date", " badge-success"), "pending": ("pending", " badge-warning"), + "never": ("never deployed", ""), "unknown": ("unknown", "")} + + +def state_badge(state): + label, cls = STATE_BADGES.get(state, (str(state), "")) + return f'<span class="badge{cls}">{esc(label)}</span>' + + +def commit_cell(commit): + """The deployed commit as shown: "abc1234", "abc1234+uncommitted", or a dash.""" + return f"<code>{esc(commit)}</code>" if commit else "–" + + +def behind_cell(behind, state): + """The commits since the deployed one: none, a short list, or why the answer is unknown.""" + if state == "never": + return "" + if behind is None: + return '<span class="meta">not on the server (uncommitted or unpushed deploy)</span>' + if not behind["count"]: + return '<span class="meta">none</span>' + items = "<br>".join(f"<code>{esc(sha)}</code> {esc(subject)}" for sha, subject in behind["commits"]) + more = behind["count"] - len(behind["commits"]) + return items + (f'<br><span class="meta">and {int(more)} more</span>' if more > 0 else "") + + # ---- the server ---- class AdminServer(ThreadingHTTPServer): @@ -295,7 +322,7 @@ class Handler(BaseHTTPRequestHandler): def page(self, title, body, nav=True, current=""): mark = " aria-current=page" links = "".join(f'<a href="/{p}"{mark if current == p else ""}>{t}</a>' - for p, t in (("reports", "Reports"), ("comments", "Comments"))) + for p, t in (("reports", "Reports"), ("comments", "Comments"), ("deploy", "Deploy"))) flash = "" if nav and self.server.flash: flash = "".join(f'<div class="flash{"" if ok else " err"}">{h}</div>' for ok, h in self.server.flash) @@ -339,6 +366,8 @@ class Handler(BaseHTTPRequestHandler): return self.reports(parse_qs(url.query)) if url.path == "/comments": return self.comments_page() + if url.path == "/deploy": + return self.deploy_page() if len(parts) in (4, 5) and parts[1] == "reports" and (len(parts) == 4 or parts[4] == "record.json"): if not (valid_repo(parts[2]) and valid_n(parts[3])): return self.text(400, "Bad request") @@ -462,6 +491,34 @@ class Handler(BaseHTTPRequestHandler): f'{self.button("/comments", {"id": c["id"], "action": "remove"}, "Remove", danger=True)}</div></div>') self.send(200, self.page("fw admin", "".join(out), current="comments")) + def deploy_page(self): + data = self.server.runner.json(["deploys", "--json"]) + out = ['<h1>Deploy</h1><p class="meta">What is deployed where, against what is on the server\'s default branch. ' + 'Read-only: deploy from the laptop with <code>npm run deploy</code>.</p>'] + for r in data["repos"]: + rows = [] + for t in r["targets"]: + for e in t["entries"]: + name = esc(t["target"]) + (f' <span class="meta">on {esc(e["machine"])}</span>' if e["machine"] else "") + rows.append(f'<tr><td>{name}<br><span class="meta">{esc(t["description"])}</span></td>' + f'<td>{state_badge(e["state"])}</td><td>{commit_cell(e["commit"])}</td><td>{esc(e["at"])}</td>' + f'<td>{behind_cell(e["behind"], e["state"])}</td></tr>') + out.append(f'<h2>{esc(r["repo"])} <span class="meta">(default branch at {esc(r["head"])})</span></h2>' + '<table><tr><th>Target</th><th>State</th><th>Deployed from</th><th>When</th>' + f'<th>Changes since</th></tr>{"".join(rows)}</table>') + out.append("<h2>Apps</h2>") + if data["apps"]: + apps = sorted(data["apps"], key=lambda a: (a["state"] != "pending", a["app"])) + rows = [f'<tr><td><a href="https://apps.fuzzyworld.net/{quote(a["app"])}/" rel="noreferrer noopener">{esc(a["title"])}</a></td>' + f'<td>{esc(a["repo"])}</td><td>{state_badge(a["state"])}</td><td>{commit_cell(a["commit"])}</td>' + f'<td>{esc(a["deployed"][:16].replace("T", " "))}</td><td>{behind_cell(a["behind"], a["state"])}</td></tr>' + for a in apps] + out.append('<table><tr><th>App</th><th>Repo</th><th>State</th><th>Deployed from</th><th>When (UTC)</th>' + f'<th>Commits since</th></tr>{"".join(rows)}</table>') + else: + out.append("<p>No deployed apps.</p>") + self.send(200, self.page("fw admin", "".join(out), current="deploy")) + @staticmethod def post_link(c): label = f'{c["site"]}:{c["path"]}' diff --git a/package.json b/package.json index 6db56ec..a48cfb9 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,31 @@ "list-repos": "bin/fw list", "prepare-usage-for-pr": "python3 lib/usage.py --trailers" }, + "fuzzyworld": { + "deploy": { + "server": { + "paths": ["server"], + "command": ["server/install.sh", "--publish"], + "description": "DreamHost: tools, site files and hooks, then every repo's pages are regenerated" + }, + "runner": { + "paths": ["runner"], + "command": ["runner/install.sh"], + "description": "CI runner on TrueNAS" + }, + "laptop-runner": { + "paths": ["runner"], + "command": ["runner/install-laptop.sh"], + "perMachine": true, + "description": "CI runner on a Mac (launchd); recorded per machine" + }, + "vscode": { + "paths": ["vscode-web"], + "command": ["vscode-web/install.sh"], + "description": "VS Code in the browser at apps.fuzzyworld.net/vscode/" + } + } + }, "bin": { "fw": "bin/fw" }, @@ -20,11 +45,7 @@ }, "./package.json": "./package.json" }, - "files": [ - "bin/", - "lib/", - "vite/" - ], + "files": ["bin/", "lib/", "vite/"], "engines": { "node": ">=18" }, diff --git a/server/deploys.py b/server/deploys.py new file mode 100644 index 0000000..3ee3c60 --- /dev/null +++ b/server/deploys.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""What is deployed where. Python standard library only. + +Usage (on the server: `fw deploys`; from the laptop: `bin/fw deploys`): + + deploys [--json] plain text, or JSON for `fw admin`: + {"repos": [{repo, head, targets: [{target, description, paths, entries: [...]}]}], "apps": [...]} + +Two sources, both read-only: + * the deploy ledger ~/fw/state/deployed.json (FW_STATE), written by `bin/deploy-tools` on the laptop: per + "<repo>/<target>" (<repo> is the bare repo's name: package.json "fuzzyworld": {"repo": ...} if the package is named differently) (a per-machine target also "@<hostname>") the commit, the content hash of the folders the target + ships, and the time. A repo declares its targets in package.json, "fuzzyworld": {"deploy": {<target>: {"paths": + [...], "description": "..."}}}, read here from the repo's default branch (HEAD of the bare repo = what is shipped). + * each deployed app's apps.fuzzyworld.net/<app>/.fuzzyworld.json (written by `fw deploy`): the repo, the commit and + the time of its last deploy. + +State of a target entry: "current" (the ledger's hash equals the hash of those folders at the repo's HEAD), +"pending" (it differs), "never" (no ledger entry). `behind` lists the commits at HEAD since the deployed commit that +touch the target's folders (for an app: any file except documentation and CI config, see APP_IGNORED); it is null when the deployed commit is not in the server's +repo (for instance a deploy of uncommitted work, or of a commit that was never pushed). +""" + +import json +import os +import posixpath +import re +import subprocess +import sys + +HOME = os.path.expanduser("~") +GIT_ROOT = os.environ.get("FW_GIT_ROOT", os.path.join(HOME, "git")) +STATE = os.environ.get("FW_STATE", os.path.join(HOME, "fw", "state")) +APPS = os.environ.get("FW_APPS_SITE", os.path.join(HOME, "apps.fuzzyworld.net")) +NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*\Z") +COMMIT_RE = re.compile(r"^[0-9a-f]{7,40}\Z") +SHOWN = 8 # commits listed per "behind" +# An app's "behind" ignores commits that only touch these (they never change what is built and deployed): +# Markdown, CI config, the licence, .gitignore, docs/ and screenshots/. Git pathspecs; `*` also crosses folders. +APP_IGNORED = ("*.md", ".fw-ci.yml", ".gitlab-ci.yml", "LICENSE*", ".gitignore", "docs/", "screenshots/") + + +def git(repo, *args): + p = subprocess.run(["git", "-C", repo, *args], capture_output=True, text=True, encoding="utf-8", errors="replace") + return p.stdout.strip() if p.returncode == 0 else None + + +def read_json(path): + try: + with open(path) as f: + data = json.load(f) + except (OSError, ValueError): + return None + return data if isinstance(data, dict) else None + + +def behind(repo, commit, paths=(), ignore=()): + """{"count", "commits": [[short sha, subject]]} since `commit` at HEAD, or None if `commit` is unknown here. + `paths` limits it to those folders; `ignore` leaves out commits that touch only those pathspecs.""" + commit = (commit or "").split("+")[0] + if not COMMIT_RE.match(commit) or git(repo, "cat-file", "-e", commit + "^{commit}") is None: + return None + scope = ["--", *(paths or ["."]), *(f":(exclude){x}" for x in ignore)] if paths or ignore else [] + count = git(repo, "rev-list", "--count", f"{commit}..HEAD", *scope) + lines = git(repo, "log", f"-n{SHOWN}", "--format=%h %s", f"{commit}..HEAD", *scope) or "" + return {"count": int(count or 0), "commits": [(l.split(" ", 1) + [""])[:2] for l in lines.splitlines()]} # a commit may have no subject + + +def clean_path(p): + """A folder inside the repo, written plainly (no "./", trailing "/", "..", or absolute): the form `bin/deploy-tools` accepts.""" + return isinstance(p, str) and p != "" and posixpath.normpath(p) == p and not p.startswith(("/", "../")) and p != ".." + + +def declared(repo_dir): + """{target: {paths, description, perMachine}} from package.json at the repo's HEAD ({} if none or unreadable).""" + text = git(repo_dir, "show", "HEAD:package.json") + try: + deploy = (json.loads(text).get("fuzzyworld") or {}).get("deploy") or {} + except (ValueError, AttributeError, TypeError): + return {} + out = {} + for name, t in deploy.items() if isinstance(deploy, dict) else []: + paths = t.get("paths") if isinstance(t, dict) else None + if NAME_RE.match(name) and isinstance(paths, list) and paths and all(clean_path(p) for p in paths): + out[name] = {"paths": paths, "description": str(t.get("description", "")), "perMachine": bool(t.get("perMachine"))} + return out + + +def tree_hash(repo_dir, paths): + """The same value `bin/deploy-tools` records: the tree hash of each folder, joined by commas.""" + hashes = [git(repo_dir, "rev-parse", f"HEAD:{p}") for p in paths] + return None if None in hashes else ",".join(hashes) + + +def collect_repos(ledger): + repos = [] + for entry in sorted(os.listdir(GIT_ROOT)) if os.path.isdir(GIT_ROOT) else []: + name = entry[:-4] + repo_dir = os.path.join(GIT_ROOT, entry) + if not entry.endswith(".git") or not NAME_RE.match(name): + continue + targets = declared(repo_dir) + if not targets: + continue + head = git(repo_dir, "rev-parse", "HEAD") or "" + out = [] + for target, t in targets.items(): + now = tree_hash(repo_dir, t["paths"]) + prefix = f"{name}/{target}" + keys = sorted(k for k in ledger if k == prefix or (t["perMachine"] and k.startswith(prefix + "@"))) + entries = [] + for k in keys: + rec = ledger[k] if isinstance(ledger[k], dict) else {} + entries.append({"key": k, "machine": k.partition("@")[2] or None, "commit": str(rec.get("commit", "")), + "at": str(rec.get("at", "")), "state": "current" if rec.get("hash") == now and now else "pending", + "behind": behind(repo_dir, str(rec.get("commit", "")), t["paths"])}) + if not entries: + entries = [{"key": prefix, "machine": None, "commit": "", "at": "", "state": "never", "behind": None}] + out.append({"target": target, "description": t["description"], "paths": t["paths"], + "per_machine": t["perMachine"], "entries": entries}) + repos.append({"repo": name, "head": head[:7], "targets": out}) + return repos + + +def collect_apps(): + apps = [] + for entry in sorted(os.listdir(APPS)) if os.path.isdir(APPS) else []: + meta = read_json(os.path.join(APPS, entry, ".fuzzyworld.json")) + if not meta or not NAME_RE.match(entry): + continue + repo = str(meta.get("repo") or entry) + repo_dir = os.path.join(GIT_ROOT, repo + ".git") + have = NAME_RE.match(repo) and os.path.isdir(repo_dir) + head = (git(repo_dir, "rev-parse", "HEAD") or "") if have else "" + commit = str(meta.get("commit", "")) + b = behind(repo_dir, commit, ignore=APP_IGNORED) if have else None + state = "unknown" if b is None else "current" if b["count"] == 0 else "pending" + apps.append({"app": entry, "title": str(meta.get("title") or entry), "repo": repo, "commit": commit[:7], + "deployed": str(meta.get("deployed", "")), "head": head[:7], "state": state, "behind": b}) + return apps + + +def collect(): + ledger = read_json(os.path.join(STATE, "deployed.json")) or {} + return {"repos": collect_repos(ledger), "apps": collect_apps()} + + +def text(data): + out = [] + for r in data["repos"]: + out.append(f'{r["repo"]} (HEAD {r["head"]})') + for t in r["targets"]: + for e in t["entries"]: + n = f' ({e["behind"]["count"]} commits behind)' if e["behind"] and e["behind"]["count"] else "" + out.append(f' {t["target"]:14} {e["state"]:8} {e["commit"] or "-":22} {e["at"]}{n}') + out.append("apps") + for a in data["apps"]: + n = f' ({a["behind"]["count"]} commits behind)' if a["behind"] and a["behind"]["count"] else "" + out.append(f' {a["app"]:24} {a["state"]:8} {a["commit"] or "-":8} {a["deployed"]}{n}') + return "\n".join(out) + + +def main(argv): + if argv[1:] not in ([], ["--json"]): + raise SystemExit(__doc__) + data = collect() + print(json.dumps(data) if argv[1:] else text(data)) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/server/fw b/server/fw index 78678f7..a57e94c 100755 --- a/server/fw +++ b/server/fw @@ -22,6 +22,7 @@ # fw comments <cmd> moderate blog comments (list, show, approve, reject, ... ingest, imap-check) # fw issues <repo> public|reviewed|private who sees a repo's bug-report issue pages (default private) # fw issues <cmd> moderate issues (list, show, record, approve, hide, close, reopen, publish, publish-all, ingest, imap-check) +# fw deploys [--json] what is deployed where: the deploy ledger and each deployed app against its repo's HEAD # # Visibility is `git config fuzzyworld.visibility` inside each repo. Anything other # than exactly "public" is treated as private. @@ -266,6 +267,7 @@ case "${1:-}" in ci-dashboard) "${GEN[@]}" ci-dashboard ;; comments) shift; python3 "$TOOLS/server/comments.py" "$@" ;; issues) shift; cmd_issues "$@" ;; - ""|-h|--help|help) sed -n '2,27p' "$0" | sed 's/^# \{0,1\}//' ;; + deploys) shift; python3 "$TOOLS/server/deploys.py" "$@" ;; + ""|-h|--help|help) sed -n '2,28p' "$0" | sed 's/^# \{0,1\}//' ;; *) die "unknown command: $1 (try: fw help)" ;; esac diff --git a/tests/test_admin.py b/tests/test_admin.py index 44fab5f..adef4a7 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -49,6 +49,24 @@ def comment_list(**over): return {"pending": [c], "approved": [{**c, "id": "20241230T101010Z-123456", "source": None, "text": "Old one"}]} +def deploys_reply(**over): + d = {"repos": [{"repo": "tools", "head": "abc1234", "targets": [ + {"target": "server", "description": "the <b>server</b>", "paths": ["server"], "per_machine": False, + "entries": [{"key": "tools/server", "machine": None, "commit": "1111111", "at": "2026-11-04 14:22", "state": "pending", + "behind": {"count": 10, "commits": [["2222222", "Fix <i>it</i>"]] * 8}}]}, + {"target": "laptop-runner", "description": "", "paths": ["runner"], "per_machine": True, + "entries": [{"key": "tools/laptop-runner@mac-a", "machine": "mac-a", "commit": "3333333+uncommitted", "at": "t", + "state": "pending", "behind": None}]}, + {"target": "vscode", "description": "", "paths": ["vscode-web"], "per_machine": False, + "entries": [{"key": "tools/vscode", "machine": None, "commit": "", "at": "", "state": "never", "behind": None}]}]}], + "apps": [{"app": "zed", "title": "Zed", "repo": "zed", "commit": "4444444", "deployed": "2026-11-03T10:00:00Z", "head": "4444444", + "state": "current", "behind": {"count": 0, "commits": []}}, + {"app": "alpha", "title": EVIL, "repo": "alpha", "commit": "5555555", "deployed": "2026-11-03T10:00:00Z", "head": "6666666", + "state": "pending", "behind": {"count": 1, "commits": [["6666666", "More"]]}}]} + d.update(over) + return d + + class FakeRunner: host = "fake" @@ -77,6 +95,7 @@ class Base(unittest.TestCase): self.fake.replies[("issues", "list", "--json")] = issue_list() self.fake.replies[("issues", "show", "demo", "3", "--json")] = issue_show() self.fake.replies[("comments", "list", "--json")] = comment_list() + self.fake.replies[("deploys", "--json")] = deploys_reply() self.server = admin.make_server(self.fake, 0, code_dir=self.tmp) self.port = self.server.port t = threading.Thread(target=self.server.serve_forever, daemon=True) @@ -116,6 +135,7 @@ class Base(unittest.TestCase): class Access(Base): def test_no_cookie_is_forbidden(self): self.assertEqual(self.get("/reports", login=False)[0], 403) + self.assertEqual(self.get("/deploy", login=False)[0], 403) self.assertEqual(self.get("/style.css", login=False)[0], 403) self.assertEqual(self.fake.calls, []) @@ -183,7 +203,7 @@ class Access(Base): def test_unknown_route_is_404_and_no_javascript(self): self.assertEqual(self.get("/nothing")[0], 404) - body = self.get("/reports")[2] + self.get("/comments")[2] + self.get("/reports/demo/3")[2] + body = self.get("/reports")[2] + self.get("/comments")[2] + self.get("/reports/demo/3")[2] + self.get("/deploy")[2] self.assertNotIn("<script", body) self.assertNotIn("onclick", body) self.assertIn("<title>fw admin</title>", body) @@ -321,12 +341,31 @@ class Pages(Base): self.fake.replies[("issues", "show", "demo", "3", "--json")] = issue_show( title=EVIL, details=[[EVIL, EVIL]], messages=[{**issue_show()["messages"][0], "text": EVIL, "label": EVIL}]) self.fake.replies[("comments", "list", "--json")] = comment_list(text=EVIL, author=EVIL, site=EVIL, post_url="javascript:" + EVIL) - for path in ("/reports", "/reports/demo/3", "/comments"): + for path in ("/reports", "/reports/demo/3", "/comments", "/deploy"): body = self.get(path)[2] self.assertNotIn("<script>", body, path) self.assertIn("<script>alert(1)</script>", self.get("/reports/demo/3")[2]) self.assertNotIn('href="javascript', self.get("/comments")[2]) + def test_deploy_page_is_read_only_and_escapes(self): + status, _, body = self.get("/deploy") + self.assertEqual(status, 200) + self.assertIn('aria-current=page>Deploy</a>', body) + self.assertNotIn("<form", body) # nothing to click yet + self.assertNotIn(EVIL, body) + self.assertIn("the <b>server</b>", body) + self.assertIn("Fix <i>it</i>", body) + self.assertIn("and 2 more", body) # 10 commits behind, 8 listed + self.assertIn("on mac-a", body) + self.assertIn("not on the server (uncommitted or unpushed deploy)", body) + self.assertIn("never deployed", body) + self.assertLess(body.index("<script>"), body.index("Zed")) # a pending app is listed before a current one + self.assertEqual(self.fake.calls[-1], ["deploys", "--json"]) + + def test_deploy_page_with_nothing_deployed(self): + self.fake.replies[("deploys", "--json")] = {"repos": [], "apps": []} + self.assertIn("No deployed apps.", self.get("/deploy")[2]) + def test_comments_page(self): body = self.get("/comments")[2] for text in ("Nice post.", "Approve + trust", "Reject", "Untrust", "Recently approved", "Old one", "Remove", diff --git a/tests/test_deploy_tools.py b/tests/test_deploy_tools.py index d3a4d64..e5ff067 100644 --- a/tests/test_deploy_tools.py +++ b/tests/test_deploy_tools.py @@ -78,7 +78,7 @@ class DeployTools(unittest.TestCase): self.assertEqual(len(calls), 2) dt.main(["x", "server", "--force"]) self.assertEqual(len(calls), 3) - rec = self.ledger["server"] + rec = self.ledger[f"{dt.REPO}/server"] self.assertTrue(rec["commit"].endswith("+uncommitted")) def test_failed_deploy_leaves_ledger_alone(self): @@ -114,14 +114,16 @@ class DeployTools(unittest.TestCase): real_run = subprocess.run fake = lambda cmd, **kw: real_run(cmd, **kw) if cmd[0] == "git" else mock.Mock(returncode=0) targets = {"laptop-runner": (["server"], ["server/x.sh"])} - with mock.patch.object(dt, "TARGETS", targets), mock.patch.object(dt.subprocess, "run", side_effect=fake), \ + with mock.patch.object(dt, "TARGETS", targets), mock.patch.object(dt, "PER_MACHINE", {"laptop-runner"}), \ + mock.patch.object(dt, "REPO", "demo"), mock.patch.object(dt.subprocess, "run", side_effect=fake), \ mock.patch.object(dt.socket, "gethostname", return_value="mac-a"): dt.main(["x", "laptop-runner"]) - self.assertEqual(list(self.ledger), ["laptop-runner@mac-a"]) - self.assertFalse(self.ledger["laptop-runner@mac-a"]["commit"].endswith("+uncommitted")) + self.assertEqual(list(self.ledger), ["demo/laptop-runner@mac-a"]) + self.assertFalse(self.ledger["demo/laptop-runner@mac-a"]["commit"].endswith("+uncommitted")) calls = [] fake2 = lambda cmd, **kw: real_run(cmd, **kw) if cmd[0] == "git" else calls.append((cmd, kw)) or mock.Mock(returncode=0) - with mock.patch.object(dt, "TARGETS", targets), mock.patch.object(dt.subprocess, "run", side_effect=fake2), \ + with mock.patch.object(dt, "TARGETS", targets), mock.patch.object(dt, "PER_MACHINE", {"laptop-runner"}), \ + mock.patch.object(dt, "REPO", "demo"), mock.patch.object(dt.subprocess, "run", side_effect=fake2), \ mock.patch.object(dt.socket, "gethostname", return_value="mac-b"): dt.main(["x", "laptop-runner"]) # another Mac: not deployed there yet self.assertEqual(len(calls), 1) @@ -134,13 +136,36 @@ class DeployTools(unittest.TestCase): dt.main(["x", "typo"]) def test_status_lists_targets(self): - self.ledger["server"] = {"hash": "stale", "commit": "abc1234", "at": "2026-11-04 10:00"} + self.ledger[f"{dt.REPO}/server"] = {"hash": "stale", "commit": "abc1234", "at": "2026-11-04 10:00"} with mock.patch("builtins.print") as pr: self.assertEqual(dt.main(["x", "status"]), 0) text = "\n".join(c.args[0] for c in pr.call_args_list) self.assertIn("PENDING", text) self.assertIn("never deployed", text) + def test_targets_are_read_from_package_json_and_checked(self): + def declare(deploy): + self.write("package.json", json.dumps({"name": "demo", "fuzzyworld": {"deploy": deploy}})) + declare({"a": {"paths": ["server"], "command": ["server/x.sh"]}, + "b": {"paths": ["server"], "command": ["x"], "perMachine": True}}) + with mock.patch.object(dt, "PER_MACHINE", set()) as per: + repo, targets = dt.declared_targets() + self.assertEqual((repo, targets["a"], per), ("demo", (["server"], ["server/x.sh"]), {"b"})) + for bad in ({"a": {"paths": ["./server"], "command": ["x"]}}, {"a": {"paths": ["server/"], "command": ["x"]}}, + {"a@x": {"paths": ["server"], "command": ["x"]}}, {"a/b": {"paths": ["server"], "command": ["x"]}}, + {"a": {"paths": ["../etc"], "command": ["x"]}}, {"a": {"paths": ["/abs"], "command": ["x"]}}, + {"a": {"paths": [], "command": ["x"]}}, {"a": {"paths": ["server"], "command": "x"}}): + declare(bad) + with self.assertRaises(SystemExit): + dt.declared_targets() + + def test_repo_name_can_be_overridden_and_must_be_plain(self): + self.write("package.json", json.dumps({"name": "@scope/pkg", "fuzzyworld": {"repo": "server-name", "deploy": {}}})) + self.assertEqual(dt.declared_targets()[0], "server-name") + self.write("package.json", json.dumps({"name": "@scope/pkg", "fuzzyworld": {"deploy": {}}})) + with self.assertRaises(SystemExit): + dt.declared_targets() + if __name__ == "__main__": unittest.main() diff --git a/tests/test_deploys.py b/tests/test_deploys.py new file mode 100644 index 0000000..8cd816e --- /dev/null +++ b/tests/test_deploys.py @@ -0,0 +1,164 @@ +"""Tests for server/deploys.py (`fw deploys`: the deploy ledger and deployed apps against their repos).""" +import importlib.machinery +import importlib.util +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from unittest import mock + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +spec = importlib.util.spec_from_file_location("deploys", os.path.join(ROOT, "server", "deploys.py")) +deploys = importlib.util.module_from_spec(spec) +spec.loader.exec_module(deploys) + +ENV = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@x", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@x"} +PKG = {"name": "tools", "fuzzyworld": {"deploy": { + "server": {"paths": ["server"], "command": ["server/install.sh"], "description": "the server"}, + "laptop-runner": {"paths": ["runner"], "command": ["x"], "perMachine": True}, + "../evil": {"paths": ["server"], "command": ["x"]}, + "bad": {"paths": [], "command": ["x"]}}}} + + +class Deploys(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + self.git_root, self.state, self.apps = (os.path.join(self.tmp, d) for d in ("git", "state", "apps")) + for d in (self.git_root, self.state, self.apps): + os.makedirs(d) + self.work = os.path.join(self.tmp, "work") + subprocess.run(["git", "init", "-q", self.work], check=True) + os.makedirs(os.path.join(self.work, "server")) + os.makedirs(os.path.join(self.work, "runner")) + self.write("package.json", json.dumps(PKG)) + self.write("server/a.txt", "1") + self.write("runner/r.txt", "1") + self.first = self.commit("first") + for name, value in [("GIT_ROOT", self.git_root), ("STATE", self.state), ("APPS", self.apps)]: + p = mock.patch.object(deploys, name, value) + p.start() + self.addCleanup(p.stop) + + def write(self, rel, text): + with open(os.path.join(self.work, rel), "w") as f: + f.write(text) + + def commit(self, msg): + subprocess.run(["git", "-C", self.work, "add", "-A"], check=True) + subprocess.run(["git", "-C", self.work, "commit", "-qm", msg], check=True, env=ENV) + sha = subprocess.run(["git", "-C", self.work, "rev-parse", "HEAD"], check=True, capture_output=True, text=True).stdout.strip() + self.sync("tools") + return sha + + def sync(self, name): + bare = os.path.join(self.git_root, name + ".git") + if os.path.isdir(bare): + shutil.rmtree(bare) + subprocess.run(["git", "clone", "-q", "--bare", self.work, bare], check=True) + + def tree(self, path): + return subprocess.run(["git", "-C", self.work, "rev-parse", f"HEAD:{path}"], check=True, capture_output=True, text=True).stdout.strip() + + def ledger(self, data): + with open(os.path.join(self.state, "deployed.json"), "w") as f: + json.dump(data, f) + + def target(self, data, name): + (repo,) = data["repos"] + return next(t for t in repo["targets"] if t["target"] == name) + + def test_never_current_pending_and_behind(self): + data = deploys.collect() + self.assertEqual([t["target"] for t in data["repos"][0]["targets"]], ["server", "laptop-runner"]) # bad names/paths dropped + self.assertEqual(self.target(data, "server")["entries"][0]["state"], "never") + self.ledger({"tools/server": {"hash": self.tree("server"), "commit": self.first[:7], "at": "2026-11-04 10:00"}}) + self.assertEqual(self.target(deploys.collect(), "server")["entries"][0]["state"], "current") + self.write("README.md", "docs") # unrelated change: still current, not behind for this target + self.commit("docs only") + e = self.target(deploys.collect(), "server")["entries"][0] + self.assertEqual((e["state"], e["behind"]["count"]), ("current", 0)) + self.write("server/a.txt", "2") + self.commit("change server") + e = self.target(deploys.collect(), "server")["entries"][0] + self.assertEqual((e["state"], e["behind"]["count"], e["behind"]["commits"][0][1]), ("pending", 1, "change server")) + + def test_uncommitted_deploy_or_unpushed_commit_is_pending_with_unknown_behind(self): + self.ledger({"tools/server": {"hash": "deadbeef", "commit": "abc1234+uncommitted", "at": "t"}}) + e = self.target(deploys.collect(), "server")["entries"][0] + self.assertEqual((e["state"], e["behind"]), ("pending", None)) + + def test_per_machine_target_lists_each_machine(self): + h = self.tree("runner") + self.ledger({"tools/laptop-runner@mac-a": {"hash": h, "commit": "x", "at": "1"}, + "tools/laptop-runner@mac-b": {"hash": "old", "commit": "y", "at": "2"}}) + entries = self.target(deploys.collect(), "laptop-runner")["entries"] + self.assertEqual([(e["machine"], e["state"]) for e in entries], [("mac-a", "current"), ("mac-b", "pending")]) + + def test_apps_against_their_repo(self): + for app, commit in [("one", self.first), ("two", "f" * 40), ("three", self.first)]: + os.makedirs(os.path.join(self.apps, app)) + with open(os.path.join(self.apps, app, ".fuzzyworld.json"), "w") as f: + json.dump({"app": app, "title": app.title(), "repo": "tools" if app != "three" else "missing", + "commit": commit, "deployed": "2026-11-04T10:00:00Z"}, f) + self.write("server/a.txt", "3") + self.commit("later") + by = {a["app"]: a for a in deploys.collect()["apps"]} + self.assertEqual((by["one"]["state"], by["one"]["behind"]["count"]), ("pending", 1)) + self.assertEqual((by["two"]["state"], by["two"]["behind"]), ("unknown", None)) # commit not in the repo + self.assertEqual(by["three"]["state"], "unknown") # repo not on the server + + def test_app_ignores_commits_that_only_touch_docs_and_ci_config(self): + os.makedirs(os.path.join(self.apps, "one")) + with open(os.path.join(self.apps, "one", ".fuzzyworld.json"), "w") as f: + json.dump({"app": "one", "repo": "tools", "commit": self.first, "deployed": "2026-11-04T10:00:00Z"}, f) + for rel, text in [("README.md", "x"), (".fw-ci.yml", "x"), ("docs/a.txt", "x"), ("sub/NOTES.md", "x")]: + os.makedirs(os.path.dirname(os.path.join(self.work, rel)) or self.work, exist_ok=True) + self.write(rel, text) + self.commit("docs " + rel) + app = deploys.collect()["apps"][0] + self.assertEqual((app["state"], app["behind"]["count"]), ("current", 0)) + self.write("server/a.txt", "code") + self.write("README.md", "also docs") + self.commit("real change") # touches code and a doc: counts + app = deploys.collect()["apps"][0] + self.assertEqual((app["state"], app["behind"]["count"], app["behind"]["commits"][0][1]), ("pending", 1, "real change")) + + def test_hash_equals_what_deploy_tools_records(self): + loader = importlib.machinery.SourceFileLoader("deploy_tools", os.path.join(ROOT, "bin", "deploy-tools")) + dt = importlib.util.module_from_spec(importlib.util.spec_from_loader("deploy_tools", loader)) + loader.exec_module(dt) + with mock.patch.object(dt, "ROOT", self.work): + recorded = dt.content_hash(["server", "runner"]) + self.assertEqual(deploys.tree_hash(os.path.join(self.git_root, "tools.git"), ["server", "runner"]), recorded) + + def test_commit_without_a_subject_and_odd_ledger_and_app_values(self): + self.write("server/a.txt", "2") + subprocess.run(["git", "-C", self.work, "add", "-A"], check=True) + subprocess.run(["git", "-C", self.work, "commit", "-q", "--allow-empty-message", "-m", ""], check=True, env=ENV) + self.sync("tools") + self.ledger({"tools/server": {"hash": "old", "commit": self.first[:7], "at": "t"}, "tools/laptop-runner@m": "not a dict"}) + data = deploys.collect() + self.assertEqual(self.target(data, "server")["entries"][0]["behind"]["commits"][0][1], "") + self.assertEqual(self.target(data, "laptop-runner")["entries"][0]["state"], "pending") + os.makedirs(os.path.join(self.apps, "odd")) + with open(os.path.join(self.apps, "odd", ".fuzzyworld.json"), "w") as f: + json.dump({"repo": "../x", "commit": "zzz"}, f) + self.assertEqual({a["app"]: a["state"] for a in deploys.collect()["apps"]}, {"odd": "unknown"}) + + def test_paths_must_be_written_plainly(self): + for ok in ("server", "a/b"): + self.assertTrue(deploys.clean_path(ok)) + for bad in ("", "./server", "server/", "/abs", "..", "../x", "a//b", None, 3): + self.assertFalse(deploys.clean_path(bad), bad) + + def test_text_and_nothing_to_report(self): + self.assertIn("never", deploys.text(deploys.collect())) + shutil.rmtree(self.git_root) + self.assertEqual(deploys.collect(), {"repos": [], "apps": []}) + + +if __name__ == "__main__": + unittest.main()