fuzzyworld-tools / files / server / gen.py

gen.py

Blame: the commit that last changed each line · file · history

`623239b7` 2026-10-26 Pavel Safronov	1	#!/usr/bin/env python3
	2	"""Static browse pages for code.fuzzyworld.net.
	3
	4	gen.py repo <name> build or update the pages of one public repo
	5	gen.py index rebuild the site index (public repos only)
`f9ca0752` 2026-10-27 Pavel Safronov	6	gen.py apps rebuild the apps.fuzzyworld.net index from deployed apps
`d62e3357` 2026-11-01 Pavel Safronov	7	gen.py landing <src> <dest> write the landing page: src with <!--SITE-NAV--> filled in from NAV, <!--SITE-FOOTER--> from site_footer() and <!--LANDING-TEXT--> from landing.md
`ec9751fa` 2026-10-27 Pavel Safronov	8	gen.py content render server/content/<name>.md to code.fuzzyworld.net/<name>/ (site pages written in Markdown)
`a5a72f55` 2026-10-27 Pavel Safronov	9	gen.py ci <name> rebuild one public repo's CI pages, then the CI dashboard
	10	gen.py ci-dashboard rebuild code.fuzzyworld.net/ci/ (queue, running, recent; public repos only)
	11	gen.py ci-status the same, as text for the terminal, private repos included (fw ci-watch)
`623239b7` 2026-10-26 Pavel Safronov	12
`1758548c` 2026-10-27 Pavel Safronov	13	Only the git CLI and the standard library are required. From ~/fw/pylib, when
`73733e6e` 2026-11-01 Pavel Safronov	14	importable: Python-Markdown renders README/.md files (otherwise plain text; `landing`
	15	and `content` need it) and Pygments colors source code (otherwise uncolored).
`623239b7` 2026-10-26 Pavel Safronov	16
	17	Pages are incremental: a manifest per repo (~/fw/state/<name>.json) records the
	18	blob SHA behind every file page, so a push rewrites only what changed.
	19	"""
	20
`1758548c` 2026-10-27 Pavel Safronov	21	import datetime
`ced0eafa` 2026-11-04 Pavel Safronov	22	import hashlib
`623239b7` 2026-10-26 Pavel Safronov	23	import html
	24	import json
	25	import os
	26	import posixpath
	27	import re
`1758548c` 2026-10-27 Pavel Safronov	28	import shutil
`623239b7` 2026-10-26 Pavel Safronov	29	import subprocess
	30	import sys
	31	import time
	32	import urllib.parse
	33
	34	HOME = os.path.expanduser("~")
	35	GIT_ROOT = os.environ.get("FW_GIT_ROOT", os.path.join(HOME, "git"))
	36	SITE = os.environ.get("FW_CODE_SITE", os.path.join(HOME, "code.fuzzyworld.net"))
	37	APPS = os.environ.get("FW_APPS_SITE", os.path.join(HOME, "apps.fuzzyworld.net"))
	38	STATE = os.environ.get("FW_STATE", os.path.join(HOME, "fw", "state"))
	39	PYLIB = os.environ.get("FW_PYLIB", os.path.join(HOME, "fw", "pylib"))
`1758548c` 2026-10-27 Pavel Safronov	40	CI_ROOT = os.environ.get("FW_CI", os.path.join(HOME, "fw", "ci"))
`a5a72f55` 2026-10-27 Pavel Safronov	41	QUEUE = os.environ.get("FW_QUEUE", os.path.join(HOME, "fw", "queue"))
`a4997156` 2026-10-31 Pavel Safronov	42	RUNNERS = os.environ.get("FW_RUNNERS", os.path.join(HOME, "fw", "runners")) # one file per runner; its time is its last check-in
`623239b7` 2026-10-26 Pavel Safronov	43	BASE_URL = "https://code.fuzzyworld.net"
	44	APPS_URL = "https://apps.fuzzyworld.net"
	45
	46	# Bump when page templates change, so every repo gets a full rebuild once.
`ced0eafa` 2026-11-04 Pavel Safronov	47	GEN_VERSION = 10
	48
	49
	50	def gen_key():
	51	"""The version recorded in a repo's manifest: GEN_VERSION plus a hash of prices.json, so a rate-card
	52	refresh rebuilds the commit pages (they show estimated costs)."""
	53	try:
	54	with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "prices.json"), "rb") as f:
	55	return f"{GEN_VERSION}-{hashlib.sha1(f.read()).hexdigest()[:8]}"
	56	except OSError:
	57	return f"{GEN_VERSION}-noprices"
`1758548c` 2026-10-27 Pavel Safronov	58
	59	COMMIT_LIMIT = 2000 # commit pages per repo, across all branches; older commits: clone
	60	HISTORY_LIMIT = 200 # commits listed on a file's history page
	61	CI_LIMIT = 100 # pipelines listed on a repo's CI page
`a5a72f55` 2026-10-27 Pavel Safronov	62	CI_RECENT = 50 # finished pipelines on the CI dashboard
`a4997156` 2026-10-31 Pavel Safronov	63	RUNNER_GONE = 15 * 60 # seconds without a check-in before a runner doesn't count (runners ask every 2 minutes)
`a5a72f55` 2026-10-27 Pavel Safronov	64	CI_REFRESH = 15 # seconds: pages showing something running reload themselves
`1758548c` 2026-10-27 Pavel Safronov	65	SHOTS = "_shots" # apps.fuzzyworld.net/_shots/<app>/: screenshots from CI runners (no app name starts with "_")
	66	SHOT_EXTENSIONS = (".png", ".jpg", ".jpeg", ".webp")
	67	VSCODE_APP = "vscode" # apps.fuzzyworld.net/<this>/: the static VS Code web build (vscode-web/)
`623239b7` 2026-10-26 Pavel Safronov	68	MAX_TEXT_BYTES = 1024 * 1024
	69	MAX_DIFF_BYTES = 300 * 1024
	70	MAX_RAW_BYTES = 2 * 1024 * 1024
`1758548c` 2026-10-27 Pavel Safronov	71	MAX_HIGHLIGHT_BYTES = 512 * 1024
	72	MAX_BLAME_BYTES = 256 * 1024
	73	MAX_LOG_BYTES = 256 * 1024 # the end of each CI job log is shown
`623239b7` 2026-10-26 Pavel Safronov	74	# Raw copies are written only for these: other extensions (.php, .shtml, ...) could be
	75	# executed by the web server, and the browse pages already show text files.
	76	# SVG is safe here because the site's CSP header (script-src 'none') covers SVG documents too.
	77	RAW_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".avif", ".bmp", ".svg"}
	78	README_NAMES = ["readme.md", "readme.markdown", "readme", "readme.txt", "readme.rst"]
	79
	80	sys.path.insert(0, PYLIB)
	81	try:
	82	import markdown # type: ignore
	83	except ImportError: # pragma: no cover
	84	markdown = None
`1758548c` 2026-10-27 Pavel Safronov	85	try:
	86	import pygments.lexers as pygments_lexers # type: ignore
	87	import pygments.token as pygments_token # type: ignore
	88	except ImportError: # pragma: no cover
	89	pygments_lexers = pygments_token = None
`623239b7` 2026-10-26 Pavel Safronov	90
	91
	92	# ---------------------------------------------------------------- git helpers
	93
	94	def git(repo, *args, check=True):
`1758548c` 2026-10-27 Pavel Safronov	95	# DreamHost sometimes refuses to start a process for a moment (EAGAIN) when many run in a row,
	96	# e.g. one `git blame` per file on a first build; wait and try again.
	97	for attempt in range(5):
	98	try:
	99	r = subprocess.run(["git", "-C", repo, *args], capture_output=True, check=False)
	100	break
	101	except BlockingIOError:
	102	if attempt == 4:
	103	raise
	104	time.sleep(1 + attempt)
`623239b7` 2026-10-26 Pavel Safronov	105	if check and r.returncode != 0:
	106	raise RuntimeError(f"git {' '.join(args)}: {r.stderr.decode(errors='replace').strip()}")
	107	return r.stdout
	108
	109
	110	def git_text(repo, *args, check=True):
	111	return git(repo, *args, check=check).decode("utf-8", errors="replace")
	112
	113
	114	class BlobReader:
	115	"""Streams blob contents through one `git cat-file --batch` process."""
	116
	117	def __init__(self, repo):
	118	self.p = subprocess.Popen(["git", "-C", repo, "cat-file", "--batch"],
	119	stdin=subprocess.PIPE, stdout=subprocess.PIPE)
	120
	121	def read(self, sha):
	122	self.p.stdin.write(sha.encode() + b"\n")
	123	self.p.stdin.flush()
	124	header = self.p.stdout.readline().split()
	125	if len(header) < 3 or header[1] == b"missing":
	126	return b""
	127	data = self.p.stdout.read(int(header[2]))
	128	self.p.stdout.read(1) # trailing newline
	129	return data
	130
	131	def close(self):
	132	self.p.stdin.close()
	133	self.p.wait()
	134
	135
	136	def repo_config(repo, key, default=""):
	137	v = git_text(repo, "config", "--get", key, check=False).strip()
	138	return v or default
	139
	140
	141	def is_public(repo):
	142	# Anything but the exact value "public" counts as private.
	143	return repo_config(repo, "fuzzyworld.visibility") == "public"
	144
	145
	146	# ---------------------------------------------------------------- html helpers
	147
	148	esc = html.escape
	149
	150
	151	def url(*parts):
	152	"""Site-absolute URL from path parts, each part quoted per segment."""
	153	path = "/".join(p.strip("/") for p in parts if p)
	154	return "/" + "/".join(urllib.parse.quote(s) for s in path.split("/"))
	155
	156
`0a08554d` 2026-10-27 Pavel Safronov	157	HOME_URL = "https://fuzzyworld.net"
	158	# Site-wide navigation, in the header of every page on both sites: (section, label, site, path).
`7e1f4430` 2026-11-01 Pavel Safronov	159	NAV = [("code", "Code", "code", "/"), ("apps", "Apps", "apps", "/"), ("writing", "Writing", "writing", "/"),
	160	("projects", "Projects", "code", "/projects/"),
`5a659fb7` 2026-10-29 Pavel Safronov	161	("my-ai-use", "Agentic", "code", "/my-ai-use/"), ("ci", "CI", "code", "/ci/"),
`799905ae` 2026-11-04 Pavel Safronov	162	("usage", "Usage", "code", "/usage/"),
`829da61a` 2026-10-28 Pavel Safronov	163	("contribute", "Contribute", "code", "/contribute/")]
`7e1f4430` 2026-11-01 Pavel Safronov	164	WRITING_URL = "https://writing.fuzzyworld.net" # a static site built by `fw blog` (blog.py), not by this file
	165	SITE_URLS = {"code": BASE_URL, "apps": APPS_URL, "writing": WRITING_URL}
`0a08554d` 2026-10-27 Pavel Safronov	166
	167
`0330e724` 2026-11-01 Pavel Safronov	168	def site_nav():
	169	"""NAV as [label, absolute URL] pairs: the one source for the nav on the landing page and the blog's `site` bar."""
	170	return [[label, SITE_URLS[st] + path] for _, label, st, path in NAV]
	171
	172
`d62e3357` 2026-11-01 Pavel Safronov	173	def site_footer(here):
	174	"""The columns of the site-wide footer (inside <div class="cols">), with links relative to the site `here`
	175	("code", "apps", "writing", or "landing" for fuzzyworld.net). One source for the code and apps pages, the
	176	landing page and the blog."""
	177	def link(st, path):
	178	return SITE_URLS[st] + path if st != here else path
	179	return (f'<div><h2>fuzzyworld</h2><a href="{HOME_URL}/">fuzzyworld.net</a>'
	180	f'<a href="{link("code", "/")}">Code: all repositories</a>'
	181	f'<a href="{link("apps", "/")}">Apps</a>'
	182	f'<a href="{WRITING_URL}/">Writing</a>'
	183	f'<a href="{link("code", "/ci/")}">CI: what is building</a></div>'
	184	f'<div><h2>About this site</h2><a href="{link("code", "/why-i-left-gitlab/")}">Why I left GitLab</a>'
	185	f'<a href="{link("code", "/contribute/")}">Contribute and report bugs</a></div>'
	186	f'<div><h2>Built with</h2><a href="{link("code", "/fuzzyworld-tools/")}">fuzzyworld-tools</a>'
	187	'<span>Static files only: no accounts, no sign-in, no rate limits. MIT-licensed.</span></div>')
	188
	189
`0330e724` 2026-11-01 Pavel Safronov	190	def build_landing(src, dest):
`d62e3357` 2026-11-01 Pavel Safronov	191	"""Writes the landing page: `src` with <!--SITE-NAV--> replaced by the nav links built from NAV,
	192	<!--SITE-FOOTER--> by site_footer("landing") and, if the
`73733e6e` 2026-11-01 Pavel Safronov	193	page has a <!--LANDING-TEXT-->, that replaced by landing.md (next to `src`) rendered from Markdown."""
`0330e724` 2026-11-01 Pavel Safronov	194	links = "".join(f'<a href="{html.escape(url, quote=True)}">{html.escape(label)}</a>' for label, url in site_nav())
	195	with open(src, encoding="utf-8") as f:
	196	text = f.read()
`d62e3357` 2026-11-01 Pavel Safronov	197	for marker in ("<!--SITE-NAV-->", "<!--SITE-FOOTER-->"):
	198	if marker not in text:
	199	raise SystemExit(f"{src}: no {marker} placeholder")
`73733e6e` 2026-11-01 Pavel Safronov	200	if "<!--LANDING-TEXT-->" in text:
	201	if not markdown:
	202	raise SystemExit("gen.py landing needs Python-Markdown in " + PYLIB)
	203	md_path = os.path.join(os.path.dirname(src), "landing.md")
	204	if not os.path.exists(md_path):
	205	raise SystemExit(f"{src}: has <!--LANDING-TEXT--> but {md_path} does not exist")
	206	with open(md_path, encoding="utf-8") as f:
	207	body = markdown.markdown(f.read(), extensions=["sane_lists"], output_format="html")
	208	text = text.replace("<!--LANDING-TEXT-->", body)
`d62e3357` 2026-11-01 Pavel Safronov	209	text = text.replace("<!--SITE-FOOTER-->", site_footer("landing"))
`0330e724` 2026-11-01 Pavel Safronov	210	write(dest, text.replace("<!--SITE-NAV-->", links))
	211
	212
`0a08554d` 2026-10-27 Pavel Safronov	213	def site_link(site, path, here):
	214	"""Links within a site stay relative; links to the other site are absolute."""
	215	return path if site == here else SITE_URLS[site] + path
	216
	217
	218	def page(title, body, crumbs=None, view=None, site="code", refresh=None, section=None):
	219	"""`site` is the site being generated; `section` is the nav entry to highlight (default: the site's home)."""
	220	host = "apps.fuzzyworld.net" if site == "apps" else "code.fuzzyworld.net"
	221	section = section or site
	222	links = "".join(
	223	f'<a href="{site_link(st, path, site)}"' + (' aria-current="page"' if key == section else "") + f'>{label}</a>'
	224	for key, label, st, path in NAV)
	225	header = (f'<header class="site"><div class="bar"><a class="brand" href="{HOME_URL}/">fuzzyworld</a>'
	226	f'<nav class="site-nav" aria-label="Site">{links}</nav></div></header>')
`d62e3357` 2026-11-01 Pavel Safronov	227	footer = f'<footer class="site"><div class="cols">{site_footer(site)}</div></footer>'
`623239b7` 2026-10-26 Pavel Safronov	228	nav = ""
`1758548c` 2026-10-27 Pavel Safronov	229	if view:
	230	nav = (f'<nav class="repo-nav"><a href="{url(view.name)}/">Summary</a>'
	231	f'<a href="{view.url("tree")}/">Files</a>'
	232	f'<a href="{view.url("log")}/">Log</a>'
	233	f'<a href="{url(view.name, "refs")}/">Branches & tags</a>'
`85702b14` 2026-11-04 Pavel Safronov	234	+ (f'<a href="{url(view.name, "issues")}/">Issues</a>'
	235	if os.path.exists(os.path.join(SITE, view.name, "issues", "index.html")) else "")
`1758548c` 2026-10-27 Pavel Safronov	236	+ (f'<a href="{url(view.name, "ci")}/">CI</a>' if ci_pipelines(view.name) else "")
`57d16c8b` 2026-10-29 Pavel Safronov	237	+ (f'<a href="{url(view.name, "usage")}/">Usage</a>' if repo_usage(view.name)["costs"] else "")
`1758548c` 2026-10-27 Pavel Safronov	238	+ "</nav>")
`623239b7` 2026-10-26 Pavel Safronov	239	crumb_html = ""
	240	if crumbs:
	241	crumb_html = '<p class="crumbs">' + " / ".join(crumbs) + "</p>"
	242	return f"""<!doctype html>
	243	<html lang="en">
	244	<head>
	245	<meta charset="utf-8">
	246	<meta name="viewport" content="width=device-width, initial-scale=1">
`a5a72f55` 2026-10-27 Pavel Safronov	247	{f'<meta http-equiv="refresh" content="{refresh}">' if refresh else ""}
`f9ca0752` 2026-10-27 Pavel Safronov	248	<title>{esc(title)} · {host}</title>
`2483c47c` 2026-10-28 Pavel Safronov	249	<link rel="icon" href="/favicon.ico">
`623239b7` 2026-10-26 Pavel Safronov	250	<link rel="stylesheet" href="/style.css">
	251	</head>
	252	<body>
`0a08554d` 2026-10-27 Pavel Safronov	253	{header}
`623239b7` 2026-10-26 Pavel Safronov	254	<main>
	255	{nav}
	256	{crumb_html}
	257	{body}
	258	</main>
`0a08554d` 2026-10-27 Pavel Safronov	259	{footer}
`b447a9f2` 2026-11-03 Pavel Safronov	260	{reload_timer(refresh)}
`623239b7` 2026-10-26 Pavel Safronov	261	</body>
	262	</html>
	263	"""
	264
	265
`b447a9f2` 2026-11-03 Pavel Safronov	266	def reload_timer(refresh):
	267	"""A thin bar along the top that empties as the page's meta refresh nears, and a countdown in
	268	any `.reload-in` element (user request, 2026-11-03: the CI page said it reloads every 15
	269	seconds but not when). Pure CSS, because the site's CSP is `script-src 'none'`: the countdown is
	270	an animated integer property shown through a counter. Starts at page load, as the refresh does.
	271	Browsers without @property support show the bar only."""
	272	if not refresh:
	273	return ""
	274	n = int(refresh)
	275	return f"""<div class="reload-bar" aria-hidden="true"></div>
	276	<style>
	277	.reload-bar {{ position: fixed; top: 0; left: 0; height: 3px; width: 100%; background: currentColor; opacity: .45;
	278	transform-origin: left; animation: reload-bar {n}s linear forwards; }}
	279	@keyframes reload-bar {{ from {{ transform: scaleX(1); }} to {{ transform: scaleX(0); }} }}
	280	@property --reload-left {{ syntax: "<integer>"; inherits: false; initial-value: {n}; }}
	281	@keyframes reload-count {{ from {{ --reload-left: {n}; }} to {{ --reload-left: 0; }} }}
	282	.reload-in::before {{ content: "Next reload in " counter(reload-left) " s."; counter-reset: reload-left var(--reload-left);
	283	animation: reload-count {n}s steps({n}) forwards; }}
	284	@media (prefers-reduced-motion: reduce) {{ .reload-bar {{ display: none; }} }}
	285	</style>"""
	286
	287
`623239b7` 2026-10-26 Pavel Safronov	288	def write(path, content):
	289	os.makedirs(os.path.dirname(path), exist_ok=True)
	290	tmp = path + ".tmp"
	291	mode = "wb" if isinstance(content, bytes) else "w"
	292	with open(tmp, mode, **({} if mode == "wb" else {"encoding": "utf-8"})) as f:
	293	f.write(content)
	294	os.replace(tmp, path)
	295
	296
	297	def remove(path):
	298	try:
	299	os.remove(path)
	300	except FileNotFoundError:
	301	pass
	302
	303
	304	def human_size(n):
	305	for unit in ("B", "KB", "MB", "GB"):
	306	if n < 1024 or unit == "GB":
	307	return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
	308	n /= 1024
	309
	310
	311	def is_binary(data):
	312	return b"\0" in data[:8000]
	313
	314
	315	def safe_path(p):
	316	parts = p.split("/")
	317	return all(part not in ("", ".", "..") for part in parts)
	318
	319
`1758548c` 2026-10-27 Pavel Safronov	320	# ---------------------------------------------------------------- CI results
	321
	322	# CI runners (server/runner/) upload each pipeline to ~/fw/ci/<repo>/<sha>/: pipeline.json
	323	# plus one log file per job. These pages show them for public repos only.
	324	CI_STATES = { # status: (mark, words, badge color)
	325	"success": ("✓", "passed", "#2da44e"), "failed": ("✗", "failed", "#cf222e"),
	326	"running": ("●", "running", "#bf8700"), "error": ("!", "could not run", "#6e7781"),
	327	}
	328	_ci_cache = {}
	329
	330
	331	def ci_pipelines(name):
	332	"""{sha: pipeline} for one repo, read once per run."""
	333	if name not in _ci_cache:
	334	out, d = {}, os.path.join(CI_ROOT, name)
	335	for sha in (os.listdir(d) if os.path.isdir(d) else []):
	336	if not re.fullmatch(r"[0-9a-f]{40}", sha):
	337	continue
	338	try:
	339	with open(os.path.join(d, sha, "pipeline.json")) as f:
	340	p = json.load(f)
	341	except (OSError, ValueError):
	342	continue
	343	if isinstance(p, dict) and p.get("status") in CI_STATES:
	344	out[sha] = p
	345	_ci_cache[name] = out
	346	return _ci_cache[name]
	347
	348
	349	def ci_mark(name, sha, p):
	350	if not p:
	351	return ""
	352	mark, words, _ = CI_STATES[p["status"]]
	353	return f'<a class="ci ci-{p["status"]}" href="{url(name, "ci", sha)}.html" title="CI {words}">{mark}</a> '
	354
	355
	356	def ci_badge(name, sha):
	357	p = ci_pipelines(name).get(sha)
	358	if not p:
	359	return ""
	360	return f' · CI: {ci_mark(name, sha, p)}<a href="{url(name, "ci", sha)}.html">{CI_STATES[p["status"]][1]}</a>'
	361
	362
	363	def badge_svg(status):
	364	_, words, color = CI_STATES[status]
	365	w = 18 + 7 * len(words)
	366	return (f'<svg xmlns="http://www.w3.org/2000/svg" width="{28 + w}" height="20" role="img" aria-label="CI: {words}">'
	367	f'<rect width="28" height="20" fill="#555"/><rect x="28" width="{w}" height="20" fill="{color}"/>'
	368	f'<g fill="#fff" font-family="Verdana,DejaVu Sans,sans-serif" font-size="11">'
	369	f'<text x="6" y="14">CI</text><text x="37" y="14">{words}</text></g></svg>')
	370
	371
	372	_ANSI = re.compile(r"\x1b\[[0-9;?][A-Za-z]\|\x1b\][^\x07]\x07\|\r(?!\n)")
	373
	374
	375	def duration(seconds):
	376	s = int(seconds or 0)
	377	return f"{s // 60}m {s % 60:02d}s" if s >= 60 else f"{s}s"
	378
	379
	380	def build_ci_pages(view):
	381	"""ci/index.html, one page per pipeline, and ci/badge.svg (latest pipeline on the default branch)."""
	382	name, out = view.name, os.path.join(SITE, view.name, "ci")
	383	pipelines = ci_pipelines(name)
	384	if not pipelines:
	385	shutil.rmtree(out, ignore_errors=True)
	386	return
	387	ordered = sorted(pipelines.items(), key=lambda kv: kv[1].get("started", ""), reverse=True)[:CI_LIMIT]
	388	keep = {sha + ".html" for sha, _ in ordered} \| {"index.html", "badge.svg"}
	389	for f in (os.listdir(out) if os.path.isdir(out) else []):
	390	if f not in keep:
	391	remove(os.path.join(out, f))
	392	rows = []
	393	for sha, p in ordered:
	394	rows.append(f'<tr><td class="date">{esc(p.get("started", "")[:16].replace("T", " "))}</td>'
	395	f'<td>{ci_mark(name, sha, p)}<a href="{url(name, "ci", sha)}.html"><code>{sha[:12]}</code></a></td>'
	396	f'<td><code>{esc(p.get("ref", ""))}</code></td><td>{esc(CI_STATES[p["status"]][1])}</td>'
	397	f'<td class="num">{duration(p.get("duration"))}</td></tr>')
	398	page_path = os.path.join(out, sha + ".html")
`a5a72f55` 2026-10-27 Pavel Safronov	399	# Rebuilt when pipeline.json or a log is newer: a runner uploads a running job's log every few seconds.
	400	src_dir = os.path.join(CI_ROOT, name, sha)
	401	newest = max(os.path.getmtime(os.path.join(src_dir, f)) for f in os.listdir(src_dir))
	402	if os.path.exists(page_path) and os.path.getmtime(page_path) >= newest:
`1758548c` 2026-10-27 Pavel Safronov	403	continue
	404	jobs = []
	405	for j in p.get("jobs", []):
	406	st = j.get("status", "error")
`a5a72f55` 2026-10-27 Pavel Safronov	407	secs = j.get("duration")
	408	if st == "running" and j.get("started"):
	409	secs = time.time() - iso_time(j["started"])
`1758548c` 2026-10-27 Pavel Safronov	410	log, log_name = "", j.get("log", "")
	411	if re.fullmatch(r"[A-Za-z0-9._-]+\.log", log_name or ""):
	412	try:
	413	with open(os.path.join(CI_ROOT, name, sha, log_name), "rb") as f:
	414	f.seek(0, 2)
	415	size = f.tell()
	416	f.seek(max(0, size - MAX_LOG_BYTES))
	417	text = f.read().decode("utf-8", errors="replace")
	418	cut = "(start of log cut; showing the last part)\n" if size > MAX_LOG_BYTES else ""
	419	log = f'<pre class="ci-log">{esc(cut + _ANSI.sub("", text))}</pre>'
	420	except OSError:
	421	pass
	422	mark = CI_STATES.get(st, ("–", st, ""))
`a5a72f55` 2026-10-27 Pavel Safronov	423	jobs.append(f'<details{" open" if st in ("failed", "error", "running") else ""}><summary>'
`1758548c` 2026-10-27 Pavel Safronov	424	f'<span class="ci ci-{esc(st)}">{mark[0]}</span> <strong>{esc(j.get("name", "?"))}</strong>'
`a5a72f55` 2026-10-27 Pavel Safronov	425	f' · {esc(j.get("stage", ""))} · {esc(mark[1])} · {duration(secs)}'
`1758548c` 2026-10-27 Pavel Safronov	426	f'{" · " + esc(j["message"]) if j.get("message") else ""}</summary>{log}</details>')
	427	mark, words, _ = CI_STATES[p["status"]]
`a5a72f55` 2026-10-27 Pavel Safronov	428	running = p["status"] == "running"
`1758548c` 2026-10-27 Pavel Safronov	429	body = (f'<h1>CI {esc(words)}: <code>{sha[:12]}</code></h1>'
	430	f'<p class="meta">Branch <code>{esc(p.get("ref", ""))}</code> · '
	431	f'<a href="{url(name, "commit", sha)}.html">commit</a> · started {esc(p.get("started", "").replace("T", " ").replace("Z", " UTC"))} · '
`a5a72f55` 2026-10-27 Pavel Safronov	432	f'{"running" if running else duration(p.get("duration"))} · runner {esc(p.get("runner", ""))}</p>'
`b447a9f2` 2026-11-03 Pavel Safronov	433	+ (f'<p class="meta">This page reloads every {CI_REFRESH} seconds while the pipeline runs. <span class="reload-in"></span></p>' if running else "")
`1758548c` 2026-10-27 Pavel Safronov	434	+ (f'<p>{esc(p["message"])}</p>' if p.get("message") else "")
	435	+ "".join(jobs))
`a5a72f55` 2026-10-27 Pavel Safronov	436	write(page_path, page(f"{name} CI {sha[:12]}", body, [f'<a href="{url(name)}/">{esc(name)}</a>', "CI"], view=view,
	437	refresh=CI_REFRESH if running else None))
	438	any_running = any(p["status"] == "running" for _, p in ordered)
`1758548c` 2026-10-27 Pavel Safronov	439	write(os.path.join(out, "index.html"),
`a5a72f55` 2026-10-27 Pavel Safronov	440	page(f"{name} CI", f'<h1>CI</h1><p class="meta"><a href="/ci/">All CI</a></p><table class="log">{"".join(rows)}</table>',
	441	view=view, refresh=CI_REFRESH if any_running else None))
`1758548c` 2026-10-27 Pavel Safronov	442	latest = next((p for _, p in ordered if p.get("ref") == view.branch), None)
	443	if latest:
	444	write(os.path.join(out, "badge.svg"), badge_svg(latest["status"]))
	445	else:
	446	remove(os.path.join(out, "badge.svg"))
	447
	448
`a5a72f55` 2026-10-27 Pavel Safronov	449	def iso_time(s):
	450	try:
	451	return datetime.datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()
	452	except (ValueError, AttributeError):
	453	return 0
	454
	455
	456	def queue_jobs(kind):
	457	"""Jobs in ~/fw/queue/<kind>/ (pending or running), oldest first, with "since": the file's time."""
	458	d, out = os.path.join(QUEUE, kind), []
	459	for f in sorted(os.listdir(d)) if os.path.isdir(d) else []:
	460	try:
	461	with open(os.path.join(d, f)) as fh:
	462	j = json.load(fh)
	463	j["since"] = os.path.getmtime(os.path.join(d, f))
	464	out.append(j)
	465	except (OSError, ValueError):
	466	continue
	467	return out
	468
	469
`a4997156` 2026-10-31 Pavel Safronov	470	def runners_seen():
	471	"""[(name, tags, seconds since it last checked in)], most recent first."""
	472	out = []
	473	for name in os.listdir(RUNNERS) if os.path.isdir(RUNNERS) else []:
	474	try:
	475	with open(os.path.join(RUNNERS, name)) as f:
	476	tags = f.read().strip().strip("-")
	477	out.append((name, tags, time.time() - os.path.getmtime(os.path.join(RUNNERS, name))))
	478	except OSError:
	479	continue
	480	return sorted(out, key=lambda r: r[2])
	481
	482
	483	def runner_notice(pending):
	484	"""One sentence for the CI pages: "" if some runner is around, else why queued jobs are not moving."""
	485	if not pending:
	486	return ""
	487	seen = runners_seen()
	488	if any(age < RUNNER_GONE for _, _, age in seen):
	489	return ""
	490	if seen:
	491	return f"Waiting for a runner: the last one checked in {duration(seen[0][2])} ago. Jobs wait here until one does."
	492	return "Waiting for a runner: none has checked in yet. Jobs wait here until one does."
	493
	494
`a5a72f55` 2026-10-27 Pavel Safronov	495	def public_repo(name, _cache={}):
	496	if name not in _cache:
	497	repo = os.path.join(GIT_ROOT, name + ".git")
	498	_cache[name] = os.path.isdir(repo) and is_public(repo)
	499	return _cache[name]
	500
	501
	502	def ci_overview(include_private):
	503	"""(pending jobs, running jobs with their pipelines, recent finished pipelines) across all repos."""
	504	pending, running = queue_jobs("pending"), queue_jobs("running")
	505	for j in running:
	506	j["pipeline"] = ci_pipelines(j["repo"]).get(j["sha"], {})
	507	recent = []
	508	for name in (os.listdir(CI_ROOT) if os.path.isdir(CI_ROOT) else []):
	509	if include_private or public_repo(name):
	510	recent += [(p.get("started", ""), name, sha, p) for sha, p in ci_pipelines(name).items() if p["status"] != "running"]
	511	recent.sort(reverse=True)
	512	return pending, running, recent[:CI_RECENT]
	513
	514
	515	def job_progress(p):
	516	""" "test ✓ 44s · pages ● 12s" for a running pipeline."""
	517	parts = []
	518	for j in p.get("jobs", []):
	519	st = j.get("status", "pending")
	520	secs = time.time() - iso_time(j["started"]) if st == "running" and j.get("started") else j.get("duration")
	521	mark = CI_STATES.get(st, ("·" if st == "pending" else "–", st, ""))[0]
	522	parts.append(f'{j.get("name", "?")} {mark}' + (f" {duration(secs)}" if st not in ("pending", "skipped") else ""))
	523	return " · ".join(parts)
	524
	525
	526	def build_ci_dashboard():
	527	"""code.fuzzyworld.net/ci/: what is queued, running and recently finished. Private repos are never named."""
	528	pending, running, recent = ci_overview(include_private=False)
	529	def ref(j):
	530	return j["ref"].split("/", 2)[-1]
	531	hidden = sum(1 for j in pending + running if not public_repo(j["repo"]))
	532	run_rows = "".join(
	533	f'<tr><td><a href="{url(j["repo"])}/">{esc(j["repo"])}</a></td><td><code>{esc(ref(j))}</code></td>'
	534	f'<td><a href="{url(j["repo"], "ci", j["sha"])}.html"><code>{j["sha"][:12]}</code></a></td>'
	535	f'<td>{esc(job_progress(j["pipeline"]) or "starting")}</td>'
	536	f'<td class="num">{duration(time.time() - j["since"])}</td></tr>'
	537	for j in running if public_repo(j["repo"]))
	538	queue_rows = "".join(
	539	f'<tr><td><a href="{url(j["repo"])}/">{esc(j["repo"])}</a></td><td><code>{esc(ref(j))}</code></td>'
	540	f'<td><code>{j["sha"][:12]}</code></td><td class="num">waiting {duration(time.time() - j["since"])}</td></tr>'
	541	for j in pending if public_repo(j["repo"]))
	542	done_rows = "".join(
	543	f'<tr><td class="date">{esc(started[:16].replace("T", " "))}</td><td><a href="{url(name)}/">{esc(name)}</a></td>'
	544	f'<td><code>{esc(p.get("ref", ""))}</code></td>'
	545	f'<td>{ci_mark(name, sha, p)}<a href="{url(name, "ci", sha)}.html"><code>{sha[:12]}</code></a></td>'
	546	f'<td>{esc(CI_STATES[p["status"]][1])}</td><td class="num">{duration(p.get("duration"))}</td></tr>'
	547	for started, name, sha, p in recent)
	548	busy = bool(pending or running)
`a4997156` 2026-10-31 Pavel Safronov	549	notice = runner_notice(pending)
`a5a72f55` 2026-10-27 Pavel Safronov	550	body = ('<h1>CI</h1>'
`a4997156` 2026-10-31 Pavel Safronov	551	'<p class="meta">Pipelines run on whichever runner is available (a home server, sometimes a laptop), one at a time, as pushes queue them. '
`b447a9f2` 2026-11-03 Pavel Safronov	552	+ (f'This page reloads every {CI_REFRESH} seconds while anything is queued or running. <span class="reload-in"></span>' if busy else "Nothing is running.")
`a5a72f55` 2026-10-27 Pavel Safronov	553	+ (f' Also {hidden} job{"s" if hidden != 1 else ""} for private repositories.' if hidden else "") + '</p>'
`a4997156` 2026-10-31 Pavel Safronov	554	+ (f'<p><strong>{esc(notice)}</strong></p>' if notice else "")
`a5a72f55` 2026-10-27 Pavel Safronov	555	+ (f'<h2>Running</h2><table class="log">{run_rows}</table>' if run_rows else "")
	556	+ (f'<h2>Queued</h2><table class="log">{queue_rows}</table>' if queue_rows else "")
	557	+ (f'<h2>Recent</h2><table class="log">{done_rows}</table>' if done_rows else "<p>No pipelines yet.</p>"))
`0a08554d` 2026-10-27 Pavel Safronov	558	write(os.path.join(SITE, "ci", "index.html"), page("CI", body, refresh=CI_REFRESH if busy else None, section="ci"))
`a5a72f55` 2026-10-27 Pavel Safronov	559
	560
	561	def ci_status_text():
	562	"""For the terminal (fw ci-watch): queue, running and the last few results, private repos included."""
	563	pending, running, recent = ci_overview(include_private=True)
	564	lines = []
	565	for j in running:
	566	lines.append(f'running {j["repo"]} {j["ref"].split("/", 2)[-1]} {j["sha"][:12]} {job_progress(j["pipeline"]) or "starting"}')
	567	for j in pending:
	568	lines.append(f'queued {j["repo"]} {j["ref"].split("/", 2)[-1]} {j["sha"][:12]}')
	569	for started, name, sha, p in recent[:5]:
	570	mark, words, _ = CI_STATES[p["status"]]
	571	lines.append(f'{mark} {words:<6} {name} {p.get("ref", "")} {sha[:12]} {started[:16].replace("T", " ")} {duration(p.get("duration"))}')
`a4997156` 2026-10-31 Pavel Safronov	572	if runner_notice(pending):
	573	lines.insert(0, runner_notice(pending))
`a5a72f55` 2026-10-27 Pavel Safronov	574	return "\n".join(lines) if lines else "no CI jobs yet"
	575
	576
`1758548c` 2026-10-27 Pavel Safronov	577	# ---------------------------------------------------------------- views
	578
	579	class View:
	580	"""One browsable branch.
	581
	582	The default branch keeps the short URLs (/<name>/tree/...). Other branches live under
	583	/<name>/branch/<branch>/, with each '/' in the branch name written as '~': git forbids
	584	'~' in ref names, so two branches can never share a directory.
	585	"""
	586
	587	def __init__(self, name, branch, is_default):
	588	self.name, self.branch, self.is_default = name, branch, is_default
	589	self.ref = "refs/heads/" + branch
	590	self.base = [name] if is_default else [name, "branch", branch.replace("/", "~")]
	591
	592	def url(self, *parts):
	593	return url(self.base, parts)
	594
	595	def path(self, *parts):
	596	return os.path.join(SITE, self.base, parts)
	597
	598	@property
	599	def key(self):
	600	return "/".join(self.base[1:]) or "."
	601
	602
`623239b7` 2026-10-26 Pavel Safronov	603	# ---------------------------------------------------------------- markdown
	604
	605	class LinkContext:
	606	"""Rewrites relative links in a repo's markdown to this site's pages."""
	607
`1758548c` 2026-10-27 Pavel Safronov	608	def __init__(self, view, base_dir, files, dirs, raw):
	609	self.view, self.base_dir, self.files, self.dirs, self.raw = view, base_dir, files, dirs, raw
`623239b7` 2026-10-26 Pavel Safronov	610
	611	def rewrite(self, href, image):
	612	parsed = urllib.parse.urlsplit(href)
	613	if parsed.scheme or parsed.netloc or href.startswith(("#", "/", "mailto:")):
	614	return href
	615	target = posixpath.normpath(posixpath.join(self.base_dir, urllib.parse.unquote(parsed.path)))
	616	frag = ("#" + parsed.fragment) if parsed.fragment else ""
	617	if image and target in self.raw:
`1758548c` 2026-10-27 Pavel Safronov	618	return self.view.url("raw", target)
`623239b7` 2026-10-26 Pavel Safronov	619	if target in self.files:
`1758548c` 2026-10-27 Pavel Safronov	620	return self.view.url("blob", target) + ".html" + frag
`623239b7` 2026-10-26 Pavel Safronov	621	if target in self.dirs or target == ".":
`1758548c` 2026-10-27 Pavel Safronov	622	return self.view.url("tree", "" if target == "." else target) + "/" + frag
`623239b7` 2026-10-26 Pavel Safronov	623	return href
	624
	625
	626	_ATTR = re.compile(r'(<(?:img\|source)\b[^>]?\bsrc\|<a\b[^>]?\bhref)=(["\'])(.*?)\2', re.IGNORECASE \| re.DOTALL)
	627
	628
	629	def render_markdown(text, ctx):
	630	if not markdown:
	631	return f"<pre>{esc(text)}</pre>"
	632	out = markdown.Markdown(extensions=["tables", "fenced_code", "sane_lists"], output_format="html").convert(text)
	633	# Rewrite links in the finished HTML, so raw <img>/<a> tags in the markdown are covered too.
	634	def fix(m):
	635	is_img = not m.group(1).lower().startswith("<a")
	636	new = ctx.rewrite(html.unescape(m.group(3)), image=is_img)
	637	return f"{m.group(1)}={m.group(2)}{esc(new, quote=True)}{m.group(2)}"
	638	return _ATTR.sub(fix, out)
	639
	640
`ec9751fa` 2026-10-27 Pavel Safronov	641	CONTENT = os.environ.get("FW_CONTENT", os.path.join(os.path.dirname(os.path.abspath(__file__)), "content"))
	642
	643
	644	def build_content():
`0a08554d` 2026-10-27 Pavel Safronov	645	"""Site pages: content/<name>.md (Markdown) or content/<name>.html (a fragment) becomes
	646	<site>/<name>/index.html. The title is a `title: ...` line at the top of a Markdown file
	647	(Python-Markdown's meta extension) or a leading `<!-- title: ... -->` in an HTML one.
	648	Files starting with "_" are drafts and are skipped. Also writes 404.html."""
`ec9751fa` 2026-10-27 Pavel Safronov	649	if not markdown:
	650	raise SystemExit("gen.py content needs Python-Markdown in " + PYLIB)
	651	for name in sorted(os.listdir(CONTENT)):
	652	stem, ext = os.path.splitext(name)
`0a08554d` 2026-10-27 Pavel Safronov	653	if ext not in (".md", ".html") or stem.startswith("_"):
`ec9751fa` 2026-10-27 Pavel Safronov	654	continue
	655	with open(os.path.join(CONTENT, name), encoding="utf-8") as f:
`0a08554d` 2026-10-27 Pavel Safronov	656	text = f.read()
	657	if ext == ".md":
`ec9751fa` 2026-10-27 Pavel Safronov	658	md = markdown.Markdown(extensions=["meta", "tables", "fenced_code", "sane_lists"], output_format="html")
`0a08554d` 2026-10-27 Pavel Safronov	659	body = md.convert(text)
	660	title = " ".join(md.Meta.get("title", [stem]))
	661	else:
	662	m = re.match(r"<!-- title: (.*?) -->\n", text)
	663	title, body = (m.group(1), text[m.end():]) if m else (stem, text)
	664	write(os.path.join(SITE, stem, "index.html"), page(title, body, section=stem))
`57d16c8b` 2026-10-29 Pavel Safronov	665	# Images beside the page that it mentions (`![...](./tower.png)`) are copied next to its index.html.
	666	for img in sorted(os.listdir(CONTENT)):
	667	if img.lower().endswith(SHOT_EXTENSIONS) and img in text:
	668	shutil.copyfile(os.path.join(CONTENT, img), os.path.join(SITE, stem, img))
`0a08554d` 2026-10-27 Pavel Safronov	669	write(os.path.join(SITE, "404.html"), page("Not found", (
	670	"<h1>Not found</h1><p>There's nothing at this address. Private repositories aren't listed here, "
	671	'and only the default branch can be browsed.</p><p><a href="/">All repositories</a></p>'), section="-"))
`ec9751fa` 2026-10-27 Pavel Safronov	672
	673
`1758548c` 2026-10-27 Pavel Safronov	674	# ---------------------------------------------------------------- highlighting
	675
	676	# Pygments token types folded into a few CSS classes (colors in style.css). The most
	677	# specific match wins, so Name.Function gets "nf" and other names get nothing.
	678	_TOKEN_CLASSES = []
	679	if pygments_token:
	680	T = pygments_token
	681	_TOKEN_CLASSES = [(T.Comment.Preproc, "cp"), (T.Comment, "c"), (T.Keyword, "k"), (T.String, "s"),
	682	(T.Number, "m"), (T.Name.Function, "nf"), (T.Name.Class, "nc"), (T.Name.Builtin, "nb"),
	683	(T.Name.Tag, "nt"), (T.Name.Attribute, "na"), (T.Name.Decorator, "nd"),
	684	(T.Generic.Inserted, "gi"), (T.Generic.Deleted, "gd"), (T.Generic.Heading, "gh"),
	685	(T.Error, "")]
	686	_class_cache = {}
	687
	688
	689	def token_class(ttype):
	690	if ttype not in _class_cache:
	691	_class_cache[ttype] = next((c for t, c in _TOKEN_CLASSES if ttype in t), "")
	692	return _class_cache[ttype]
	693
	694
	695	def highlight_lines(path, text):
	696	"""The text as escaped HTML lines, colored when Pygments knows the file type."""
	697	lines = text.split("\n")
	698	if lines and lines[-1] == "":
	699	lines.pop()
	700	plain = [esc(l) for l in lines]
	701	if not pygments_lexers or len(text) > MAX_HIGHLIGHT_BYTES:
	702	return plain
	703	try:
	704	lexer = pygments_lexers.get_lexer_for_filename(posixpath.basename(path), stripnl=False, ensurenl=False)
	705	except Exception: # ClassNotFound, or a lexer that fails to load
	706	return plain
	707	out, cur = [], []
	708	try:
	709	for ttype, value in lexer.get_tokens(text):
	710	cls = token_class(ttype)
	711	for i, part in enumerate(value.split("\n")):
	712	if i:
	713	out.append("".join(cur))
	714	cur = []
	715	if part:
	716	cur.append(f'<span class="{cls}">{esc(part)}</span>' if cls else esc(part))
	717	except Exception: # a lexer bug must not stop the page
	718	return plain
	719	out.append("".join(cur))
	720	# Pygments turns \r\n into \n; if that changed the line count, fall back to the plain lines.
	721	while len(out) > len(lines) and out[-1] == "":
	722	out.pop()
	723	return out if len(out) == len(lines) else plain
	724
	725
	726	def source_table(lines, blame=None):
	727	"""Numbered source lines; with blame, a column naming the commit behind each run of lines."""
	728	rows = []
	729	for i, code in enumerate(lines, 1):
	730	cell = ""
	731	if blame is not None:
	732	b = blame[i - 1] if i - 1 < len(blame) else None
	733	starts = b and (i == 1 or blame[i - 2] is None or blame[i - 2]["sha"] != b["sha"])
	734	cell = f'<td class="blame">{b["html"] if starts else ""}</td>'
	735	run = ' class="run"' if starts and i > 1 else ""
	736	rows.append(f'<tr id="L{i}"{run}>{cell}'
	737	f'<td class="ln"><a href="#L{i}">{i}</a></td><td class="code">{code or " "}</td></tr>')
	738	else:
	739	rows.append(f'<tr id="L{i}"><td class="ln"><a href="#L{i}">{i}</a></td><td class="code">{code or " "}</td></tr>')
	740	return f'<table class="source{" blamed" if blame is not None else ""}">{"".join(rows)}</table>'
	741
	742
`623239b7` 2026-10-26 Pavel Safronov	743	# ---------------------------------------------------------------- repo pages
	744
`1758548c` 2026-10-27 Pavel Safronov	745	def tree_entries(repo, ref):
	746	"""Files at `ref` as {path: (mode, sha, size)}; submodules get size -1."""
`623239b7` 2026-10-26 Pavel Safronov	747	out = {}
`1758548c` 2026-10-27 Pavel Safronov	748	data = git(repo, "ls-tree", "-r", "-l", "-z", ref)
`623239b7` 2026-10-26 Pavel Safronov	749	for rec in data.split(b"\0"):
	750	if not rec:
	751	continue
	752	meta, path = rec.split(b"\t", 1)
	753	mode, typ, sha, size = meta.decode().split()
	754	p = path.decode("utf-8", errors="replace")
	755	if not safe_path(p):
	756	continue
	757	out[p] = (mode, sha, -1 if typ == "commit" else int(size))
	758	return out
	759
	760
	761	def all_dirs(files):
	762	dirs = set()
	763	for p in files:
	764	d = posixpath.dirname(p)
	765	while d:
	766	dirs.add(d)
	767	d = posixpath.dirname(d)
	768	return dirs
	769
	770
`1758548c` 2026-10-27 Pavel Safronov	771	def branch_note(view):
	772	if view.is_default:
	773	return ""
	774	return (f'<p class="branch-note">Branch <code>{esc(view.branch)}</code> · '
	775	f'<a href="{url(view.name, "tree")}/">default branch</a> · <a href="{url(view.name, "refs")}/">all branches</a></p>')
	776
	777
	778	def crumbs_for(view, path):
	779	crumbs = [f'<a href="{url(view.name)}/">{esc(view.name)}</a>']
	780	if not view.is_default:
	781	crumbs.append(f"<code>{esc(view.branch)}</code>")
	782	crumbs.append(f'<a href="{view.url("tree")}/">files</a>')
`623239b7` 2026-10-26 Pavel Safronov	783	parts = path.split("/") if path else []
	784	for i, part in enumerate(parts):
	785	sub = "/".join(parts[: i + 1])
	786	if i == len(parts) - 1:
	787	crumbs.append(f"<strong>{esc(part)}</strong>")
	788	else:
`1758548c` 2026-10-27 Pavel Safronov	789	crumbs.append(f'<a href="{view.url("tree", sub)}/">{esc(part)}</a>')
`623239b7` 2026-10-26 Pavel Safronov	790	return crumbs
	791
	792
`1758548c` 2026-10-27 Pavel Safronov	793	def build_tree_page(view, d, files, dirs):
`623239b7` 2026-10-26 Pavel Safronov	794	prefix = d + "/" if d else ""
	795	subdirs = sorted({p[len(prefix):].split("/")[0] for p in dirs if p.startswith(prefix) and p != d and "/" not in p[len(prefix):]})
	796	here = sorted(p for p in files if posixpath.dirname(p) == d)
	797	rows = []
	798	if d:
	799	parent = posixpath.dirname(d)
`1758548c` 2026-10-27 Pavel Safronov	800	rows.append(f'<tr><td><a href="{view.url("tree", parent)}/">..</a></td><td></td></tr>')
`623239b7` 2026-10-26 Pavel Safronov	801	for s in subdirs:
`1758548c` 2026-10-27 Pavel Safronov	802	rows.append(f'<tr><td class="dir"><a href="{view.url("tree", prefix + s)}/">{esc(s)}/</a></td><td></td></tr>')
`623239b7` 2026-10-26 Pavel Safronov	803	for p in here:
	804	size = files[p][2]
	805	label = "submodule" if size < 0 else human_size(size)
`1758548c` 2026-10-27 Pavel Safronov	806	rows.append(f'<tr><td><a href="{view.url("blob", p)}.html">{esc(posixpath.basename(p))}</a></td><td class="num">{label}</td></tr>')
	807	title = d or (view.name if view.is_default else view.branch)
	808	body = f'{branch_note(view)}<h1>{esc(title)}</h1>\n<table class="files">{"".join(rows)}</table>'
	809	write(view.path("tree", d, "index.html"),
	810	page(f"{view.name}/{d}" if d else f"{view.name} files", body, crumbs_for(view, d) if d or not view.is_default else None, view=view))
	811
	812
	813	def blame_lines(repo, view, path, commit_set):
	814	"""[{sha, html}] per line of the file at the view's branch, from `git blame --porcelain`."""
	815	data = git(repo, "blame", "--porcelain", view.ref, "--", path, check=False)
	816	info, out, cur = {}, [], None
	817	for raw in data.split(b"\n"):
	818	if raw.startswith(b"\t"):
	819	out.append(cur)
	820	continue
	821	head = raw.split(b" ", 1)[0]
	822	if len(head) == 40 and re.fullmatch(rb"[0-9a-f]{40}", head):
	823	cur = head.decode()
	824	info.setdefault(cur, {})
	825	elif cur and raw:
	826	k, _, v = raw.decode("utf-8", errors="replace").partition(" ")
	827	info[cur].setdefault(k, v)
	828	cells = {}
	829	for sha, i in info.items():
	830	when = time.strftime("%Y-%m-%d", time.gmtime(int(i.get("author-time", "0") or 0)))
	831	short = f'<code>{sha[:8]}</code>'
	832	link = f'<a href="{url(view.name, "commit", sha)}.html" title="{esc(i.get("summary", ""), quote=True)}">{short}</a>' if sha in commit_set else short
	833	cells[sha] = {"sha": sha, "html": f'{link} <span class="when">{when}</span> <span class="who">{esc(i.get("author", ""))}</span>'}
	834	return [cells.get(s) for s in out]
	835
	836
	837	def build_blob_page(repo, view, path, entry, blobs, ctx, raw_set, commit_set):
	838	"""The file page, plus its blame page for text files. Returns True if a blame page was written."""
`623239b7` 2026-10-26 Pavel Safronov	839	mode, sha, size = entry
`1758548c` 2026-10-27 Pavel Safronov	840	title = f"{view.name}/{path}"
	841	crumbs = crumbs_for(view, path)
	842	blamed = False
	843	links = f' · <a href="{view.url("history", path)}.html">history</a>'
`623239b7` 2026-10-26 Pavel Safronov	844	if size < 0:
	845	body = f"<h1>{esc(posixpath.basename(path))}</h1><p>Git submodule at commit <code>{esc(sha)}</code>.</p>"
	846	else:
	847	data = blobs.read(sha)
`1758548c` 2026-10-27 Pavel Safronov	848	if path in raw_set:
	849	links += f' · <a href="{view.url("raw", path)}">raw</a>'
`623239b7` 2026-10-26 Pavel Safronov	850	ext = posixpath.splitext(path)[1].lower()
`1758548c` 2026-10-27 Pavel Safronov	851	text = None
`623239b7` 2026-10-26 Pavel Safronov	852	if path in raw_set:
`1758548c` 2026-10-27 Pavel Safronov	853	content = f'<p><img class="blob" src="{view.url("raw", path)}" alt="{esc(posixpath.basename(path))}"></p>'
`623239b7` 2026-10-26 Pavel Safronov	854	elif is_binary(data):
	855	content = "<p>Binary file, not shown. Clone the repository to get it.</p>"
	856	elif size > MAX_TEXT_BYTES:
	857	content = "<p>File too large to show here. Clone the repository to get it.</p>"
	858	else:
	859	text = data.decode("utf-8", errors="replace")
`1758548c` 2026-10-27 Pavel Safronov	860	lines = highlight_lines(path, text)
	861	source = source_table(lines)
`623239b7` 2026-10-26 Pavel Safronov	862	if ext in (".md", ".markdown") and markdown:
	863	ctx.base_dir = posixpath.dirname(path)
	864	content = (f'<article class="markdown">{render_markdown(text, ctx)}</article>'
	865	f"<details><summary>Source</summary>{source}</details>")
	866	else:
	867	content = source
`1758548c` 2026-10-27 Pavel Safronov	868	if size <= MAX_BLAME_BYTES:
	869	blame = blame_lines(repo, view, path, commit_set)
	870	if blame:
	871	links += f' · <a href="{view.url("blame", path)}.html">blame</a>'
	872	bbody = (f'{branch_note(view)}<h1>{esc(posixpath.basename(path))}</h1>'
	873	f'<p class="meta">Blame: the commit that last changed each line · '
	874	f'<a href="{view.url("blob", path)}.html">file</a> · '
	875	f'<a href="{view.url("history", path)}.html">history</a></p>{source_table(lines, blame)}')
	876	write(view.path("blame", path + ".html"), page(f"{title} blame", bbody, crumbs, view=view))
	877	blamed = True
	878	meta = f'<p class="meta">{human_size(size)}{links}</p>'
	879	body = f"{branch_note(view)}<h1>{esc(posixpath.basename(path))}</h1>{meta}{content}"
	880	if not blamed:
	881	remove(view.path("blame", path + ".html"))
	882	write(view.path("blob", path + ".html"), page(title, body, crumbs, view=view))
	883	return blamed
	884
	885
	886	COMMIT_FORMAT = "%H%x1f%h%x1f%an%x1f%ad%x1f%s"
	887
	888
	889	def commit_list(repo, ref, limit):
	890	raw = git_text(repo, "log", f"-n{limit}", "--date=short", f"--format={COMMIT_FORMAT}", ref, check=False)
`623239b7` 2026-10-26 Pavel Safronov	891	return [line.split("\x1f") for line in raw.splitlines() if line.count("\x1f") == 4]
	892
	893
`1758548c` 2026-10-27 Pavel Safronov	894	def file_histories(repo, ref, paths):
	895	"""{path: [commit, ...]} newest first, at most HISTORY_LIMIT each, from one `git log` pass."""
	896	raw = git(repo, "-c", "core.quotePath=false", "log", "--no-renames", "--name-only", "--date=short",
	897	f"-n{COMMIT_LIMIT}", f"--format=%x1e{COMMIT_FORMAT}", ref, check=False).decode("utf-8", errors="replace")
	898	hist = {}
	899	for chunk in raw.split("\x1e"):
	900	lines = chunk.split("\n")
	901	commit = lines[0].split("\x1f")
	902	if len(commit) != 5:
	903	continue
	904	for p in lines[1:]:
	905	if p in paths:
	906	h = hist.setdefault(p, [])
	907	if len(h) < HISTORY_LIMIT:
	908	h.append(commit)
	909	return hist
	910
	911
`57d16c8b` 2026-10-29 Pavel Safronov	912	# Commits written with Claude Code carry the session's running token tally as git trailers:
	913	# FW-Tokens-In: 10638
	914	# FW-Tokens-Out: 101300
	915	# FW-Tokens-Cached: 5110043
`ced0eafa` 2026-11-04 Pavel Safronov	916	# FW-Tokens-Method: 2 (counting method; absent = 1, which counted duplicate transcript lines and
	917	# missed subagents, so those tallies run roughly 1.5-2x high)
	918	# FW-Model-Tokens: claude-opus-5-5 in=40 out=20311 read=4100233 write5m=0 write1h=88120 (one per model)
`57d16c8b` 2026-10-29 Pavel Safronov	919	# FW-Session: <session id> (provenance only: a resumed session may change it while the tally goes on)
	920	# Older commits have one rounded line instead: "Usage: 10.6K in / 101.3K out / 5.1M cached (session tally)".
	921	# A commit's own cost is the step from the previous commit's tally. The tally falls when a new
`ced0eafa` 2026-11-04 Pavel Safronov	922	# session starts, and that commit's cost is its whole tally. Dollar cost needs the per-model lines and
	923	# prices.json; without them the cost is unknown, never zero.
`57d16c8b` 2026-10-29 Pavel Safronov	924	USAGE_KEYS = ("FW-Tokens-In", "FW-Tokens-Out", "FW-Tokens-Cached")
	925	USAGE_RE = re.compile(r"^Usage:\s*([\d.]+)([KMG]?) in / ([\d.]+)([KMG]?) out / ([\d.]+)([KMG]?) cached", re.M)
`ced0eafa` 2026-11-04 Pavel Safronov	926	MODEL_TOKENS_RE = re.compile(r"^FW-Model-Tokens:\s(\S+)\s+in=(\d+)\s+out=(\d+)\s+read=(\d+)\s+write5m=(\d+)\s+write1h=(\d+)\s$", re.M)
	927	AI_COAUTHOR_RE = re.compile(r"^Co-Authored-By:.*(noreply@anthropic\.com\|\bClaude\b\|Copilot\|ChatGPT\|openai)", re.M \| re.I)
`57d16c8b` 2026-10-29 Pavel Safronov	928	USAGE_UNITS = {"": 1, "K": 103, "M": 106, "G": 10**9}
	929	USAGE_HOT = 2 # a commit with this many times the median output is highlighted
`ced0eafa` 2026-11-04 Pavel Safronov	930	OVER_100K = "@over100k" # model-key suffix for requests whose prompt exceeded 100,000 tokens
	931	MAX_WEEKS = 104 # a longer history is charted by month
`57d16c8b` 2026-10-29 Pavel Safronov	932
	933
	934	def parse_usage(message):
	935	"""([in, out, cached], [slack, ...]) from a message's usage trailers, or None.
	936
	937	Trailer counts are exact. The old Usage: line is rounded to a tenth of its unit, so each count
	938	is only known to within half a tenth: the slack.
	939	"""
	940	counts = [re.search(rf"^{key}:\s(\d+)\s$", message, re.M) for key in USAGE_KEYS]
	941	if all(counts):
	942	return [float(m.group(1)) for m in counts], [0, 0, 0]
	943	m = USAGE_RE.search(message)
	944	if not m:
	945	return None
	946	units = [USAGE_UNITS[m.group(i + 1)] for i in (1, 3, 5)]
	947	return [float(m.group(i)) * u for i, u in zip((1, 3, 5), units)], [u / 20 for u in units]
	948
	949
`ced0eafa` 2026-11-04 Pavel Safronov	950	def parse_model_usage(message):
	951	"""{model: [in, out, read, write5m, write1h]} from FW-Model-Tokens trailers, or None (older commits)."""
	952	found = MODEL_TOKENS_RE.findall(message)
	953	if not found:
	954	return None
	955	models = {}
	956	for m in found:
	957	cur = models.setdefault(m[0], [0] * 5)
	958	models[m[0]] = [a + int(b) for a, b in zip(cur, m[1:])]
	959	return models
	960
	961
	962	def usage_method(message):
	963	"""The FW-Tokens-Method trailer as an int; commits without it used method 1."""
	964	m = re.search(r"^FW-Tokens-Method:\s(\d+)\s$", message, re.M)
	965	return int(m.group(1)) if m else 1
	966
	967
	968	_prices = None
	969
	970
	971	def load_prices():
	972	"""server/prices.json beside this file (USD per 1M tokens), read once; {} if missing or broken."""
	973	global _prices
	974	if _prices is None:
	975	try:
	976	with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "prices.json")) as f:
	977	_prices = json.load(f)
	978	except (OSError, ValueError):
	979	_prices = {}
	980	return _prices
	981
	982
	983	def commit_cost(model_delta, prices):
	984	"""(dollars, unknown_models): the list-price estimate of a per-model delta, or (None, [...]) when it is
	985	unknown: no model data, or a model with tokens that the price table lacks. Keys ending '@over100k'
	986	use that model's above_100k rates."""
	987	if not model_delta:
	988	return None, []
	989	models = prices.get("models", {})
	990	total, unknown = 0.0, []
	991	for key, (inp, out, read, w5, w1) in model_delta.items():
	992	if not any((inp, out, read, w5, w1)):
	993	continue
	994	base = key[:-len(OVER_100K)] if key.endswith(OVER_100K) else key
	995	rates = models.get(base, {})
	996	if key.endswith(OVER_100K):
	997	rates = rates.get("above_100k")
	998	if not rates:
	999	unknown.append(key)
	1000	continue
	1001	total += (inp * rates["input"] + out * rates["output"] + read * rates["cache_read"]
	1002	+ w5 * rates["cache_write_5m"] + w1 * rates["cache_write_1h"]) / 1e6
	1003	return (None, unknown) if unknown else (total, [])
	1004
	1005
`57d16c8b` 2026-10-29 Pavel Safronov	1006	def fmt_tokens(n):
	1007	"""10638 -> '10.6K': the units the trailer is written in."""
	1008	for unit, suffix in ((109, "G"), (106, "M"), (10**3, "K")):
	1009	if n >= unit:
	1010	return f"{n / unit:.1f}{suffix}"
	1011	return str(round(n))
	1012
	1013
	1014	def approx_tokens(n, slack):
	1015	return "≈0" if n <= slack else ("~" if slack else "") + fmt_tokens(n)
	1016
	1017
	1018	def usage_costs(repo, ref):
`ced0eafa` 2026-11-04 Pavel Safronov	1019	"""(costs, total, commits) for `ref`'s first-parent chain: see compute_costs."""
`57d16c8b` 2026-10-29 Pavel Safronov	1020	raw = git_text(repo, "log", "--first-parent", "--reverse", "--date=short",
	1021	f"--format={COMMIT_FORMAT}%x1f%B%x1e", ref, check=False)
`ced0eafa` 2026-11-04 Pavel Safronov	1022	parsed = []
`57d16c8b` 2026-10-29 Pavel Safronov	1023	for chunk in raw.split("\x1e"):
	1024	fields = chunk.strip("\n").split("\x1f", 5)
`ced0eafa` 2026-11-04 Pavel Safronov	1025	if len(fields) == 6:
	1026	parsed.append((fields[:5], fields[5]))
	1027	return compute_costs(parsed)
	1028
	1029
	1030	def models_restarted(prev, cur):
	1031	"""True when per-model tallies cannot continue one session: a counter fell, a model with tokens vanished
	1032	(counters never disappear within a session), or model data appears where the previous commit had none."""
	1033	if prev is None:
	1034	return cur is not None
	1035	if cur is None:
	1036	return False
	1037	for m, v in prev.items():
	1038	now = cur.get(m)
	1039	if now is None:
	1040	if any(v):
	1041	return True
	1042	elif any(n < o for n, o in zip(now, v)):
	1043	return True
	1044	return False
	1045
	1046
	1047	def compute_costs(commits, prices=None):
	1048	"""commits: [(fields, message)] oldest first, fields = COMMIT_FORMAT's five values (sha, short, author,
	1049	date, subject). Returns (costs, total, allc):
	1050	costs {sha: {"row", "tally", "delta", "slack", "session", "new", "method", "models", "cost", "unknown"}}
	1051	for each commit with a usage trailer. `models` is the per-model delta (None for old commits), `cost`
	1052	its list-price estimate in dollars (None = unknown), `unknown` the models that could not be priced.
	1053	total the number of commits
	1054	allc [{"sha", "date", "ai", "has_tokens"}] for every commit, so trailer-less ones can be bucketed.
	1055	A new session starts (delta = whole tally) at the first trailer commit, when the counting method
	1056	changes, when the output or cached tally falls, when any per-model counter falls or a model with tokens
	1057	vanishes, or when model data appears after commits that had none."""
	1058	prices = load_prices() if prices is None else prices
	1059	costs, allc, prev, session = {}, [], None, 0
	1060	for fields, message in commits:
	1061	parsed = parse_usage(message)
	1062	allc.append({"sha": fields[0], "date": fields[3], "has_tokens": bool(parsed),
	1063	"ai": bool(parsed) or bool(AI_COAUTHOR_RE.search(message))})
`57d16c8b` 2026-10-29 Pavel Safronov	1064	if not parsed:
	1065	continue
	1066	tally, slack = parsed
`ced0eafa` 2026-11-04 Pavel Safronov	1067	method, tmodels = usage_method(message), parse_model_usage(message)
	1068	new = (prev is None or method != prev["method"] or tally[1] < prev["tally"][1] or tally[2] < prev["tally"][2]
	1069	or models_restarted(prev["models"], tmodels))
`57d16c8b` 2026-10-29 Pavel Safronov	1070	if new:
	1071	session += 1
	1072	delta, delta_slack = tally, slack
`ced0eafa` 2026-11-04 Pavel Safronov	1073	mdelta = tmodels
`57d16c8b` 2026-10-29 Pavel Safronov	1074	else:
`ced0eafa` 2026-11-04 Pavel Safronov	1075	delta = [t - p for t, p in zip(tally, prev["tally"])]
	1076	delta_slack = [s + p for s, p in zip(slack, prev["slack"])]
	1077	mdelta = ({m: [a - b for a, b in zip(v, (prev["models"] or {}).get(m, [0] * 5))] for m, v in tmodels.items()}
	1078	if tmodels is not None else None)
	1079	cost, unknown = commit_cost(mdelta, prices)
	1080	costs[fields[0]] = {"row": fields, "tally": tally, "delta": delta, "slack": delta_slack,
	1081	"session": session, "new": new, "method": method, "models": mdelta,
	1082	"cost": cost, "unknown": unknown}
	1083	prev = {"tally": tally, "slack": slack, "method": method, "models": tmodels}
	1084	return costs, len(allc), allc
	1085
	1086
	1087	def week_buckets(allc, costs):
	1088	"""Commits grouped by Monday-started week of their author date, empty weeks included; by month
	1089	when the history is longer than MAX_WEEKS weeks. Per bucket: label, commits, ai, with_tokens,
	1090	with_cost, out, read, cost (sum over commits whose cost is known), nodata (commits without usage data)."""
	1091	if not allc:
	1092	return []
	1093	dates = [datetime.date.fromisoformat(c["date"]) for c in allc]
	1094	first, last = min(dates), max(dates)
	1095	monday = lambda d: d - datetime.timedelta(days=d.weekday())
	1096	by_month = (monday(last) - monday(first)).days // 7 + 1 > MAX_WEEKS
	1097	if by_month:
	1098	key = lambda d: (d.year, d.month)
	1099	keys, y, m = [], first.year, first.month
	1100	while (y, m) <= key(last):
	1101	keys.append((y, m))
	1102	y, m = (y + 1, 1) if m == 12 else (y, m + 1)
	1103	labels = [f"month of {y}-{m:02d}" for y, m in keys]
	1104	else:
	1105	key = monday
	1106	keys = [monday(first) + datetime.timedelta(weeks=i) for i in range((monday(last) - monday(first)).days // 7 + 1)]
	1107	labels = [f"week of {k.isoformat()}" for k in keys]
	1108	buckets = [{"label": lb, "commits": 0, "ai": 0, "with_tokens": 0, "with_cost": 0, "out": 0, "read": 0,
	1109	"cost": 0.0, "nodata": 0} for lb in labels]
	1110	index = {k: i for i, k in enumerate(keys)}
	1111	for c, d in zip(allc, dates):
	1112	b = buckets[index[key(d)]]
	1113	b["commits"] += 1
	1114	b["ai"] += c["ai"]
	1115	e = costs.get(c["sha"])
	1116	if not e:
	1117	b["nodata"] += 1
	1118	continue
	1119	b["with_tokens"] += 1
	1120	b["out"] += e["delta"][1]
	1121	b["read"] += e["delta"][2]
	1122	if e["cost"] is not None:
	1123	b["with_cost"] += 1
	1124	b["cost"] += e["cost"]
	1125	return buckets
	1126
	1127
	1128	def bucket_unit(buckets):
	1129	return "month" if buckets and buckets[0]["label"].startswith("month") else "week"
	1130
	1131
`799905ae` 2026-11-04 Pavel Safronov	1132	def line_chart_svg(labels, values, *, fmt, unknown, second=None, legend=None, aria, axis_fmt=fmt_tokens,
	1133	pattern_id="hatch", tips=None, hatch_note="hatched = commits without usage data (unknown, not zero)"):
	1134	"""An inline SVG line chart without script. labels: one text per bucket, ending in its date (the axis shows
	1135	the first and last); values: one number per bucket, or None where the value is unknown: the line breaks
	1136	there (a gap), it never drops to zero, and a single isolated point is only a dot; fmt(value) -> text for the
	1137	tooltip; unknown: per bucket (commits lacking data, commits), drawn as a hatched strip under the axis;
	1138	second: optional second series (drawn in the accent colour, the first then grey); legend: [(text, "muted"\|"")]
	1139	drawn under the chart; axis_fmt formats the top axis label; pattern_id keeps the hatch pattern's id unique when
	1140	a page has several charts; tips: optional full tooltip text per bucket instead of the default; hatch_note: the
	1141	line under the axis ("" for none). Every bucket has a transparent hover target with a <title>, every known
	1142	point a dot with the same <title>. Colours come from CSS classes only."""
	1143	if not labels:
	1144	return ""
`ced0eafa` 2026-11-04 Pavel Safronov	1145	left, right, top, base = 44, 716, 12, 140
	1146	n = len(values)
`799905ae` 2026-11-04 Pavel Safronov	1147	known = [v for v in list(values) + list(second or []) if v is not None]
	1148	vmax = max(known, default=0) or 1
`ced0eafa` 2026-11-04 Pavel Safronov	1149	step = (right - left) / max(n, 1)
`799905ae` 2026-11-04 Pavel Safronov	1150	x = lambda i: left + (i + 0.5) * step
	1151	y = lambda v: base - (base - top) * v / vmax
	1152	tip_of = []
`ced0eafa` 2026-11-04 Pavel Safronov	1153	for i, v in enumerate(values):
	1154	bad, total = unknown[i]
`799905ae` 2026-11-04 Pavel Safronov	1155	tip_of.append(tips[i] if tips else (
	1156	f"{labels[i]}: " + ("no data" if v is None else fmt(v))
	1157	+ (f", {total - bad} of {total} commits with data" if total else ", no commits")))
	1158	hits = "".join(f'<rect class="hit" x="{left + i * step:.1f}" y="{top}" width="{step:.1f}" height="{base - top}">'
	1159	f'<title>{esc(tip_of[i])}</title></rect>' for i in range(n))
	1160
	1161	def series(vals, muted):
	1162	cls = " muted" if muted else ""
	1163	segments, cur = [], []
	1164	for i, v in enumerate(vals):
	1165	if v is None:
	1166	if cur:
	1167	segments.append(cur)
	1168	cur = []
	1169	else:
	1170	cur.append((i, v))
	1171	if cur:
	1172	segments.append(cur)
	1173	out = []
	1174	for seg in segments:
	1175	if len(seg) > 1:
	1176	pts = " L".join(f"{x(i):.1f} {y(v):.1f}" for i, v in seg)
	1177	out.append(f'<path class="area{cls}" d="M{x(seg[0][0]):.1f} {base} L{pts} L{x(seg[-1][0]):.1f} {base} Z"/>')
	1178	out.append(f'<path class="line{cls}" d="M{pts}"/>')
	1179	out += [f'<circle class="dot{cls}" cx="{x(i):.1f}" cy="{y(v):.1f}" r="3"><title>{esc(tip_of[i])}</title></circle>'
	1180	for seg in segments for i, v in seg]
	1181	return "".join(out)
	1182
	1183	parts = [series(values, second is not None)] + ([series(second, False)] if second is not None else [])
	1184	bad_strips = "".join(
	1185	f'<rect class="unk" x="{left + i * step:.1f}" y="{base + 4}" width="{max(step - 1, 1):.1f}" height="6" '
	1186	f'fill="url(#{pattern_id})"><title>{esc(tip_of[i])}</title></rect>' for i in range(n) if unknown[i][0])
`ced0eafa` 2026-11-04 Pavel Safronov	1187	first, last = labels[0].rsplit(" ", 1)[-1], labels[-1].rsplit(" ", 1)[-1]
`799905ae` 2026-11-04 Pavel Safronov	1188	key = ""
	1189	for j, (text, cls) in enumerate(legend or []):
	1190	kx = left + j * 260
	1191	key += (f'<line class="line {cls}" x1="{kx}" y1="189" x2="{kx + 28}" y2="189"/>'
	1192	f'<text x="{kx + 34}" y="194" text-anchor="start">{esc(text)}</text>')
`ced0eafa` 2026-11-04 Pavel Safronov	1193	return (f'<svg class="chart" viewBox="0 0 720 200" role="img" aria-label="{esc(aria)}">'
	1194	f'<defs><pattern id="{pattern_id}" width="6" height="6" patternUnits="userSpaceOnUse">'
	1195	'<path class="hatchline" d="M0 6L6 0"/></pattern></defs>'
	1196	f'<line class="axis" x1="{left}" y1="{base}" x2="{right}" y2="{base}"/>'
`799905ae` 2026-11-04 Pavel Safronov	1197	+ "".join(parts) + bad_strips + hits
`ced0eafa` 2026-11-04 Pavel Safronov	1198	+ f'<text x="{left - 4}" y="{base}" text-anchor="end">0</text>'
	1199	f'<text x="{left - 4}" y="{top + 8}" text-anchor="end">{esc(axis_fmt(vmax))}</text>'
	1200	f'<text x="{left}" y="{base + 28}" text-anchor="start">{esc(first)}</text>'
	1201	f'<text x="{right}" y="{base + 28}" text-anchor="end">{esc(last)}</text>'
`799905ae` 2026-11-04 Pavel Safronov	1202	+ (f'<text x="{left}" y="194" text-anchor="start">{esc(hatch_note)}</text>' if hatch_note else "")
	1203	+ key + '</svg>')
	1204
	1205
	1206	COST_NOTE = "estimated API list-price equivalent, not what was paid"
	1207
	1208
	1209	def usage_chart(title, caption, labels, values, , pattern_id, empty, *kw):
	1210	"""One chart section: heading, one-line caption, and the SVG; or `empty` as a message when no bucket has a
	1211	known value, so a chart never shows an empty axis. A value of None means unknown (a gap in the line); a known
	1212	0 is a real value and still gets a chart. The one function every Usage chart goes through."""
	1213	head = f"<h2>{esc(title)}</h2><p class=\"meta\">{caption}</p>"
	1214	if not labels or all(v is None for v in values):
	1215	return head + f'<p class="meta">{esc(empty)}</p>'
	1216	return head + line_chart_svg(labels, values, pattern_id=pattern_id, **kw)
	1217
	1218
	1219	def usage_charts(buckets, prefix):
	1220	"""The Usage chart sections (output tokens, est. cost, cached reads, cost per commit, AI-marked commits) for
	1221	`buckets` from week_buckets, as HTML. `prefix` keeps the SVG pattern ids unique per page."""
	1222	unit = bucket_unit(buckets)
	1223	labels = [b["label"] for b in buckets]
	1224	nodata = [(b["nodata"], b["commits"]) for b in buckets]
	1225	nocost = [(b["commits"] - b["with_cost"], b["commits"]) for b in buckets]
	1226	money = lambda v: f"${v:,.2f}"
	1227	cost_note = "hatched = commits without a known cost (unknown, not zero)"
	1228	per_commit = [b["cost"] / b["with_cost"] if b["with_cost"] else None for b in buckets]
	1229	known = lambda b, key, n: b[key] if b[n] else None # None = no commit has the data: unknown, not zero
	1230	out = usage_chart(
	1231	f"Output tokens per {unit}", f"Output tokens of commits with usage trailers; the main measure of work done.",
	1232	labels, [known(b, "out", "with_tokens") for b in buckets], pattern_id=prefix + "-out", empty="No output tokens recorded yet.",
	1233	fmt=lambda v: f"{fmt_tokens(v)} output tokens", unknown=nodata,
	1234	aria=f"Output tokens per {unit}, from commits with usage trailers; hatched marks commits without usage data")
	1235	cost = usage_chart(
	1236	f"Estimated cost per {unit}", f"Sum of known per-commit costs: {COST_NOTE}. Commits without a known cost are "
	1237	"left out, not counted as zero.", labels, [known(b, "cost", "with_cost") for b in buckets], pattern_id=prefix + "-cost",
	1238	empty="No commit has a known cost yet (cost unknown, not zero).", fmt=lambda v: f"est. {money(v)} ({COST_NOTE})",
	1239	tips=[f"{b['label']}: est. {money(b['cost'])} ({COST_NOTE}), {b['with_cost']} of {b['commits']} commits with a known cost"
	1240	if b["with_cost"] else f"{b['label']}: " + ("cost unknown, 0 commits with a known cost" if b["commits"] else "no commits")
	1241	for b in buckets],
	1242	unknown=nocost, axis_fmt=money, hatch_note=cost_note,
	1243	aria=f"Estimated cost per {unit}, {COST_NOTE}; hatched marks commits whose cost is unknown")
	1244	cached = usage_chart(
	1245	f"Cached reads per {unit}", "Tokens read from the prompt cache. They grow with context length and turns, so "
	1246	"they track session length more than work done.", labels, [known(b, "read", "with_tokens") for b in buckets],
	1247	pattern_id=prefix + "-read", empty="No cached reads recorded yet.",
	1248	fmt=lambda v: f"{fmt_tokens(v)} cached reads", unknown=nodata,
	1249	aria=f"Cached reads per {unit}, from commits with usage trailers; hatched marks commits without usage data")
	1250	tips = [f"{b['label']}: est. {money(v)} per commit ({COST_NOTE}), n={plural(b['with_cost'], 'commit')} with a known cost"
	1251	if b["with_cost"] else f"{b['label']}: " + ("cost unknown, n=0 commits with a known cost" if b["commits"] else "no commits")
	1252	for b, v in zip(buckets, per_commit)]
	1253	each = usage_chart(
	1254	f"Cost per commit, average per {unit}", f"Known cost divided by the commits with a known cost that {unit} "
	1255	f"(n is in the tooltip): {COST_NOTE}. A commit's cost includes dead ends since the previous trailer.",
	1256	labels, per_commit, pattern_id=prefix + "-each", empty="No commit has a known cost yet (cost unknown, not zero).",
	1257	fmt=lambda v: money(v), unknown=nocost, axis_fmt=money, hatch_note=cost_note, tips=tips,
	1258	aria=f"Average estimated cost per commit per {unit}, {COST_NOTE}; hatched marks commits whose cost is unknown")
	1259	ai_tips = [f"{b['label']}: {plural(b['commits'], 'commit')}, {b['ai']} AI-marked, {b['commits'] - b['ai']} with no AI marker"
	1260	for b in buckets]
	1261	ai = usage_chart(
	1262	f"AI-marked commits per {unit}", "Grey line: all commits. Blue line: commits with an AI marker (an AI "
	1263	"Co-Authored-By trailer, or a usage trailer). The rest have no AI marker, which does not mean a human "
	1264	"wrote them: a marker is only there when the tool added it.", labels, [b["commits"] for b in buckets],
	1265	pattern_id=prefix + "-ai", empty="No commits yet.", fmt=lambda v: f"{v} commits",
	1266	unknown=[(0, b["commits"]) for b in buckets], second=[b["ai"] for b in buckets],
	1267	legend=[("All commits", "muted"), ("AI-marked commits", "")], hatch_note="", tips=ai_tips,
	1268	aria=f"Commits per {unit}: one line for all commits, one for the AI-marked ones; the rest have no AI marker")
	1269	return out + cost + cached + each + ai
`57d16c8b` 2026-10-29 Pavel Safronov	1270
	1271
	1272	_usage_cache = {}
	1273
	1274
	1275	def repo_usage(name):
`ced0eafa` 2026-11-04 Pavel Safronov	1276	"""{"costs", "commits", "all", "median", "max", "dollars", "priced"} for a repo's default branch:
	1277	output tokens per commit, and the summed list-price estimate over the `priced` commits whose cost is known."""
`57d16c8b` 2026-10-29 Pavel Safronov	1278	if name not in _usage_cache:
`ced0eafa` 2026-11-04 Pavel Safronov	1279	costs, total, allc = usage_costs(os.path.join(GIT_ROOT, name + ".git"), "HEAD")
`57d16c8b` 2026-10-29 Pavel Safronov	1280	outs = sorted(c["delta"][1] for c in costs.values())
`ced0eafa` 2026-11-04 Pavel Safronov	1281	known = [c["cost"] for c in costs.values() if c["cost"] is not None]
	1282	_usage_cache[name] = {"costs": costs, "commits": total, "all": allc,
	1283	"median": outs[len(outs) // 2] if outs else 0, "max": outs[-1] if outs else 0,
	1284	"dollars": sum(known), "priced": len(known)}
`57d16c8b` 2026-10-29 Pavel Safronov	1285	return _usage_cache[name]
	1286
	1287
`ced0eafa` 2026-11-04 Pavel Safronov	1288	def cost_text(c):
	1289	"""The estimated dollar cost with its full label, or why the cost is unknown."""
	1290	if c["cost"] is not None:
	1291	return (f"est. ${c['cost']:.2f} (API list-price equivalent at {load_prices().get('as_of', '?')} rates, "
	1292	"not what was paid)")
	1293	if c["unknown"]:
	1294	return f"cost unknown (no price for {', '.join(c['unknown'])})"
	1295	return "cost unknown (no per-model trailer)"
	1296
	1297
`57d16c8b` 2026-10-29 Pavel Safronov	1298	def usage_cell(usage, sha):
	1299	c = usage["costs"].get(sha)
	1300	if not c:
	1301	return '<td class="cost"></td>'
	1302	out, slack = c["delta"][1], c["slack"][1]
	1303	median = usage["median"]
	1304	real = out > slack
	1305	hot = real and median and out >= USAGE_HOT * median
	1306	title = (f"{approx_tokens(out, slack)} output tokens, {approx_tokens(c['delta'][2], c['slack'][2])} cached reads"
	1307	+ (f" ({out / median:.1f}× the median commit)" if real and median else "")
`ced0eafa` 2026-11-04 Pavel Safronov	1308	+ f" · {cost_text(c)}"
`57d16c8b` 2026-10-29 Pavel Safronov	1309	+ (" · first commit of a session: counts everything since it began" if c["new"] else ""))
	1310	width = 100 * out / usage["max"] if usage["max"] else 0
	1311	return (f'<td class="cost" title="{esc(title)}"><span class="track"><span class="bar{" hot" if hot else ""}" '
	1312	f'style="width:{width:.1f}%"></span></span>{approx_tokens(out, slack)}</td>')
	1313
	1314
	1315	def usage_panel(name, sha):
	1316	"""Usage line for a commit page. Only facts fixed by the commit's own history: no rank, so the page never goes stale."""
	1317	c = repo_usage(name)["costs"].get(sha)
	1318	if not c:
	1319	return ""
	1320	d, s, t = c["delta"], c["slack"], c["tally"]
`ced0eafa` 2026-11-04 Pavel Safronov	1321	cost = f" · {esc(cost_text(c))}"
`57d16c8b` 2026-10-29 Pavel Safronov	1322	return ('<p class="meta usage">Claude Code usage: this commit '
	1323	f'{approx_tokens(d[1], s[1])} out / {approx_tokens(d[2], s[2])} cached'
	1324	+ (" (first commit of its session: everything since it began)" if c["new"] else "")
`ced0eafa` 2026-11-04 Pavel Safronov	1325	+ cost
`57d16c8b` 2026-10-29 Pavel Safronov	1326	+ f' · session {c["session"]} tally {fmt_tokens(t[0])} in / {fmt_tokens(t[1])} out / {fmt_tokens(t[2])} cached'
	1327	f' · <a href="{url(name, "usage")}/">usage by commit and session</a></p>')
	1328
	1329
	1330	def commits_table(name, commits, commit_set=None, usage=None):
`1758548c` 2026-10-27 Pavel Safronov	1331	def sha_cell(h, s):
	1332	if commit_set is not None and h not in commit_set:
	1333	return f"<code>{esc(s)}</code>"
	1334	return f'<a href="{url(name, "commit", h)}.html"><code>{esc(s)}</code></a>'
	1335	ci = ci_pipelines(name)
	1336	rows = "".join(f'<tr><td class="date">{esc(d)}</td><td>{sha_cell(h, s)}</td>'
`57d16c8b` 2026-10-29 Pavel Safronov	1337	f'<td>{ci_mark(name, h, ci.get(h))}{esc(subj)}</td><td class="author">{esc(a)}</td>'
	1338	f'{usage_cell(usage, h) if usage else ""}</tr>'
`1758548c` 2026-10-27 Pavel Safronov	1339	for h, s, a, d, subj in commits)
`623239b7` 2026-10-26 Pavel Safronov	1340	return f'<table class="log">{rows}</table>'
	1341
	1342
`799905ae` 2026-11-04 Pavel Safronov	1343	def plural(n, word):
	1344	return f"{n} {word}" + ("" if n == 1 else "s")
	1345
	1346
	1347	def cost_stat(dollars, priced, as_of):
	1348	"""('Estimated cost', html) for usage_stats: the summed estimate over the `priced` commits with a known cost,
	1349	with the full label in a small note, or "unknown" (never $0) when no commit has one."""
	1350	if not priced:
	1351	return "Estimated cost", "unknown, not zero (no commit has per-model trailers)"
	1352	return "Estimated cost", (f'est. ${dollars:,.2f} over the {plural(priced, "commit")} with a known cost'
	1353	f'<small>estimated API list-price equivalent at prices of {esc(as_of)}, not what was paid</small>')
	1354
	1355
	1356	def usage_stats(items):
	1357	"""A compact list of facts, one per line: [(label, html value)]. Values are built by the caller (already escaped)."""
	1358	return '<dl class="stats">' + "".join(f"<div><dt>{esc(k)}</dt><dd>{v}</dd></div>" for k, v in items) + "</dl>"
	1359
	1360
	1361	def session_cost(cs):
	1362	"""Summed estimated cost of a session's commits, or None unless every commit has a known cost."""
	1363	return None if any(c["cost"] is None for c in cs) else sum(c["cost"] for c in cs)
	1364
	1365
	1366	def session_cost_cell(cs):
	1367	cost = session_cost(cs)
	1368	if cost is None:
	1369	return '<td class="num" title="at least one commit in this session has no known cost">unknown</td>'
	1370	return (f'<td class="num" title="estimated API list-price equivalent at {esc(load_prices().get("as_of", "?"))} '
	1371	f'rates, not what was paid">est. ${cost:,.2f}</td>')
	1372
	1373
`57d16c8b` 2026-10-29 Pavel Safronov	1374	def build_usage_page(view, commit_set):
	1375	"""/<repo>/usage/: commits by output tokens, and a table of sessions. Removed when no commit has a trailer."""
	1376	name = view.name
	1377	usage = repo_usage(name)
	1378	costs = usage["costs"]
	1379	path = view.path("usage", "index.html")
	1380	if not costs:
	1381	remove(path)
	1382	return
	1383	sessions = {}
	1384	for c in costs.values():
	1385	sessions.setdefault(c["session"], []).append(c)
	1386	total_out = sum(c["delta"][1] for c in costs.values()) or 1
	1387	total_cached = sum(c["delta"][2] for c in costs.values())
	1388	session_rows = "".join(
	1389	f'<tr><td class="num">{n}</td><td class="date">{esc(cs[0]["row"][3])}'
	1390	+ (f' – {esc(cs[-1]["row"][3])}' if cs[-1]["row"][3] != cs[0]["row"][3] else "") + "</td>"
	1391	f'<td class="num">{len(cs)}</td><td class="num">{fmt_tokens(sum(c["delta"][1] for c in cs))}</td>'
	1392	f'<td class="num">{fmt_tokens(sum(c["delta"][2] for c in cs))}</td>'
`799905ae` 2026-11-04 Pavel Safronov	1393	f'<td class="num">{100 * sum(c["delta"][1] for c in cs) / total_out:.0f}%</td>'
	1394	f'{session_cost_cell(cs)}</tr>'
`57d16c8b` 2026-10-29 Pavel Safronov	1395	for n, cs in sessions.items())
	1396	ranked = sorted(costs.values(), key=lambda c: -c["delta"][1])
`ced0eafa` 2026-11-04 Pavel Safronov	1397	buckets = week_buckets(usage["all"], costs)
`799905ae` 2026-11-04 Pavel Safronov	1398	charts = usage_charts(buckets, "u")
`ced0eafa` 2026-11-04 Pavel Safronov	1399	as_of = load_prices().get("as_of", "?")
`799905ae` 2026-11-04 Pavel Safronov	1400	stats = usage_stats([
	1401	("Commits with usage data", f'{len(costs)} of {usage["commits"]}'),
	1402	("Sessions", str(len(sessions))),
	1403	("Output tokens", f"~{fmt_tokens(total_out)}"),
	1404	("Cached reads", f"~{fmt_tokens(total_cached)}"),
	1405	cost_stat(usage["dollars"], usage["priced"], as_of)])
	1406	body = ('<h1>Claude Code usage</h1><p class="meta">Read from the <code>FW-Tokens-*</code> trailers in commit '
	1407	'messages (or older <code>Usage:</code> lines).</p>'
	1408	'<p class="meta"><a href="/usage/">Usage across all repositories</a></p>'
	1409	+ stats +
	1410	f'{charts}'
`57d16c8b` 2026-10-29 Pavel Safronov	1411	f'<h2>Commits by output tokens</h2>'
	1412	f'{commits_table(name, [c["row"] for c in ranked[:30]], commit_set, usage)}'
	1413	f'<h2>Sessions</h2><table class="log"><tr><th>#</th><th>Dates</th><th>Commits</th><th>Output</th>'
`799905ae` 2026-11-04 Pavel Safronov	1414	f'<th>Cached</th><th>Share of output</th><th>Est. cost (not what was paid)</th></tr>{session_rows}</table>'
`57d16c8b` 2026-10-29 Pavel Safronov	1415	'<h2>How this is measured</h2><ul>'
	1416	'<li>A trailer is the running token tally of one Claude Code session. A commit\'s cost is the step '
`ced0eafa` 2026-11-04 Pavel Safronov	1417	'from the previous commit\'s tally. A new session starts, and its first commit gets the whole tally, '
	1418	'at the first trailer, when the counting method changes, when the output or cached total falls, '
	1419	'when any model\'s counter falls, when a model that had tokens disappears, or when per-model '
	1420	'lines appear after commits that had none.</li>'
`799905ae` 2026-11-04 Pavel Safronov	1421	'<li>The bars and rankings in the commit table use output tokens. Cached reads grow with how long the context is and how many '
`57d16c8b` 2026-10-29 Pavel Safronov	1422	'turns ran, so they track session length more than the work; input tokens are negligible. These are '
`ced0eafa` 2026-11-04 Pavel Safronov	1423	'tokens; dollar figures are estimated API list-price equivalents, not what was paid.</li>'
	1424	'<li>Costs need the per-model trailers (<code>FW-Model-Tokens</code>) and are estimates at list prices '
	1425	f'as of {esc(as_of)}. Commits without them show cost unknown, and unknown is not zero: it is left out of '
	1426	'the cost total and drawn hatched in the chart.</li>'
	1427	'<li>Trailers without <code>FW-Tokens-Method: 2</code> counted duplicate transcript lines '
	1428	'(roughly 1.5-2× too high) and missed subagents; they are shown as written.</li>'
`57d16c8b` 2026-10-29 Pavel Safronov	1429	'<li>Tallies are rounded to a tenth of K, M or G, so a step is only known to within about two '
	1430	'such tenths. Steps at or under that show as ≈0.</li>'
	1431	'<li>A commit with no trailer shows no bar, and the work before it is counted in the next commit '
	1432	'that has one. The cost is of everything done since the previous trailer, including dead ends, '
	1433	'not only the code in the commit.</li>'
`799905ae` 2026-11-04 Pavel Safronov	1434	f'<li>Highlighted bars in the commit table: at least {USAGE_HOT}× the median commit.</li></ul>')
`57d16c8b` 2026-10-29 Pavel Safronov	1435	write(path, page(f"{name} usage", body, [f'<a href="{url(name)}/">{esc(name)}</a>', "usage"], view=view))
	1436
	1437
`1758548c` 2026-10-27 Pavel Safronov	1438	def build_history_page(view, path, commits, commit_set):
	1439	body = (f'{branch_note(view)}<h1>{esc(posixpath.basename(path))}</h1>'
	1440	f'<p class="meta">Commits that changed this file, newest first · '
	1441	f'<a href="{view.url("blob", path)}.html">file</a></p>'
	1442	+ (commits_table(view.name, commits, commit_set) if commits else
	1443	"<p>No recent commits changed this file (it may have been renamed or only changed long ago).</p>"))
	1444	write(view.path("history", path + ".html"), page(f"{view.name}/{path} history", body, crumbs_for(view, path), view=view))
	1445
	1446
`1cc63426` 2026-10-27 Pavel Safronov	1447	SHORT_SHA_LENGTHS = range(7, 13)
	1448
	1449
	1450	def short_sha_aliases(commits):
	1451	"""{alias: full sha} for 7–12 character prefixes, skipping any prefix two commits share.
	1452
	1453	Apps link to `commit/<git rev-parse --short>.html`, and the short length varies by clone.
	1454	"""
	1455	seen = {}
`1758548c` 2026-10-27 Pavel Safronov	1456	for full in commits:
`1cc63426` 2026-10-27 Pavel Safronov	1457	for n in SHORT_SHA_LENGTHS:
	1458	seen.setdefault(full[:n], set()).add(full)
	1459	return {alias: fulls.pop() for alias, fulls in seen.items() if len(fulls) == 1}
	1460
	1461
`1758548c` 2026-10-27 Pavel Safronov	1462	def write_short_sha_alias(out, view, alias, full):
`1cc63426` 2026-10-27 Pavel Safronov	1463	target = f"{full}.html"
	1464	body = f'<h1>Commit {esc(alias)}</h1><p>See <a href="{esc(target)}"><code>{esc(full)}</code></a>.</p>'
`1758548c` 2026-10-27 Pavel Safronov	1465	html_out = page(f"{view.name} {alias}", body, view=view).replace(
`1cc63426` 2026-10-27 Pavel Safronov	1466	"<head>\n", f'<head>\n<meta http-equiv="refresh" content="0; url={esc(target)}">\n', 1)
	1467	write(os.path.join(out, "commit", alias + ".html"), html_out)
	1468
	1469
`57d16c8b` 2026-10-29 Pavel Safronov	1470	def build_commit_page(repo, out, view, sha, force=False):
`623239b7` 2026-10-26 Pavel Safronov	1471	path = os.path.join(out, "commit", sha + ".html")
`57d16c8b` 2026-10-29 Pavel Safronov	1472	if os.path.exists(path) and not force:
`623239b7` 2026-10-26 Pavel Safronov	1473	return # commits never change
`1758548c` 2026-10-27 Pavel Safronov	1474	name = view.name
`623239b7` 2026-10-26 Pavel Safronov	1475	head = git_text(repo, "show", "-s", "--format=%H%n%an <%ae>%n%ad%n%P%n%B", "--date=iso", sha)
	1476	full, author, date, parents, *msg = head.split("\n")
	1477	author = author.split(" <")[0] # don't publish email addresses
	1478	diff = git(repo, "show", "--format=", "--stat", "--patch", "-M", "--diff-merges=first-parent", sha)
	1479	truncated = len(diff) > MAX_DIFF_BYTES
	1480	text = diff[:MAX_DIFF_BYTES].decode("utf-8", errors="replace")
	1481	lines = []
	1482	for l in text.split("\n"):
	1483	cls = ""
	1484	if l.startswith("+") and not l.startswith("+++"):
	1485	cls = "add"
	1486	elif l.startswith("-") and not l.startswith("---"):
	1487	cls = "del"
	1488	elif l.startswith("@@"):
	1489	cls = "hunk"
	1490	elif l.startswith("diff --git"):
	1491	cls = "file"
	1492	lines.append(f'<span class="{cls}">{esc(l)}</span>' if cls else esc(l))
	1493	parent_links = " ".join(f'<a href="{url(name, "commit", p)}.html"><code>{p[:12]}</code></a>' for p in parents.split())
	1494	body = (f"<h1>{esc(msg[0] if msg else full)}</h1>"
	1495	f'<p class="meta"><code>{full}</code> · {esc(author)} · {esc(date)}'
	1496	f'{" · parents " + parent_links if parent_links else ""}</p>'
`57d16c8b` 2026-10-29 Pavel Safronov	1497	f'{usage_panel(name, full)}'
`623239b7` 2026-10-26 Pavel Safronov	1498	f'<pre class="message">{esc(chr(10).join(msg).strip())}</pre>'
	1499	f'<pre class="diff">{chr(10).join(lines)}</pre>'
	1500	+ ("<p>Diff truncated. Clone the repository to see all of it.</p>" if truncated else ""))
`1758548c` 2026-10-27 Pavel Safronov	1501	write(path, page(f"{name} {full[:12]}", body, [f'<a href="{url(name)}/">{esc(name)}</a>', "commit"], view=view))
`623239b7` 2026-10-26 Pavel Safronov	1502
	1503
	1504	def refs_rows(repo, pattern, datefield):
`1758548c` 2026-10-27 Pavel Safronov	1505	fmt = f"%(refname:short)%1f%({datefield}:short)%1f%(subject)%1f%(objectname)"
`623239b7` 2026-10-26 Pavel Safronov	1506	raw = git_text(repo, "for-each-ref", f"--sort=-{datefield}", f"--format={fmt}", pattern, check=False)
`1758548c` 2026-10-27 Pavel Safronov	1507	return [l.split("\x1f") for l in raw.splitlines() if l.count("\x1f") == 3]
`623239b7` 2026-10-26 Pavel Safronov	1508
	1509
`1758548c` 2026-10-27 Pavel Safronov	1510	def build_refs_page(repo, view, branch_views, commit_set):
`623239b7` 2026-10-26 Pavel Safronov	1511	branches = refs_rows(repo, "refs/heads", "committerdate")
	1512	tags = refs_rows(repo, "refs/tags", "creatordate")
`1758548c` 2026-10-27 Pavel Safronov	1513	def branch_cell(n, sha):
	1514	v = branch_views.get(n)
	1515	label = f"<code>{esc(n)}</code>"
	1516	if v:
	1517	return f'<a href="{v.url("tree")}/">{label}</a>{" <em>(default)</em>" if v.is_default else ""}'
	1518	if sha in commit_set: # merged into the default branch: no pages of its own
	1519	return f'{label} <em>(merged, <a href="{url(view.name, "commit", sha)}.html">last commit</a>)</em>'
	1520	return label
	1521	def table(rows, cell):
`623239b7` 2026-10-26 Pavel Safronov	1522	if not rows:
	1523	return "<p>None.</p>"
	1524	return '<table class="log">' + "".join(
`1758548c` 2026-10-27 Pavel Safronov	1525	f'<tr><td>{cell(n, h)}</td><td class="date">{esc(d)}</td><td>{esc(s)}</td></tr>' for n, d, s, h in rows) + "</table>"
	1526	body = (f"<h1>Branches</h1>{table(branches, branch_cell)}<h1>Tags</h1>{table(tags, lambda n, h: f'<code>{esc(n)}</code>')}"
	1527	f"<p>Tags can be cloned: <code>git clone --branch <tag> {BASE_URL}/{esc(view.name)}.git</code></p>")
	1528	write(os.path.join(SITE, view.name, "refs", "index.html"), page(f"{view.name} branches & tags", body, view=view))
	1529
	1530
	1531	def app_shots(app, repo=None):
	1532	"""[(url, label)]: the screenshots a CI runner took of the app; otherwise (public repos)
	1533	the images in the repo's screenshots/ folder."""
	1534	try:
	1535	with open(os.path.join(APPS, SHOTS, app, "shots.json")) as f:
	1536	listed = json.load(f).get("shots", [])
	1537	found = [(f"{APPS_URL}/{SHOTS}/{urllib.parse.quote(app)}/{urllib.parse.quote(s['file'])}", s.get("label", ""))
	1538	for s in listed if isinstance(s, dict) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*\.png", s.get("file", ""))
	1539	and os.path.exists(os.path.join(APPS, SHOTS, app, s["file"]))]
	1540	if found:
	1541	return found
	1542	except (OSError, ValueError, AttributeError):
	1543	pass
	1544	d = os.path.join(SITE, repo or "", "raw", "screenshots")
	1545	if repo and os.path.isdir(d):
	1546	return [(f"{BASE_URL}{url(repo, 'raw', 'screenshots', f)}", os.path.splitext(f)[0])
	1547	for f in sorted(os.listdir(d)) if f.lower().endswith(SHOT_EXTENSIONS)][:4]
	1548	return []
`623239b7` 2026-10-26 Pavel Safronov	1549
	1550
	1551	def app_url(name):
	1552	return f"{APPS_URL}/{name}/" if os.path.exists(os.path.join(APPS, name, "index.html")) else None
	1553
	1554
`1758548c` 2026-10-27 Pavel Safronov	1555	LICENSE_NAMES = ["license", "license.md", "license.txt", "licence", "licence.md", "licence.txt", "copying", "copying.md", "copying.txt"]
	1556	# (SPDX id, phrases that must all appear), checked in order; first match wins.
	1557	LICENSE_PATTERNS = [
	1558	("AGPL-3.0", ["gnu affero general public license"]),
	1559	("LGPL-3.0", ["gnu lesser general public license", "version 3"]),
	1560	("LGPL-2.1", ["gnu lesser general public license", "version 2.1"]),
	1561	("GPL-3.0", ["gnu general public license", "version 3"]),
	1562	("GPL-2.0", ["gnu general public license", "version 2"]),
	1563	("Apache-2.0", ["apache license", "version 2.0"]),
	1564	("MPL-2.0", ["mozilla public license", "2.0"]),
	1565	("MIT", ["permission is hereby granted, free of charge"]),
	1566	("ISC", ["permission to use, copy, modify, and/or distribute this software for any purpose"]),
	1567	("BSD-3-Clause", ["redistribution and use in source and binary forms", "neither the name"]),
	1568	("BSD-2-Clause", ["redistribution and use in source and binary forms"]),
	1569	("Unlicense", ["this is free and unencumbered software released into the public domain"]),
	1570	("CC0-1.0", ["cc0 1.0 universal"]),
	1571	]
	1572
	1573
	1574	def detect_license(files, blobs):
	1575	"""(license name, path of the license file) from a root LICENSE/COPYING file or package.json, or None."""
	1576	lower = {p.lower(): p for p in files if "/" not in p}
	1577	for cand in LICENSE_NAMES:
	1578	if cand in lower:
	1579	p = lower[cand]
	1580	text = " ".join(blobs.read(files[p][1])[:20000].decode("utf-8", errors="replace").lower().split())
	1581	for spdx, phrases in LICENSE_PATTERNS:
	1582	if all(ph in text for ph in phrases):
	1583	return spdx, p
	1584	return "see license file", p
	1585	if "package.json" in files:
	1586	try:
	1587	lic = json.loads(blobs.read(files["package.json"][1])).get("license")
	1588	except (ValueError, AttributeError):
	1589	lic = None
	1590	if isinstance(lic, str) and lic.strip() and lic.strip().upper() != "UNLICENSED":
	1591	return lic.strip(), "package.json"
	1592	return None
	1593
	1594
	1595	def build_summary(repo, view, desc, files, ctx, commits, empty, commit_set):
	1596	name = view.name
	1597	clone_url = f"{BASE_URL}/{name}.git"
`623239b7` 2026-10-26 Pavel Safronov	1598	app = app_url(name)
	1599	links = f'<p class="meta">App: <a href="{app}">{esc(app)}</a></p>' if app else ""
`b77f572b` 2026-10-27 Pavel Safronov	1600	contrib, bugs = f"contrib-68uq+{name}@fuzzyworld.net", f"bugs-k7q4+{name}@fuzzyworld.net"
	1601	links += (f'<p class="meta">Contribute: <a href="mailto:{esc(contrib)}">{esc(contrib)}</a>'
`55e5fc3e` 2026-11-04 Pavel Safronov	1602	f' (<a href="/contribute/">how</a>) · Report a bug: <a href="mailto:{esc(bugs)}">{esc(bugs)}</a>'
	1603	+ (f' · <a href="{url(name, "issues")}/">Issues</a>' if os.path.isfile(os.path.join(SITE, name, "issues", "index.html")) else "")
	1604	+ "</p>")
`1758548c` 2026-10-27 Pavel Safronov	1605	readme_html, license_html = "", ""
`623239b7` 2026-10-26 Pavel Safronov	1606	if not empty:
`1758548c` 2026-10-27 Pavel Safronov	1607	blobs = BlobReader(repo)
	1608	lic = detect_license(files, blobs)
	1609	if lic:
	1610	license_html = f' · License: <a href="{view.url("blob", lic[1])}.html">{esc(lic[0])}</a>'
`623239b7` 2026-10-26 Pavel Safronov	1611	lower = {p.lower(): p for p in files if "/" not in p}
	1612	for cand in README_NAMES:
	1613	if cand in lower:
	1614	p = lower[cand]
	1615	text = blobs.read(files[p][1]).decode("utf-8", errors="replace")
	1616	ctx.base_dir = ""
	1617	rendered = render_markdown(text, ctx) if p.lower().endswith((".md", ".markdown")) else f"<pre>{esc(text)}</pre>"
	1618	readme_html = f'<article class="markdown readme">{rendered}</article>'
	1619	break
`1758548c` 2026-10-27 Pavel Safronov	1620	blobs.close()
	1621	# "Open in VS Code" clones into the visitor's own VS Code; "in the browser" opens our static VS Code web build.
`eeffb145` 2026-10-28 Pavel Safronov	1622	vscode = f'<a href="vscode://vscode.git/clone?url={urllib.parse.quote(clone_url, safe="")}" class="vscode-link">Open in VS Code</a>'
`1758548c` 2026-10-27 Pavel Safronov	1623	if not empty and os.path.exists(os.path.join(APPS, VSCODE_APP, "index.html")):
`eeffb145` 2026-10-28 Pavel Safronov	1624	vscode += f' · <a href="{APPS_URL}/{VSCODE_APP}/?repo={urllib.parse.quote(name)}" class="vscode-link">Open in VS Code in the browser</a>'
`1758548c` 2026-10-27 Pavel Safronov	1625	recent = commits_table(name, commits[:5], commit_set) if commits else ""
	1626	shots = app_shots(name, name) if app else []
	1627	shots_html = ('<p class="shots">' + "".join(f'<a href="{app}"><img src="{esc(u)}" alt="{esc(l)}" loading="lazy"></a>' for u, l in shots[:4])
	1628	+ "</p>") if shots else ""
`623239b7` 2026-10-26 Pavel Safronov	1629	body = (f"<h1>{esc(name)}</h1>"
	1630	+ (f'<p class="desc">{esc(desc)}</p>' if desc else "")
`1758548c` 2026-10-27 Pavel Safronov	1631	+ links + shots_html
	1632	+ f'<pre class="clone">git clone {esc(clone_url)}</pre>'
	1633	+ f'<p class="meta">{vscode}{license_html}{ci_badge(name, commits[0][0]) if commits else ""}</p>'
`623239b7` 2026-10-26 Pavel Safronov	1634	+ ("<p>This repository is empty.</p>" if empty else f"<h2>Recent commits</h2>{recent}<p><a href=\"{url(name, 'log')}/\">Full log</a></p>")
	1635	+ readme_html)
`1758548c` 2026-10-27 Pavel Safronov	1636	write(os.path.join(SITE, name, "index.html"), page(name, body, view=None if empty else view))
`623239b7` 2026-10-26 Pavel Safronov	1637
	1638
`1758548c` 2026-10-27 Pavel Safronov	1639	def is_merged(repo, branch):
	1640	r = subprocess.run(["git", "-C", repo, "merge-base", "--is-ancestor", "refs/heads/" + branch, "HEAD"], capture_output=True)
	1641	return r.returncode == 0
`623239b7` 2026-10-26 Pavel Safronov	1642
`1758548c` 2026-10-27 Pavel Safronov	1643
	1644	def build_view(repo, view, vstate, commit_set):
	1645	"""Tree, file, history, blame and log pages for one branch. Returns the view's new state."""
	1646	files = tree_entries(repo, view.ref)
`623239b7` 2026-10-26 Pavel Safronov	1647	dirs = all_dirs(files)
	1648	raw_set = {p for p, (m, s, size) in files.items()
	1649	if 0 <= size <= MAX_RAW_BYTES and posixpath.splitext(p)[1].lower() in RAW_EXTENSIONS}
`1758548c` 2026-10-27 Pavel Safronov	1650	ctx = LinkContext(view, "", set(files), dirs, raw_set)
	1651	commits = commit_list(repo, view.ref, COMMIT_LIMIT)
	1652	head = commits[0][0] if commits else None
	1653	head_changed = vstate.get("head") != head
`623239b7` 2026-10-26 Pavel Safronov	1654
`1758548c` 2026-10-27 Pavel Safronov	1655	old_files = vstate.get("files", {})
`623239b7` 2026-10-26 Pavel Safronov	1656	for p in set(old_files) - set(files):
`1758548c` 2026-10-27 Pavel Safronov	1657	for kind, suffix in (("blob", ".html"), ("blame", ".html"), ("history", ".html"), ("raw", "")):
	1658	remove(view.path(kind, p + suffix))
	1659	for d in set(vstate.get("dirs", [])) - dirs:
	1660	remove(view.path("tree", d, "index.html"))
`623239b7` 2026-10-26 Pavel Safronov	1661
`1758548c` 2026-10-27 Pavel Safronov	1662	# File and blame pages and raw copies: only where the blob changed. Markdown pages are
	1663	# rebuilt whenever the branch moves, because their links depend on the whole tree.
`623239b7` 2026-10-26 Pavel Safronov	1664	blobs = BlobReader(repo)
	1665	written = 0
	1666	for p, entry in files.items():
	1667	is_md = p.lower().endswith((".md", ".markdown"))
	1668	if old_files.get(p) == entry[1] and not (is_md and head_changed):
	1669	continue
	1670	try:
`1758548c` 2026-10-27 Pavel Safronov	1671	build_blob_page(repo, view, p, entry, blobs, ctx, raw_set, commit_set)
`623239b7` 2026-10-26 Pavel Safronov	1672	if p in raw_set:
`1758548c` 2026-10-27 Pavel Safronov	1673	write(view.path("raw", p), blobs.read(entry[1]))
`623239b7` 2026-10-26 Pavel Safronov	1674	else:
`1758548c` 2026-10-27 Pavel Safronov	1675	remove(view.path("raw", p))
`623239b7` 2026-10-26 Pavel Safronov	1676	written += 1
	1677	except OSError as e: # e.g. a path component too long for the filesystem
	1678	print(f"skip {p}: {e}", file=sys.stderr)
	1679	blobs.close()
	1680
`1758548c` 2026-10-27 Pavel Safronov	1681	# History pages: only where the newest commit touching the file changed.
	1682	hist = file_histories(repo, view.ref, set(files)) if head_changed or not vstate.get("hist") else None
	1683	new_hist = vstate.get("hist", {}) if hist is None else {}
	1684	if hist is not None:
	1685	old_hist = vstate.get("hist", {})
	1686	for p in files:
	1687	commits_p = hist.get(p, [])
	1688	newest = commits_p[0][0] if commits_p else ""
	1689	new_hist[p] = newest
	1690	if old_hist.get(p) != newest or not os.path.exists(view.path("history", p + ".html")):
	1691	try:
	1692	build_history_page(view, p, commits_p, commit_set)
	1693	except OSError as e:
	1694	print(f"skip history of {p}: {e}", file=sys.stderr)
	1695
	1696	# For VS Code in the browser (vscode-web/): every file's blob SHA and size. The contents are
	1697	# objects/<sha>, written by build_repo for files up to MAX_TEXT_BYTES.
	1698	write(view.path("files.json"), json.dumps({"repo": view.name, "branch": view.branch, "commit": head,
	1699	"maxBytes": MAX_TEXT_BYTES,
	1700	"files": {p: [e[1], e[2]] for p, e in sorted(files.items())}}))
	1701	if files:
	1702	build_tree_page(view, "", files, dirs)
`623239b7` 2026-10-26 Pavel Safronov	1703	for d in dirs:
`1758548c` 2026-10-27 Pavel Safronov	1704	build_tree_page(view, d, files, dirs)
	1705	log_title = f"Last {len(commits)} commits" + ("" if view.is_default else f" on {esc(view.branch)}")
`57d16c8b` 2026-10-29 Pavel Safronov	1706	usage = repo_usage(view.name)
	1707	usage_note = (f'<p class="meta">Bars: Claude Code output tokens per commit, highlighted at {USAGE_HOT}× the median or more · '
	1708	f'<a href="{url(view.name, "usage")}/">details</a></p>' if usage["costs"] else "")
`1758548c` 2026-10-27 Pavel Safronov	1709	write(view.path("log", "index.html"),
`57d16c8b` 2026-10-29 Pavel Safronov	1710	page(f"{view.name} log", f"{branch_note(view)}<h1>{log_title}</h1>{usage_note}"
	1711	f"{commits_table(view.name, commits, commit_set, usage if usage['costs'] else None)}", view=view))
`1758548c` 2026-10-27 Pavel Safronov	1712	print(f"{view.name} [{view.branch}]: {len(files)} files, {written} file pages written, {len(commits)} commits")
	1713	return {"head": head, "files": {p: e[1] for p, e in files.items()}, "dirs": sorted(dirs), "hist": new_hist,
	1714	"_files": files, "_ctx": ctx, "_commits": commits}
	1715
	1716
	1717	def build_repo(name):
	1718	repo = os.path.join(GIT_ROOT, name + ".git")
	1719	if not os.path.isdir(repo):
	1720	raise SystemExit(f"no such repo: {name}")
	1721	if not is_public(repo):
	1722	raise SystemExit(f"{name} is not public; refusing to publish")
	1723	out = os.path.join(SITE, name)
	1724	os.makedirs(STATE, exist_ok=True)
	1725	state_path = os.path.join(STATE, name + ".json")
	1726	try:
	1727	with open(state_path) as f:
	1728	state = json.load(f)
	1729	except (FileNotFoundError, ValueError):
	1730	state = {}
`ced0eafa` 2026-11-04 Pavel Safronov	1731	stale = state.get("version") != gen_key()
`57d16c8b` 2026-10-29 Pavel Safronov	1732	if stale:
`1758548c` 2026-10-27 Pavel Safronov	1733	state = {}
	1734
	1735	desc = repo_config(repo, "fuzzyworld.description")
	1736	default_branch = git_text(repo, "symbolic-ref", "--short", "HEAD", check=False).strip()
	1737	branch_names = [b for b in git_text(repo, "for-each-ref", "--format=%(refname:short)", "refs/heads", check=False).split()
	1738	if safe_path(b)]
	1739	empty = default_branch not in branch_names
	1740	# Browse pages for the default branch and for branches not yet merged into it. A merged
	1741	# branch adds nothing new to read, and would duplicate every page.
	1742	views = {b: View(name, b, b == default_branch) for b in branch_names
	1743	if b == default_branch or empty or not is_merged(repo, b)}
	1744	default_view = views.get(default_branch) or View(name, default_branch or "main", True)
	1745
	1746	# Commit pages for every commit on every branch (up to COMMIT_LIMIT), shared by all branches.
	1747	commit_shas = git_text(repo, "rev-list", f"-n{COMMIT_LIMIT}", "--branches", check=False).split()
	1748	commit_set = set(commit_shas)
	1749	for h in set(state.get("commits", [])) - commit_set:
	1750	remove(os.path.join(out, "commit", h + ".html"))
	1751	aliases = short_sha_aliases(commit_shas)
	1752	for a in set(state.get("aliases", [])) - set(aliases):
	1753	remove(os.path.join(out, "commit", a + ".html"))
	1754	for h in commit_shas:
`57d16c8b` 2026-10-29 Pavel Safronov	1755	build_commit_page(repo, out, default_view, h, force=stale)
`1758548c` 2026-10-27 Pavel Safronov	1756	for alias, full in aliases.items():
	1757	write_short_sha_alias(out, default_view, alias, full)
	1758
	1759	# Branches that are gone lose their whole directory.
	1760	old_views = state.get("views", {})
	1761	keep = {v.key for v in views.values()}
	1762	branch_root = os.path.join(out, "branch")
	1763	if os.path.isdir(branch_root):
	1764	for d in os.listdir(branch_root):
	1765	if f"branch/{d}" not in keep:
	1766	shutil.rmtree(os.path.join(branch_root, d), ignore_errors=True)
	1767	if empty:
	1768	remove(os.path.join(out, "files.json"))
	1769	for kind in ("tree", "blob", "blame", "history", "raw", "log", "objects"):
	1770	shutil.rmtree(os.path.join(out, kind), ignore_errors=True)
	1771
	1772	new_views, built = {}, {}
	1773	for b, v in views.items():
	1774	built[b] = build_view(repo, v, old_views.get(v.key, {}), commit_set)
	1775	new_views[v.key] = {k: val for k, val in built[b].items() if not k.startswith("_")}
	1776
	1777	# objects/<sha>: file contents by blob SHA (served as downloads, never as pages), for VS Code in the browser.
	1778	wanted = {e[1] for b in built.values() for e in b["_files"].values() if 0 <= e[2] <= MAX_TEXT_BYTES}
	1779	objects = os.path.join(out, "objects")
	1780	have = set(os.listdir(objects)) if os.path.isdir(objects) else set()
	1781	for sha in have - wanted:
	1782	remove(os.path.join(objects, sha))
	1783	if wanted - have:
	1784	blobs = BlobReader(repo)
	1785	for sha in wanted - have:
	1786	write(os.path.join(objects, sha), blobs.read(sha))
	1787	blobs.close()
	1788
	1789	d = built.get(default_branch, {})
	1790	if not empty:
	1791	build_refs_page(repo, default_view, views, commit_set)
`57d16c8b` 2026-10-29 Pavel Safronov	1792	build_usage_page(default_view, commit_set)
`799905ae` 2026-11-04 Pavel Safronov	1793	if is_public(repo):
	1794	usage_summary(name, save=True) # the site-wide /usage/ page reads this instead of re-running git log
`1758548c` 2026-10-27 Pavel Safronov	1795	build_ci_pages(default_view)
	1796	build_summary(repo, default_view, desc, d.get("_files", {}), d.get("_ctx"), d.get("_commits", []), empty, commit_set)
	1797
`ced0eafa` 2026-11-04 Pavel Safronov	1798	state = {"version": gen_key(), "views": new_views, "commits": commit_shas,
`1758548c` 2026-10-27 Pavel Safronov	1799	"aliases": sorted(aliases), "built": int(time.time())}
`623239b7` 2026-10-26 Pavel Safronov	1800	write(state_path, json.dumps(state))
`1758548c` 2026-10-27 Pavel Safronov	1801	print(f"{name}: {len(views)} branches, {len(commit_shas)} commits")
`623239b7` 2026-10-26 Pavel Safronov	1802
	1803
`799905ae` 2026-11-04 Pavel Safronov	1804	# Top-level paths of the code site that are not repos. A repo with one of these names would be overwritten by
	1805	# (or overwrite) that page, so the roll-up and index skip it. (lib/admin.py RESERVED_REPOS is for `fw issues`: a different list.)
	1806	SITE_RESERVED = {"usage", "ci", "projects", "why-i-left-gitlab", "my-ai-use", "contribute", "why", "principles"}
	1807
	1808
	1809	def published_public_repos():
	1810	"""[(name, git dir)] of public repos that already have a published summary page, sorted by name.
	1811	Reserved top-level names (SITE_RESERVED) are never repos."""
	1812	found = []
`623239b7` 2026-10-26 Pavel Safronov	1813	for entry in sorted(os.listdir(GIT_ROOT)):
`799905ae` 2026-11-04 Pavel Safronov	1814	if not entry.endswith(".git") or entry[:-4] in SITE_RESERVED:
`623239b7` 2026-10-26 Pavel Safronov	1815	continue
	1816	repo = os.path.join(GIT_ROOT, entry)
`799905ae` 2026-11-04 Pavel Safronov	1817	if is_public(repo) and os.path.exists(os.path.join(SITE, entry[:-4], "index.html")):
	1818	found.append((entry[:-4], repo))
	1819	return found
	1820
	1821
	1822	def usage_summary_rows(u):
	1823	"""Compact per-commit rows [date, ai, out, read, cost] (out/read None = no usage data, cost None = unknown)
	1824	from a repo_usage result: all that the site-wide roll-up needs."""
	1825	rows = []
	1826	for c in u["all"]:
	1827	e = u["costs"].get(c["sha"])
	1828	rows.append([c["date"], int(c["ai"]), e["delta"][1] if e else None, e["delta"][2] if e else None,
	1829	e["cost"] if e else None])
	1830	return rows
	1831
	1832
	1833	def usage_summary(name, save=False):
	1834	"""{"head", "key", "rows"} for a repo: read from STATE/<name>.usage.json when its HEAD sha and gen_key() still
	1835	match, else computed from git (via repo_usage) and, when `save` or the file was missing or stale, written.
	1836	A missing, corrupt or stale file is simply recomputed."""
	1837	repo = os.path.join(GIT_ROOT, name + ".git")
	1838	head = git_text(repo, "rev-parse", "HEAD", check=False).strip()
	1839	path = os.path.join(STATE, name + ".usage.json")
	1840	if not save:
	1841	try:
	1842	with open(path) as f:
	1843	cached = json.load(f)
	1844	if (head and cached["head"] == head and cached["key"] == gen_key()
	1845	and all(isinstance(r, list) and len(r) == 5 for r in cached["rows"])):
	1846	return cached
	1847	except (OSError, ValueError, KeyError, TypeError):
	1848	pass
	1849	summary = {"head": head, "key": gen_key(), "rows": usage_summary_rows(repo_usage(name))}
	1850	try:
	1851	os.makedirs(STATE, exist_ok=True)
	1852	write(path, json.dumps(summary))
	1853	except OSError:
	1854	pass
	1855	return summary
	1856
	1857
	1858	def site_usage_totals(names, today=None):
	1859	"""(buckets, rows, start) over the named repos. buckets: week_buckets of every repo's commits together, from
	1860	`start`, the first commit with an AI marker or usage data (dates before 2000 or after tomorrow are ignored
	1861	entirely; start is None if there is none). rows: per repo name, commits, with_data, ai, out, dollars, priced,
	1862	has_page; sorted by output tokens, then name."""
	1863	today = today or datetime.date.today()
	1864	lo, hi = datetime.date(2000, 1, 1).isoformat(), (today + datetime.timedelta(days=1)).isoformat()
	1865	allc, costs, rows = [], {}, []
	1866	for name in names:
	1867	data = usage_summary(name)["rows"]
	1868	with_data = [r for r in data if r[2] is not None]
	1869	known = [r[4] for r in data if r[4] is not None]
	1870	for i, (date, ai, out, read, cost) in enumerate(data):
	1871	if not lo <= date <= hi:
	1872	continue
	1873	sha = f"{name}:{i}"
	1874	allc.append({"sha": sha, "date": date, "ai": bool(ai), "has_tokens": out is not None})
	1875	if out is not None:
	1876	costs[sha] = {"delta": [0, out, read], "cost": cost}
	1877	rows.append({"name": name, "commits": len(data), "with_data": len(with_data), "ai": sum(r[1] for r in data),
	1878	"out": sum(r[2] for r in with_data), "dollars": sum(known), "priced": len(known),
	1879	"has_page": bool(with_data)})
	1880	rows.sort(key=lambda r: (-r["out"], r["name"]))
	1881	start = min((c["date"] for c in allc if c["ai"]), default=None)
	1882	allc = [c for c in allc if start and c["date"] >= start]
	1883	return week_buckets(allc, costs), rows, start
	1884
	1885
	1886	def build_site_usage():
	1887	"""code.fuzzyworld.net/usage/: the Usage charts summed over public published repos, and a table per repo."""
	1888	repos = published_public_repos()
	1889	buckets, rows, start = site_usage_totals([n for n, _ in repos])
	1890	def cost_cell(r):
	1891	if not r["priced"]:
	1892	return '<td class="num" title="no commit in this repo has a known cost">unknown</td>'
	1893	part = "" if r["priced"] == r["with_data"] else f" ({r['priced']} of {r['with_data']} commits)"
	1894	return (f'<td class="num" title="estimated API list-price equivalent, not what was paid">'
	1895	f'est. ${r["dollars"]:,.2f}{part}</td>')
	1896	table_rows = "".join(
	1897	'<tr><td>' + (f'<a href="{url(r["name"], "usage")}/">{esc(r["name"])}</a>' if r["has_page"] else esc(r["name"]))
	1898	+ f'</td><td class="num">{r["commits"]}</td><td class="num">{r["with_data"]}</td>'
	1899	+ (f'<td class="num">{100 * r["ai"] / r["commits"]:.0f}%</td>' if r["commits"] else '<td class="num">n/a</td>')
	1900	+ f'<td class="num">{fmt_tokens(r["out"]) if r["with_data"] else "unknown"}</td>{cost_cell(r)}</tr>'
	1901	for r in rows)
	1902	if rows:
	1903	table = ('<h2>By repository</h2><table class="log"><tr><th>Repository</th><th>Commits</th>'
	1904	'<th>Commits with usage data</th><th>AI-marked commits</th><th>Output tokens</th>'
	1905	f'<th>Est. cost (not what was paid)</th></tr>{table_rows}</table>')
	1906	else:
	1907	table = '<p class="meta">No public repository is published yet.</p>'
	1908	with_data = sum(r["with_data"] for r in rows)
	1909	stats = usage_stats([
	1910	("Repositories", str(len(rows))),
	1911	("Commits with usage data", f'{with_data} of {sum(r["commits"] for r in rows)}'),
	1912	("Output tokens", f'~{fmt_tokens(sum(r["out"] for r in rows))}' if any(r["with_data"] for r in rows) else "unknown, not zero (no commit has usage data)"),
	1913	cost_stat(sum(r["dollars"] for r in rows), sum(r["priced"] for r in rows), load_prices().get("as_of", "?"))])
	1914	body = ('<h1>Claude Code usage, all repositories</h1><p class="meta">Summed over the public repositories, read '
	1915	'from the <code>FW-Tokens-*</code> trailers in their commit messages (or older <code>Usage:</code> lines). '
	1916	'Unknown is never counted as zero; each repository\'s own usage page has the per-commit detail.</p>'
	1917	+ stats
	1918	+ (f'<p class="meta">Starts at the first AI-marked commit, {esc(start)}.</p>' if start else "")
	1919	+ usage_charts(buckets, "s") + table)
	1920	write(os.path.join(SITE, "usage", "index.html"),
	1921	page("Claude Code usage, all repositories", body, ["usage"], section="usage"))
	1922	print(f"site usage: {len(rows)} repos, {len(buckets)} buckets")
	1923
	1924
`392edf54` 2026-11-04 Pavel Safronov	1925	def last_commit_time(repo):
	1926	"""Unix time of the newest commit on the repo's HEAD, or 0 (empty or unreadable repo)."""
	1927	last = git_text(repo, "log", "-1", "--format=%ct", "HEAD", check=False).strip()
	1928	return int(last) if last.isdigit() else 0
	1929
	1930
`799905ae` 2026-11-04 Pavel Safronov	1931	def build_index():
`392edf54` 2026-11-04 Pavel Safronov	1932	"""code.fuzzyworld.net/index.html: a table of public repos, most recently committed first."""
`799905ae` 2026-11-04 Pavel Safronov	1933	build_ci_dashboard()
`392edf54` 2026-11-04 Pavel Safronov	1934	rows = [(last_commit_time(repo), name, repo_config(repo, "fuzzyworld.description"), app_url(name))
	1935	for name, repo in published_public_repos()]
	1936	rows.sort(key=lambda r: (-r[0], r[1]))
`623239b7` 2026-10-26 Pavel Safronov	1937	items = []
`392edf54` 2026-11-04 Pavel Safronov	1938	for t, n, d, a in rows:
	1939	app = f'<a href="{esc(a)}" class="icon-text"><span class="icon icon-app"></span>app</a>' if a else ""
	1940	when = time.strftime("%Y-%m-%d", time.gmtime(t)) if t else "never (empty)"
	1941	items.append(f'<tr><td><a class="name" href="{url(n)}/">{esc(n)}</a></td><td>{esc(d or "")}</td>'
	1942	f'<td class="num">{when}</td><td>{app}</td></tr>')
`623239b7` 2026-10-26 Pavel Safronov	1943	body = (f"<h1>Pavel's code</h1>"
	1944	f"<p>Browse, or clone with <code>git clone {BASE_URL}/<repo>.git</code>. No account or sign-in needed. "
`57d16c8b` 2026-10-29 Pavel Safronov	1945	f'Small web apps are at <a href="{APPS_URL}/">apps.fuzzyworld.net</a>. <br /> <br />'
`799905ae` 2026-11-04 Pavel Safronov	1946	f'<a href="/usage/">Claude Code usage across all repositories</a><br /> <br />'
`57d16c8b` 2026-10-29 Pavel Safronov	1947	f'<a href="/why-i-left-gitlab/">Why aren\'t those GitHub/GitLab/Codeberg links? (Why I left GitLab)</a></p>'
`392edf54` 2026-11-04 Pavel Safronov	1948	f'<p class="meta">Newest commit first.</p>'
	1949	f'<table class="log repos"><tr><th>Repository</th><th>Description</th><th>Last commit</th><th>App</th></tr>'
	1950	f'{"".join(items)}</table>')
`623239b7` 2026-10-26 Pavel Safronov	1951	write(os.path.join(SITE, "index.html"), page("Repositories", body))
`799905ae` 2026-11-04 Pavel Safronov	1952	build_site_usage()
`623239b7` 2026-10-26 Pavel Safronov	1953	print(f"index: {len(rows)} public repos")
	1954
	1955
`f9ca0752` 2026-10-27 Pavel Safronov	1956	def build_apps_index():
`57d16c8b` 2026-10-29 Pavel Safronov	1957	"""apps.fuzzyworld.net/index.html from each deployed app's .fuzzyworld.json, newest commit first."""
`f9ca0752` 2026-10-27 Pavel Safronov	1958	apps = []
	1959	for entry in sorted(os.listdir(APPS)):
	1960	meta_path = os.path.join(APPS, entry, ".fuzzyworld.json")
	1961	if not os.path.isfile(meta_path):
	1962	continue
	1963	try:
	1964	with open(meta_path) as f:
	1965	meta = json.load(f)
	1966	except ValueError:
	1967	continue
`1758548c` 2026-10-27 Pavel Safronov	1968	apps.append({"app": entry, "href": f"/{entry}/", "title": meta.get("title") or entry,
	1969	"description": meta.get("description", ""), "repo": meta.get("repo", entry),
	1970	"commit": meta.get("commit", ""), "deployed": meta.get("deployed", "")})
	1971	for a in apps:
`57d16c8b` 2026-10-29 Pavel Safronov	1972	# Use commit date from the repo instead of deploy timestamp for consistent sorting
	1973	repo = os.path.join(GIT_ROOT, a["repo"] + ".git")
	1974	if a["repo"] and os.path.isdir(repo):
`392edf54` 2026-11-04 Pavel Safronov	1975	a["time"] = a["commit_time"] = last_commit_time(repo)
`57d16c8b` 2026-10-29 Pavel Safronov	1976	else:
	1977	# Fallback to deploy timestamp if repo is not accessible
	1978	try:
	1979	a["time"] = datetime.datetime.fromisoformat(a["deployed"].replace("Z", "+00:00")).timestamp()
	1980	except ValueError:
	1981	a["time"] = 0
`f9ca0752` 2026-10-27 Pavel Safronov	1982	# Apps hosted elsewhere (e.g. engye on Cloudflare Pages), listed in server/site-apps/extra.json.
`1758548c` 2026-10-27 Pavel Safronov	1983	# They have no deploy record, so their repo's last commit stands in for the update time.
`f9ca0752` 2026-10-27 Pavel Safronov	1984	extra = os.path.join(os.path.dirname(os.path.abspath(__file__)), "site-apps", "extra.json")
	1985	if os.path.isfile(extra):
	1986	with open(extra) as f:
`1758548c` 2026-10-27 Pavel Safronov	1987	for a in json.load(f):
	1988	repo = os.path.join(GIT_ROOT, a.get("repo", "") + ".git")
`392edf54` 2026-11-04 Pavel Safronov	1989	t = last_commit_time(repo) if a.get("repo") and os.path.isdir(repo) else 0
	1990	apps.append({**a, "time": t, "commit_time": t})
	1991	apps.sort(key=lambda a: (-a["time"], a["title"].lower())) # newest code commit first, not newest deploy
`f9ca0752` 2026-10-27 Pavel Safronov	1992	items = []
	1993	for a in apps:
	1994	repo = os.path.join(GIT_ROOT, a.get("repo", "") + ".git")
`1758548c` 2026-10-27 Pavel Safronov	1995	public = a.get("repo") and os.path.isdir(repo) and is_public(repo)
`392edf54` 2026-11-04 Pavel Safronov	1996	src = f'<a href="{BASE_URL}/{urllib.parse.quote(a["repo"])}/" class="icon-text"><span class="icon icon-code"></span>source</a>' if public else ""
	1997	# "Last commit" is blank when the repo is unreadable: "time" then holds the deploy time, only for ordering.
	1998	when = time.strftime("%Y-%m-%d", time.gmtime(a["commit_time"])) if a.get("commit_time") else ""
	1999	deployed = esc((a.get("deployed") or "")[:10])
`1758548c` 2026-10-27 Pavel Safronov	2000	shots = app_shots(a["app"], a["repo"] if public else None) if a.get("app") else []
	2001	thumb = (f'<a class="thumb" href="{esc(a["href"])}"><img src="{esc(shots[0][0])}" alt="" loading="lazy"></a>'
	2002	if shots else "")
`392edf54` 2026-11-04 Pavel Safronov	2003	items.append(f'<tr><td class="shot">{thumb}</td><td><a class="name" href="{esc(a["href"])}">{esc(a["title"])}</a></td>'
	2004	f'<td>{esc(a.get("description") or "")}</td><td class="num">{when}</td>'
	2005	f'<td class="num">{deployed}</td><td>{src}</td></tr>')
`f9ca0752` 2026-10-27 Pavel Safronov	2006	body = (f"<h1>Pavel's apps</h1>"
	2007	f'<p>Small web apps and games. No accounts, no sign-in. Source code is at '
	2008	f'<a href="{BASE_URL}/">code.fuzzyworld.net</a>.</p>'
`392edf54` 2026-11-04 Pavel Safronov	2009	f'<p class="meta">Ordered by the newest code commit, not by deploy date: an app can be redeployed '
	2010	f'often without its code changing.</p>'
	2011	f'<table class="log repos"><tr><th></th><th>App</th><th>Description</th><th>Last commit</th>'
	2012	f'<th>Deployed</th><th>Source</th></tr>{"".join(items)}</table>')
`f9ca0752` 2026-10-27 Pavel Safronov	2013	write(os.path.join(APPS, "index.html"), page("Apps", body, site="apps"))
	2014	print(f"apps index: {len(apps)} apps")
	2015
	2016
`623239b7` 2026-10-26 Pavel Safronov	2017	def main(argv):
	2018	if len(argv) == 3 and argv[1] == "repo":
	2019	build_repo(argv[2])
	2020	elif len(argv) == 2 and argv[1] == "index":
	2021	build_index()
`f9ca0752` 2026-10-27 Pavel Safronov	2022	elif len(argv) == 2 and argv[1] == "apps":
	2023	build_apps_index()
`0330e724` 2026-11-01 Pavel Safronov	2024	elif len(argv) == 4 and argv[1] == "landing":
	2025	build_landing(argv[2], argv[3])
`ec9751fa` 2026-10-27 Pavel Safronov	2026	elif len(argv) == 2 and argv[1] == "content":
	2027	build_content()
`a5a72f55` 2026-10-27 Pavel Safronov	2028	elif len(argv) == 3 and argv[1] == "ci":
	2029	name = argv[2]
	2030	if public_repo(name):
	2031	branch = git_text(os.path.join(GIT_ROOT, name + ".git"), "symbolic-ref", "--short", "HEAD", check=False).strip()
	2032	build_ci_pages(View(name, branch or "main", True))
	2033	build_ci_dashboard()
	2034	elif len(argv) == 2 and argv[1] == "ci-dashboard":
	2035	build_ci_dashboard()
	2036	elif len(argv) == 2 and argv[1] == "ci-status":
	2037	print(ci_status_text())
`623239b7` 2026-10-26 Pavel Safronov	2038	else:
	2039	raise SystemExit(__doc__)
	2040
	2041
	2042	if __name__ == "__main__":
	2043	main(sys.argv)