fuzzyworld-tools / files / server / gen.py
gen.py
| 1 | #!/usr/bin/env python3 |
| 2 | """Static browse pages for code.fuzzyworld.net. |
| 3 | |
| 4 | gen.py repo <name> build or update the pages of one public repo |
| 5 | gen.py index rebuild the site index (public repos only) |
| 6 | gen.py apps rebuild the apps.fuzzyworld.net index from deployed apps |
| 7 | gen.py landing <src> <dest> write the landing page: src with <!--SITE-NAV--> filled in from NAV, <!--SITE-FOOTER--> from site_footer() and <!--LANDING-TEXT--> from landing.md |
| 8 | gen.py content render server/content/<name>.md to code.fuzzyworld.net/<name>/ (site pages written in Markdown) |
| 9 | gen.py ci <name> rebuild one public repo's CI pages, then the CI dashboard |
| 10 | gen.py ci-dashboard rebuild code.fuzzyworld.net/ci/ (queue, running, recent; public repos only) |
| 11 | gen.py ci-status the same, as text for the terminal, private repos included (fw ci-watch) |
| 12 | |
| 13 | Only the git CLI and the standard library are required. From ~/fw/pylib, when |
| 14 | importable: Python-Markdown renders README/.md files (otherwise plain text; `landing` |
| 15 | and `content` need it) and Pygments colors source code (otherwise uncolored). |
| 16 | |
| 17 | Pages are incremental: a manifest per repo (~/fw/state/<name>.json) records the |
| 18 | blob SHA behind every file page, so a push rewrites only what changed. |
| 19 | """ |
| 20 | |
| 21 | import datetime |
| 22 | import hashlib |
| 23 | import html |
| 24 | import json |
| 25 | import os |
| 26 | import posixpath |
| 27 | import re |
| 28 | import shutil |
| 29 | import subprocess |
| 30 | import sys |
| 31 | import time |
| 32 | import urllib.parse |
| 33 | |
| 34 | HOME = os.path.expanduser("~") |
| 35 | GIT_ROOT = os.environ.get("FW_GIT_ROOT", os.path.join(HOME, "git")) |
| 36 | SITE = os.environ.get("FW_CODE_SITE", os.path.join(HOME, "code.fuzzyworld.net")) |
| 37 | APPS = os.environ.get("FW_APPS_SITE", os.path.join(HOME, "apps.fuzzyworld.net")) |
| 38 | STATE = os.environ.get("FW_STATE", os.path.join(HOME, "fw", "state")) |
| 39 | PYLIB = os.environ.get("FW_PYLIB", os.path.join(HOME, "fw", "pylib")) |
| 40 | CI_ROOT = os.environ.get("FW_CI", os.path.join(HOME, "fw", "ci")) |
| 41 | QUEUE = os.environ.get("FW_QUEUE", os.path.join(HOME, "fw", "queue")) |
| 42 | RUNNERS = os.environ.get("FW_RUNNERS", os.path.join(HOME, "fw", "runners")) # one file per runner; its time is its last check-in |
| 43 | BASE_URL = "https://code.fuzzyworld.net" |
| 44 | APPS_URL = "https://apps.fuzzyworld.net" |
| 45 | |
| 46 | # Bump when page templates change, so every repo gets a full rebuild once. |
| 47 | GEN_VERSION = 10 |
| 48 | |
| 49 | |
| 50 | def gen_key(): |
| 51 | """The version recorded in a repo's manifest: GEN_VERSION plus a hash of prices.json, so a rate-card |
| 52 | refresh rebuilds the commit pages (they show estimated costs).""" |
| 53 | try: |
| 54 | with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "prices.json"), "rb") as f: |
| 55 | return f"{GEN_VERSION}-{hashlib.sha1(f.read()).hexdigest()[:8]}" |
| 56 | except OSError: |
| 57 | return f"{GEN_VERSION}-noprices" |
| 58 | |
| 59 | COMMIT_LIMIT = 2000 # commit pages per repo, across all branches; older commits: clone |
| 60 | HISTORY_LIMIT = 200 # commits listed on a file's history page |
| 61 | CI_LIMIT = 100 # pipelines listed on a repo's CI page |
| 62 | CI_RECENT = 50 # finished pipelines on the CI dashboard |
| 63 | RUNNER_GONE = 15 * 60 # seconds without a check-in before a runner doesn't count (runners ask every 2 minutes) |
| 64 | CI_REFRESH = 15 # seconds: pages showing something running reload themselves |
| 65 | SHOTS = "_shots" # apps.fuzzyworld.net/_shots/<app>/: screenshots from CI runners (no app name starts with "_") |
| 66 | SHOT_EXTENSIONS = (".png", ".jpg", ".jpeg", ".webp") |
| 67 | VSCODE_APP = "vscode" # apps.fuzzyworld.net/<this>/: the static VS Code web build (vscode-web/) |
| 68 | MAX_TEXT_BYTES = 1024 * 1024 |
| 69 | MAX_DIFF_BYTES = 300 * 1024 |
| 70 | MAX_RAW_BYTES = 2 * 1024 * 1024 |
| 71 | MAX_HIGHLIGHT_BYTES = 512 * 1024 |
| 72 | MAX_BLAME_BYTES = 256 * 1024 |
| 73 | MAX_LOG_BYTES = 256 * 1024 # the end of each CI job log is shown |
| 74 | # Raw copies are written only for these: other extensions (.php, .shtml, ...) could be |
| 75 | # executed by the web server, and the browse pages already show text files. |
| 76 | # SVG is safe here because the site's CSP header (script-src 'none') covers SVG documents too. |
| 77 | RAW_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".avif", ".bmp", ".svg"} |
| 78 | README_NAMES = ["readme.md", "readme.markdown", "readme", "readme.txt", "readme.rst"] |
| 79 | |
| 80 | sys.path.insert(0, PYLIB) |
| 81 | try: |
| 82 | import markdown # type: ignore |
| 83 | except ImportError: # pragma: no cover |
| 84 | markdown = None |
| 85 | try: |
| 86 | import pygments.lexers as pygments_lexers # type: ignore |
| 87 | import pygments.token as pygments_token # type: ignore |
| 88 | except ImportError: # pragma: no cover |
| 89 | pygments_lexers = pygments_token = None |
| 90 | |
| 91 | |
| 92 | # ---------------------------------------------------------------- git helpers |
| 93 | |
| 94 | def git(repo, *args, check=True): |
| 95 | # DreamHost sometimes refuses to start a process for a moment (EAGAIN) when many run in a row, |
| 96 | # e.g. one `git blame` per file on a first build; wait and try again. |
| 97 | for attempt in range(5): |
| 98 | try: |
| 99 | r = subprocess.run(["git", "-C", repo, *args], capture_output=True, check=False) |
| 100 | break |
| 101 | except BlockingIOError: |
| 102 | if attempt == 4: |
| 103 | raise |
| 104 | time.sleep(1 + attempt) |
| 105 | if check and r.returncode != 0: |
| 106 | raise RuntimeError(f"git {' '.join(args)}: {r.stderr.decode(errors='replace').strip()}") |
| 107 | return r.stdout |
| 108 | |
| 109 | |
| 110 | def git_text(repo, *args, check=True): |
| 111 | return git(repo, *args, check=check).decode("utf-8", errors="replace") |
| 112 | |
| 113 | |
| 114 | class BlobReader: |
| 115 | """Streams blob contents through one `git cat-file --batch` process.""" |
| 116 | |
| 117 | def __init__(self, repo): |
| 118 | self.p = subprocess.Popen(["git", "-C", repo, "cat-file", "--batch"], |
| 119 | stdin=subprocess.PIPE, stdout=subprocess.PIPE) |
| 120 | |
| 121 | def read(self, sha): |
| 122 | self.p.stdin.write(sha.encode() + b"\n") |
| 123 | self.p.stdin.flush() |
| 124 | header = self.p.stdout.readline().split() |
| 125 | if len(header) < 3 or header[1] == b"missing": |
| 126 | return b"" |
| 127 | data = self.p.stdout.read(int(header[2])) |
| 128 | self.p.stdout.read(1) # trailing newline |
| 129 | return data |
| 130 | |
| 131 | def close(self): |
| 132 | self.p.stdin.close() |
| 133 | self.p.wait() |
| 134 | |
| 135 | |
| 136 | def repo_config(repo, key, default=""): |
| 137 | v = git_text(repo, "config", "--get", key, check=False).strip() |
| 138 | return v or default |
| 139 | |
| 140 | |
| 141 | def is_public(repo): |
| 142 | # Anything but the exact value "public" counts as private. |
| 143 | return repo_config(repo, "fuzzyworld.visibility") == "public" |
| 144 | |
| 145 | |
| 146 | # ---------------------------------------------------------------- html helpers |
| 147 | |
| 148 | esc = html.escape |
| 149 | |
| 150 | |
| 151 | def url(*parts): |
| 152 | """Site-absolute URL from path parts, each part quoted per segment.""" |
| 153 | path = "/".join(p.strip("/") for p in parts if p) |
| 154 | return "/" + "/".join(urllib.parse.quote(s) for s in path.split("/")) |
| 155 | |
| 156 | |
| 157 | HOME_URL = "https://fuzzyworld.net" |
| 158 | # Site-wide navigation, in the header of every page on both sites: (section, label, site, path). |
| 159 | NAV = [("code", "Code", "code", "/"), ("apps", "Apps", "apps", "/"), ("writing", "Writing", "writing", "/"), |
| 160 | ("projects", "Projects", "code", "/projects/"), |
| 161 | ("my-ai-use", "Agentic", "code", "/my-ai-use/"), ("ci", "CI", "code", "/ci/"), |
| 162 | ("usage", "Usage", "code", "/usage/"), |
| 163 | ("contribute", "Contribute", "code", "/contribute/")] |
| 164 | WRITING_URL = "https://writing.fuzzyworld.net" # a static site built by `fw blog` (blog.py), not by this file |
| 165 | SITE_URLS = {"code": BASE_URL, "apps": APPS_URL, "writing": WRITING_URL} |
| 166 | |
| 167 | |
| 168 | def site_nav(): |
| 169 | """NAV as [label, absolute URL] pairs: the one source for the nav on the landing page and the blog's `site` bar.""" |
| 170 | return [[label, SITE_URLS[st] + path] for _, label, st, path in NAV] |
| 171 | |
| 172 | |
| 173 | def site_footer(here): |
| 174 | """The columns of the site-wide footer (inside <div class="cols">), with links relative to the site `here` |
| 175 | ("code", "apps", "writing", or "landing" for fuzzyworld.net). One source for the code and apps pages, the |
| 176 | landing page and the blog.""" |
| 177 | def link(st, path): |
| 178 | return SITE_URLS[st] + path if st != here else path |
| 179 | return (f'<div><h2>fuzzyworld</h2><a href="{HOME_URL}/">fuzzyworld.net</a>' |
| 180 | f'<a href="{link("code", "/")}">Code: all repositories</a>' |
| 181 | f'<a href="{link("apps", "/")}">Apps</a>' |
| 182 | f'<a href="{WRITING_URL}/">Writing</a>' |
| 183 | f'<a href="{link("code", "/ci/")}">CI: what is building</a></div>' |
| 184 | f'<div><h2>About this site</h2><a href="{link("code", "/why-i-left-gitlab/")}">Why I left GitLab</a>' |
| 185 | f'<a href="{link("code", "/contribute/")}">Contribute and report bugs</a></div>' |
| 186 | f'<div><h2>Built with</h2><a href="{link("code", "/fuzzyworld-tools/")}">fuzzyworld-tools</a>' |
| 187 | '<span>Static files only: no accounts, no sign-in, no rate limits. MIT-licensed.</span></div>') |
| 188 | |
| 189 | |
| 190 | def build_landing(src, dest): |
| 191 | """Writes the landing page: `src` with <!--SITE-NAV--> replaced by the nav links built from NAV, |
| 192 | <!--SITE-FOOTER--> by site_footer("landing") and, if the |
| 193 | page has a <!--LANDING-TEXT-->, that replaced by landing.md (next to `src`) rendered from Markdown.""" |
| 194 | links = "".join(f'<a href="{html.escape(url, quote=True)}">{html.escape(label)}</a>' for label, url in site_nav()) |
| 195 | with open(src, encoding="utf-8") as f: |
| 196 | text = f.read() |
| 197 | for marker in ("<!--SITE-NAV-->", "<!--SITE-FOOTER-->"): |
| 198 | if marker not in text: |
| 199 | raise SystemExit(f"{src}: no {marker} placeholder") |
| 200 | if "<!--LANDING-TEXT-->" in text: |
| 201 | if not markdown: |
| 202 | raise SystemExit("gen.py landing needs Python-Markdown in " + PYLIB) |
| 203 | md_path = os.path.join(os.path.dirname(src), "landing.md") |
| 204 | if not os.path.exists(md_path): |
| 205 | raise SystemExit(f"{src}: has <!--LANDING-TEXT--> but {md_path} does not exist") |
| 206 | with open(md_path, encoding="utf-8") as f: |
| 207 | body = markdown.markdown(f.read(), extensions=["sane_lists"], output_format="html") |
| 208 | text = text.replace("<!--LANDING-TEXT-->", body) |
| 209 | text = text.replace("<!--SITE-FOOTER-->", site_footer("landing")) |
| 210 | write(dest, text.replace("<!--SITE-NAV-->", links)) |
| 211 | |
| 212 | |
| 213 | def site_link(site, path, here): |
| 214 | """Links within a site stay relative; links to the other site are absolute.""" |
| 215 | return path if site == here else SITE_URLS[site] + path |
| 216 | |
| 217 | |
| 218 | def page(title, body, crumbs=None, view=None, site="code", refresh=None, section=None): |
| 219 | """`site` is the site being generated; `section` is the nav entry to highlight (default: the site's home).""" |
| 220 | host = "apps.fuzzyworld.net" if site == "apps" else "code.fuzzyworld.net" |
| 221 | section = section or site |
| 222 | links = "".join( |
| 223 | f'<a href="{site_link(st, path, site)}"' + (' aria-current="page"' if key == section else "") + f'>{label}</a>' |
| 224 | for key, label, st, path in NAV) |
| 225 | header = (f'<header class="site"><div class="bar"><a class="brand" href="{HOME_URL}/">fuzzyworld</a>' |
| 226 | f'<nav class="site-nav" aria-label="Site">{links}</nav></div></header>') |
| 227 | footer = f'<footer class="site"><div class="cols">{site_footer(site)}</div></footer>' |
| 228 | nav = "" |
| 229 | if view: |
| 230 | nav = (f'<nav class="repo-nav"><a href="{url(view.name)}/">Summary</a>' |
| 231 | f'<a href="{view.url("tree")}/">Files</a>' |
| 232 | f'<a href="{view.url("log")}/">Log</a>' |
| 233 | f'<a href="{url(view.name, "refs")}/">Branches & tags</a>' |
| 234 | + (f'<a href="{url(view.name, "issues")}/">Issues</a>' |
| 235 | if os.path.exists(os.path.join(SITE, view.name, "issues", "index.html")) else "") |
| 236 | + (f'<a href="{url(view.name, "ci")}/">CI</a>' if ci_pipelines(view.name) else "") |
| 237 | + (f'<a href="{url(view.name, "usage")}/">Usage</a>' if repo_usage(view.name)["costs"] else "") |
| 238 | + "</nav>") |
| 239 | crumb_html = "" |
| 240 | if crumbs: |
| 241 | crumb_html = '<p class="crumbs">' + " / ".join(crumbs) + "</p>" |
| 242 | return f"""<!doctype html> |
| 243 | <html lang="en"> |
| 244 | <head> |
| 245 | <meta charset="utf-8"> |
| 246 | <meta name="viewport" content="width=device-width, initial-scale=1"> |
| 247 | {f'<meta http-equiv="refresh" content="{refresh}">' if refresh else ""} |
| 248 | <title>{esc(title)} · {host}</title> |
| 249 | <link rel="icon" href="/favicon.ico"> |
| 250 | <link rel="stylesheet" href="/style.css"> |
| 251 | </head> |
| 252 | <body> |
| 253 | {header} |
| 254 | <main> |
| 255 | {nav} |
| 256 | {crumb_html} |
| 257 | {body} |
| 258 | </main> |
| 259 | {footer} |
| 260 | {reload_timer(refresh)} |
| 261 | </body> |
| 262 | </html> |
| 263 | """ |
| 264 | |
| 265 | |
| 266 | def reload_timer(refresh): |
| 267 | """A thin bar along the top that empties as the page's meta refresh nears, and a countdown in |
| 268 | any `.reload-in` element (user request, 2026-11-03: the CI page said it reloads every 15 |
| 269 | seconds but not when). Pure CSS, because the site's CSP is `script-src 'none'`: the countdown is |
| 270 | an animated integer property shown through a counter. Starts at page load, as the refresh does. |
| 271 | Browsers without @property support show the bar only.""" |
| 272 | if not refresh: |
| 273 | return "" |
| 274 | n = int(refresh) |
| 275 | return f"""<div class="reload-bar" aria-hidden="true"></div> |
| 276 | <style> |
| 277 | .reload-bar {{ position: fixed; top: 0; left: 0; height: 3px; width: 100%; background: currentColor; opacity: .45; |
| 278 | transform-origin: left; animation: reload-bar {n}s linear forwards; }} |
| 279 | @keyframes reload-bar {{ from {{ transform: scaleX(1); }} to {{ transform: scaleX(0); }} }} |
| 280 | @property --reload-left {{ syntax: "<integer>"; inherits: false; initial-value: {n}; }} |
| 281 | @keyframes reload-count {{ from {{ --reload-left: {n}; }} to {{ --reload-left: 0; }} }} |
| 282 | .reload-in::before {{ content: "Next reload in " counter(reload-left) " s."; counter-reset: reload-left var(--reload-left); |
| 283 | animation: reload-count {n}s steps({n}) forwards; }} |
| 284 | @media (prefers-reduced-motion: reduce) {{ .reload-bar {{ display: none; }} }} |
| 285 | </style>""" |
| 286 | |
| 287 | |
| 288 | def write(path, content): |
| 289 | os.makedirs(os.path.dirname(path), exist_ok=True) |
| 290 | tmp = path + ".tmp" |
| 291 | mode = "wb" if isinstance(content, bytes) else "w" |
| 292 | with open(tmp, mode, **({} if mode == "wb" else {"encoding": "utf-8"})) as f: |
| 293 | f.write(content) |
| 294 | os.replace(tmp, path) |
| 295 | |
| 296 | |
| 297 | def remove(path): |
| 298 | try: |
| 299 | os.remove(path) |
| 300 | except FileNotFoundError: |
| 301 | pass |
| 302 | |
| 303 | |
| 304 | def human_size(n): |
| 305 | for unit in ("B", "KB", "MB", "GB"): |
| 306 | if n < 1024 or unit == "GB": |
| 307 | return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}" |
| 308 | n /= 1024 |
| 309 | |
| 310 | |
| 311 | def is_binary(data): |
| 312 | return b"\0" in data[:8000] |
| 313 | |
| 314 | |
| 315 | def safe_path(p): |
| 316 | parts = p.split("/") |
| 317 | return all(part not in ("", ".", "..") for part in parts) |
| 318 | |
| 319 | |
| 320 | # ---------------------------------------------------------------- CI results |
| 321 | |
| 322 | # CI runners (server/runner/) upload each pipeline to ~/fw/ci/<repo>/<sha>/: pipeline.json |
| 323 | # plus one log file per job. These pages show them for public repos only. |
| 324 | CI_STATES = { # status: (mark, words, badge color) |
| 325 | "success": ("✓", "passed", "#2da44e"), "failed": ("✗", "failed", "#cf222e"), |
| 326 | "running": ("●", "running", "#bf8700"), "error": ("!", "could not run", "#6e7781"), |
| 327 | } |
| 328 | _ci_cache = {} |
| 329 | |
| 330 | |
| 331 | def ci_pipelines(name): |
| 332 | """{sha: pipeline} for one repo, read once per run.""" |
| 333 | if name not in _ci_cache: |
| 334 | out, d = {}, os.path.join(CI_ROOT, name) |
| 335 | for sha in (os.listdir(d) if os.path.isdir(d) else []): |
| 336 | if not re.fullmatch(r"[0-9a-f]{40}", sha): |
| 337 | continue |
| 338 | try: |
| 339 | with open(os.path.join(d, sha, "pipeline.json")) as f: |
| 340 | p = json.load(f) |
| 341 | except (OSError, ValueError): |
| 342 | continue |
| 343 | if isinstance(p, dict) and p.get("status") in CI_STATES: |
| 344 | out[sha] = p |
| 345 | _ci_cache[name] = out |
| 346 | return _ci_cache[name] |
| 347 | |
| 348 | |
| 349 | def ci_mark(name, sha, p): |
| 350 | if not p: |
| 351 | return "" |
| 352 | mark, words, _ = CI_STATES[p["status"]] |
| 353 | return f'<a class="ci ci-{p["status"]}" href="{url(name, "ci", sha)}.html" title="CI {words}">{mark}</a> ' |
| 354 | |
| 355 | |
| 356 | def ci_badge(name, sha): |
| 357 | p = ci_pipelines(name).get(sha) |
| 358 | if not p: |
| 359 | return "" |
| 360 | return f' · CI: {ci_mark(name, sha, p)}<a href="{url(name, "ci", sha)}.html">{CI_STATES[p["status"]][1]}</a>' |
| 361 | |
| 362 | |
| 363 | def badge_svg(status): |
| 364 | _, words, color = CI_STATES[status] |
| 365 | w = 18 + 7 * len(words) |
| 366 | return (f'<svg xmlns="http://www.w3.org/2000/svg" width="{28 + w}" height="20" role="img" aria-label="CI: {words}">' |
| 367 | f'<rect width="28" height="20" fill="#555"/><rect x="28" width="{w}" height="20" fill="{color}"/>' |
| 368 | f'<g fill="#fff" font-family="Verdana,DejaVu Sans,sans-serif" font-size="11">' |
| 369 | f'<text x="6" y="14">CI</text><text x="37" y="14">{words}</text></g></svg>') |
| 370 | |
| 371 | |
| 372 | _ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\x07]*\x07|\r(?!\n)") |
| 373 | |
| 374 | |
| 375 | def duration(seconds): |
| 376 | s = int(seconds or 0) |
| 377 | return f"{s // 60}m {s % 60:02d}s" if s >= 60 else f"{s}s" |
| 378 | |
| 379 | |
| 380 | def build_ci_pages(view): |
| 381 | """ci/index.html, one page per pipeline, and ci/badge.svg (latest pipeline on the default branch).""" |
| 382 | name, out = view.name, os.path.join(SITE, view.name, "ci") |
| 383 | pipelines = ci_pipelines(name) |
| 384 | if not pipelines: |
| 385 | shutil.rmtree(out, ignore_errors=True) |
| 386 | return |
| 387 | ordered = sorted(pipelines.items(), key=lambda kv: kv[1].get("started", ""), reverse=True)[:CI_LIMIT] |
| 388 | keep = {sha + ".html" for sha, _ in ordered} | {"index.html", "badge.svg"} |
| 389 | for f in (os.listdir(out) if os.path.isdir(out) else []): |
| 390 | if f not in keep: |
| 391 | remove(os.path.join(out, f)) |
| 392 | rows = [] |
| 393 | for sha, p in ordered: |
| 394 | rows.append(f'<tr><td class="date">{esc(p.get("started", "")[:16].replace("T", " "))}</td>' |
| 395 | f'<td>{ci_mark(name, sha, p)}<a href="{url(name, "ci", sha)}.html"><code>{sha[:12]}</code></a></td>' |
| 396 | f'<td><code>{esc(p.get("ref", ""))}</code></td><td>{esc(CI_STATES[p["status"]][1])}</td>' |
| 397 | f'<td class="num">{duration(p.get("duration"))}</td></tr>') |
| 398 | page_path = os.path.join(out, sha + ".html") |
| 399 | # Rebuilt when pipeline.json or a log is newer: a runner uploads a running job's log every few seconds. |
| 400 | src_dir = os.path.join(CI_ROOT, name, sha) |
| 401 | newest = max(os.path.getmtime(os.path.join(src_dir, f)) for f in os.listdir(src_dir)) |
| 402 | if os.path.exists(page_path) and os.path.getmtime(page_path) >= newest: |
| 403 | continue |
| 404 | jobs = [] |
| 405 | for j in p.get("jobs", []): |
| 406 | st = j.get("status", "error") |
| 407 | secs = j.get("duration") |
| 408 | if st == "running" and j.get("started"): |
| 409 | secs = time.time() - iso_time(j["started"]) |
| 410 | log, log_name = "", j.get("log", "") |
| 411 | if re.fullmatch(r"[A-Za-z0-9._-]+\.log", log_name or ""): |
| 412 | try: |
| 413 | with open(os.path.join(CI_ROOT, name, sha, log_name), "rb") as f: |
| 414 | f.seek(0, 2) |
| 415 | size = f.tell() |
| 416 | f.seek(max(0, size - MAX_LOG_BYTES)) |
| 417 | text = f.read().decode("utf-8", errors="replace") |
| 418 | cut = "(start of log cut; showing the last part)\n" if size > MAX_LOG_BYTES else "" |
| 419 | log = f'<pre class="ci-log">{esc(cut + _ANSI.sub("", text))}</pre>' |
| 420 | except OSError: |
| 421 | pass |
| 422 | mark = CI_STATES.get(st, ("–", st, "")) |
| 423 | jobs.append(f'<details{" open" if st in ("failed", "error", "running") else ""}><summary>' |
| 424 | f'<span class="ci ci-{esc(st)}">{mark[0]}</span> <strong>{esc(j.get("name", "?"))}</strong>' |
| 425 | f' · {esc(j.get("stage", ""))} · {esc(mark[1])} · {duration(secs)}' |
| 426 | f'{" · " + esc(j["message"]) if j.get("message") else ""}</summary>{log}</details>') |
| 427 | mark, words, _ = CI_STATES[p["status"]] |
| 428 | running = p["status"] == "running" |
| 429 | body = (f'<h1>CI {esc(words)}: <code>{sha[:12]}</code></h1>' |
| 430 | f'<p class="meta">Branch <code>{esc(p.get("ref", ""))}</code> · ' |
| 431 | f'<a href="{url(name, "commit", sha)}.html">commit</a> · started {esc(p.get("started", "").replace("T", " ").replace("Z", " UTC"))} · ' |
| 432 | f'{"running" if running else duration(p.get("duration"))} · runner {esc(p.get("runner", ""))}</p>' |
| 433 | + (f'<p class="meta">This page reloads every {CI_REFRESH} seconds while the pipeline runs. <span class="reload-in"></span></p>' if running else "") |
| 434 | + (f'<p>{esc(p["message"])}</p>' if p.get("message") else "") |
| 435 | + "".join(jobs)) |
| 436 | write(page_path, page(f"{name} CI {sha[:12]}", body, [f'<a href="{url(name)}/">{esc(name)}</a>', "CI"], view=view, |
| 437 | refresh=CI_REFRESH if running else None)) |
| 438 | any_running = any(p["status"] == "running" for _, p in ordered) |
| 439 | write(os.path.join(out, "index.html"), |
| 440 | page(f"{name} CI", f'<h1>CI</h1><p class="meta"><a href="/ci/">All CI</a></p><table class="log">{"".join(rows)}</table>', |
| 441 | view=view, refresh=CI_REFRESH if any_running else None)) |
| 442 | latest = next((p for _, p in ordered if p.get("ref") == view.branch), None) |
| 443 | if latest: |
| 444 | write(os.path.join(out, "badge.svg"), badge_svg(latest["status"])) |
| 445 | else: |
| 446 | remove(os.path.join(out, "badge.svg")) |
| 447 | |
| 448 | |
| 449 | def iso_time(s): |
| 450 | try: |
| 451 | return datetime.datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() |
| 452 | except (ValueError, AttributeError): |
| 453 | return 0 |
| 454 | |
| 455 | |
| 456 | def queue_jobs(kind): |
| 457 | """Jobs in ~/fw/queue/<kind>/ (pending or running), oldest first, with "since": the file's time.""" |
| 458 | d, out = os.path.join(QUEUE, kind), [] |
| 459 | for f in sorted(os.listdir(d)) if os.path.isdir(d) else []: |
| 460 | try: |
| 461 | with open(os.path.join(d, f)) as fh: |
| 462 | j = json.load(fh) |
| 463 | j["since"] = os.path.getmtime(os.path.join(d, f)) |
| 464 | out.append(j) |
| 465 | except (OSError, ValueError): |
| 466 | continue |
| 467 | return out |
| 468 | |
| 469 | |
| 470 | def runners_seen(): |
| 471 | """[(name, tags, seconds since it last checked in)], most recent first.""" |
| 472 | out = [] |
| 473 | for name in os.listdir(RUNNERS) if os.path.isdir(RUNNERS) else []: |
| 474 | try: |
| 475 | with open(os.path.join(RUNNERS, name)) as f: |
| 476 | tags = f.read().strip().strip("-") |
| 477 | out.append((name, tags, time.time() - os.path.getmtime(os.path.join(RUNNERS, name)))) |
| 478 | except OSError: |
| 479 | continue |
| 480 | return sorted(out, key=lambda r: r[2]) |
| 481 | |
| 482 | |
| 483 | def runner_notice(pending): |
| 484 | """One sentence for the CI pages: "" if some runner is around, else why queued jobs are not moving.""" |
| 485 | if not pending: |
| 486 | return "" |
| 487 | seen = runners_seen() |
| 488 | if any(age < RUNNER_GONE for _, _, age in seen): |
| 489 | return "" |
| 490 | if seen: |
| 491 | return f"Waiting for a runner: the last one checked in {duration(seen[0][2])} ago. Jobs wait here until one does." |
| 492 | return "Waiting for a runner: none has checked in yet. Jobs wait here until one does." |
| 493 | |
| 494 | |
| 495 | def public_repo(name, _cache={}): |
| 496 | if name not in _cache: |
| 497 | repo = os.path.join(GIT_ROOT, name + ".git") |
| 498 | _cache[name] = os.path.isdir(repo) and is_public(repo) |
| 499 | return _cache[name] |
| 500 | |
| 501 | |
| 502 | def ci_overview(include_private): |
| 503 | """(pending jobs, running jobs with their pipelines, recent finished pipelines) across all repos.""" |
| 504 | pending, running = queue_jobs("pending"), queue_jobs("running") |
| 505 | for j in running: |
| 506 | j["pipeline"] = ci_pipelines(j["repo"]).get(j["sha"], {}) |
| 507 | recent = [] |
| 508 | for name in (os.listdir(CI_ROOT) if os.path.isdir(CI_ROOT) else []): |
| 509 | if include_private or public_repo(name): |
| 510 | recent += [(p.get("started", ""), name, sha, p) for sha, p in ci_pipelines(name).items() if p["status"] != "running"] |
| 511 | recent.sort(reverse=True) |
| 512 | return pending, running, recent[:CI_RECENT] |
| 513 | |
| 514 | |
| 515 | def job_progress(p): |
| 516 | """ "test ✓ 44s · pages ● 12s" for a running pipeline.""" |
| 517 | parts = [] |
| 518 | for j in p.get("jobs", []): |
| 519 | st = j.get("status", "pending") |
| 520 | secs = time.time() - iso_time(j["started"]) if st == "running" and j.get("started") else j.get("duration") |
| 521 | mark = CI_STATES.get(st, ("·" if st == "pending" else "–", st, ""))[0] |
| 522 | parts.append(f'{j.get("name", "?")} {mark}' + (f" {duration(secs)}" if st not in ("pending", "skipped") else "")) |
| 523 | return " · ".join(parts) |
| 524 | |
| 525 | |
| 526 | def build_ci_dashboard(): |
| 527 | """code.fuzzyworld.net/ci/: what is queued, running and recently finished. Private repos are never named.""" |
| 528 | pending, running, recent = ci_overview(include_private=False) |
| 529 | def ref(j): |
| 530 | return j["ref"].split("/", 2)[-1] |
| 531 | hidden = sum(1 for j in pending + running if not public_repo(j["repo"])) |
| 532 | run_rows = "".join( |
| 533 | f'<tr><td><a href="{url(j["repo"])}/">{esc(j["repo"])}</a></td><td><code>{esc(ref(j))}</code></td>' |
| 534 | f'<td><a href="{url(j["repo"], "ci", j["sha"])}.html"><code>{j["sha"][:12]}</code></a></td>' |
| 535 | f'<td>{esc(job_progress(j["pipeline"]) or "starting")}</td>' |
| 536 | f'<td class="num">{duration(time.time() - j["since"])}</td></tr>' |
| 537 | for j in running if public_repo(j["repo"])) |
| 538 | queue_rows = "".join( |
| 539 | f'<tr><td><a href="{url(j["repo"])}/">{esc(j["repo"])}</a></td><td><code>{esc(ref(j))}</code></td>' |
| 540 | f'<td><code>{j["sha"][:12]}</code></td><td class="num">waiting {duration(time.time() - j["since"])}</td></tr>' |
| 541 | for j in pending if public_repo(j["repo"])) |
| 542 | done_rows = "".join( |
| 543 | f'<tr><td class="date">{esc(started[:16].replace("T", " "))}</td><td><a href="{url(name)}/">{esc(name)}</a></td>' |
| 544 | f'<td><code>{esc(p.get("ref", ""))}</code></td>' |
| 545 | f'<td>{ci_mark(name, sha, p)}<a href="{url(name, "ci", sha)}.html"><code>{sha[:12]}</code></a></td>' |
| 546 | f'<td>{esc(CI_STATES[p["status"]][1])}</td><td class="num">{duration(p.get("duration"))}</td></tr>' |
| 547 | for started, name, sha, p in recent) |
| 548 | busy = bool(pending or running) |
| 549 | notice = runner_notice(pending) |
| 550 | body = ('<h1>CI</h1>' |
| 551 | '<p class="meta">Pipelines run on whichever runner is available (a home server, sometimes a laptop), one at a time, as pushes queue them. ' |
| 552 | + (f'This page reloads every {CI_REFRESH} seconds while anything is queued or running. <span class="reload-in"></span>' if busy else "Nothing is running.") |
| 553 | + (f' Also {hidden} job{"s" if hidden != 1 else ""} for private repositories.' if hidden else "") + '</p>' |
| 554 | + (f'<p><strong>{esc(notice)}</strong></p>' if notice else "") |
| 555 | + (f'<h2>Running</h2><table class="log">{run_rows}</table>' if run_rows else "") |
| 556 | + (f'<h2>Queued</h2><table class="log">{queue_rows}</table>' if queue_rows else "") |
| 557 | + (f'<h2>Recent</h2><table class="log">{done_rows}</table>' if done_rows else "<p>No pipelines yet.</p>")) |
| 558 | write(os.path.join(SITE, "ci", "index.html"), page("CI", body, refresh=CI_REFRESH if busy else None, section="ci")) |
| 559 | |
| 560 | |
| 561 | def ci_status_text(): |
| 562 | """For the terminal (fw ci-watch): queue, running and the last few results, private repos included.""" |
| 563 | pending, running, recent = ci_overview(include_private=True) |
| 564 | lines = [] |
| 565 | for j in running: |
| 566 | lines.append(f'running {j["repo"]} {j["ref"].split("/", 2)[-1]} {j["sha"][:12]} {job_progress(j["pipeline"]) or "starting"}') |
| 567 | for j in pending: |
| 568 | lines.append(f'queued {j["repo"]} {j["ref"].split("/", 2)[-1]} {j["sha"][:12]}') |
| 569 | for started, name, sha, p in recent[:5]: |
| 570 | mark, words, _ = CI_STATES[p["status"]] |
| 571 | lines.append(f'{mark} {words:<6} {name} {p.get("ref", "")} {sha[:12]} {started[:16].replace("T", " ")} {duration(p.get("duration"))}') |
| 572 | if runner_notice(pending): |
| 573 | lines.insert(0, runner_notice(pending)) |
| 574 | return "\n".join(lines) if lines else "no CI jobs yet" |
| 575 | |
| 576 | |
| 577 | # ---------------------------------------------------------------- views |
| 578 | |
| 579 | class View: |
| 580 | """One browsable branch. |
| 581 | |
| 582 | The default branch keeps the short URLs (/<name>/tree/...). Other branches live under |
| 583 | /<name>/branch/<branch>/, with each '/' in the branch name written as '~': git forbids |
| 584 | '~' in ref names, so two branches can never share a directory. |
| 585 | """ |
| 586 | |
| 587 | def __init__(self, name, branch, is_default): |
| 588 | self.name, self.branch, self.is_default = name, branch, is_default |
| 589 | self.ref = "refs/heads/" + branch |
| 590 | self.base = [name] if is_default else [name, "branch", branch.replace("/", "~")] |
| 591 | |
| 592 | def url(self, *parts): |
| 593 | return url(*self.base, *parts) |
| 594 | |
| 595 | def path(self, *parts): |
| 596 | return os.path.join(SITE, *self.base, *parts) |
| 597 | |
| 598 | @property |
| 599 | def key(self): |
| 600 | return "/".join(self.base[1:]) or "." |
| 601 | |
| 602 | |
| 603 | # ---------------------------------------------------------------- markdown |
| 604 | |
| 605 | class LinkContext: |
| 606 | """Rewrites relative links in a repo's markdown to this site's pages.""" |
| 607 | |
| 608 | def __init__(self, view, base_dir, files, dirs, raw): |
| 609 | self.view, self.base_dir, self.files, self.dirs, self.raw = view, base_dir, files, dirs, raw |
| 610 | |
| 611 | def rewrite(self, href, image): |
| 612 | parsed = urllib.parse.urlsplit(href) |
| 613 | if parsed.scheme or parsed.netloc or href.startswith(("#", "/", "mailto:")): |
| 614 | return href |
| 615 | target = posixpath.normpath(posixpath.join(self.base_dir, urllib.parse.unquote(parsed.path))) |
| 616 | frag = ("#" + parsed.fragment) if parsed.fragment else "" |
| 617 | if image and target in self.raw: |
| 618 | return self.view.url("raw", target) |
| 619 | if target in self.files: |
| 620 | return self.view.url("blob", target) + ".html" + frag |
| 621 | if target in self.dirs or target == ".": |
| 622 | return self.view.url("tree", "" if target == "." else target) + "/" + frag |
| 623 | return href |
| 624 | |
| 625 | |
| 626 | _ATTR = re.compile(r'(<(?:img|source)\b[^>]*?\bsrc|<a\b[^>]*?\bhref)=(["\'])(.*?)\2', re.IGNORECASE | re.DOTALL) |
| 627 | |
| 628 | |
| 629 | def render_markdown(text, ctx): |
| 630 | if not markdown: |
| 631 | return f"<pre>{esc(text)}</pre>" |
| 632 | out = markdown.Markdown(extensions=["tables", "fenced_code", "sane_lists"], output_format="html").convert(text) |
| 633 | # Rewrite links in the finished HTML, so raw <img>/<a> tags in the markdown are covered too. |
| 634 | def fix(m): |
| 635 | is_img = not m.group(1).lower().startswith("<a") |
| 636 | new = ctx.rewrite(html.unescape(m.group(3)), image=is_img) |
| 637 | return f"{m.group(1)}={m.group(2)}{esc(new, quote=True)}{m.group(2)}" |
| 638 | return _ATTR.sub(fix, out) |
| 639 | |
| 640 | |
| 641 | CONTENT = os.environ.get("FW_CONTENT", os.path.join(os.path.dirname(os.path.abspath(__file__)), "content")) |
| 642 | |
| 643 | |
| 644 | def build_content(): |
| 645 | """Site pages: content/<name>.md (Markdown) or content/<name>.html (a fragment) becomes |
| 646 | <site>/<name>/index.html. The title is a `title: ...` line at the top of a Markdown file |
| 647 | (Python-Markdown's meta extension) or a leading `<!-- title: ... -->` in an HTML one. |
| 648 | Files starting with "_" are drafts and are skipped. Also writes 404.html.""" |
| 649 | if not markdown: |
| 650 | raise SystemExit("gen.py content needs Python-Markdown in " + PYLIB) |
| 651 | for name in sorted(os.listdir(CONTENT)): |
| 652 | stem, ext = os.path.splitext(name) |
| 653 | if ext not in (".md", ".html") or stem.startswith("_"): |
| 654 | continue |
| 655 | with open(os.path.join(CONTENT, name), encoding="utf-8") as f: |
| 656 | text = f.read() |
| 657 | if ext == ".md": |
| 658 | md = markdown.Markdown(extensions=["meta", "tables", "fenced_code", "sane_lists"], output_format="html") |
| 659 | body = md.convert(text) |
| 660 | title = " ".join(md.Meta.get("title", [stem])) |
| 661 | else: |
| 662 | m = re.match(r"<!-- title: (.*?) -->\n", text) |
| 663 | title, body = (m.group(1), text[m.end():]) if m else (stem, text) |
| 664 | write(os.path.join(SITE, stem, "index.html"), page(title, body, section=stem)) |
| 665 | # Images beside the page that it mentions (``) are copied next to its index.html. |
| 666 | for img in sorted(os.listdir(CONTENT)): |
| 667 | if img.lower().endswith(SHOT_EXTENSIONS) and img in text: |
| 668 | shutil.copyfile(os.path.join(CONTENT, img), os.path.join(SITE, stem, img)) |
| 669 | write(os.path.join(SITE, "404.html"), page("Not found", ( |
| 670 | "<h1>Not found</h1><p>There's nothing at this address. Private repositories aren't listed here, " |
| 671 | 'and only the default branch can be browsed.</p><p><a href="/">All repositories</a></p>'), section="-")) |
| 672 | |
| 673 | |
| 674 | # ---------------------------------------------------------------- highlighting |
| 675 | |
| 676 | # Pygments token types folded into a few CSS classes (colors in style.css). The most |
| 677 | # specific match wins, so Name.Function gets "nf" and other names get nothing. |
| 678 | _TOKEN_CLASSES = [] |
| 679 | if pygments_token: |
| 680 | T = pygments_token |
| 681 | _TOKEN_CLASSES = [(T.Comment.Preproc, "cp"), (T.Comment, "c"), (T.Keyword, "k"), (T.String, "s"), |
| 682 | (T.Number, "m"), (T.Name.Function, "nf"), (T.Name.Class, "nc"), (T.Name.Builtin, "nb"), |
| 683 | (T.Name.Tag, "nt"), (T.Name.Attribute, "na"), (T.Name.Decorator, "nd"), |
| 684 | (T.Generic.Inserted, "gi"), (T.Generic.Deleted, "gd"), (T.Generic.Heading, "gh"), |
| 685 | (T.Error, "")] |
| 686 | _class_cache = {} |
| 687 | |
| 688 | |
| 689 | def token_class(ttype): |
| 690 | if ttype not in _class_cache: |
| 691 | _class_cache[ttype] = next((c for t, c in _TOKEN_CLASSES if ttype in t), "") |
| 692 | return _class_cache[ttype] |
| 693 | |
| 694 | |
| 695 | def highlight_lines(path, text): |
| 696 | """The text as escaped HTML lines, colored when Pygments knows the file type.""" |
| 697 | lines = text.split("\n") |
| 698 | if lines and lines[-1] == "": |
| 699 | lines.pop() |
| 700 | plain = [esc(l) for l in lines] |
| 701 | if not pygments_lexers or len(text) > MAX_HIGHLIGHT_BYTES: |
| 702 | return plain |
| 703 | try: |
| 704 | lexer = pygments_lexers.get_lexer_for_filename(posixpath.basename(path), stripnl=False, ensurenl=False) |
| 705 | except Exception: # ClassNotFound, or a lexer that fails to load |
| 706 | return plain |
| 707 | out, cur = [], [] |
| 708 | try: |
| 709 | for ttype, value in lexer.get_tokens(text): |
| 710 | cls = token_class(ttype) |
| 711 | for i, part in enumerate(value.split("\n")): |
| 712 | if i: |
| 713 | out.append("".join(cur)) |
| 714 | cur = [] |
| 715 | if part: |
| 716 | cur.append(f'<span class="{cls}">{esc(part)}</span>' if cls else esc(part)) |
| 717 | except Exception: # a lexer bug must not stop the page |
| 718 | return plain |
| 719 | out.append("".join(cur)) |
| 720 | # Pygments turns \r\n into \n; if that changed the line count, fall back to the plain lines. |
| 721 | while len(out) > len(lines) and out[-1] == "": |
| 722 | out.pop() |
| 723 | return out if len(out) == len(lines) else plain |
| 724 | |
| 725 | |
| 726 | def source_table(lines, blame=None): |
| 727 | """Numbered source lines; with blame, a column naming the commit behind each run of lines.""" |
| 728 | rows = [] |
| 729 | for i, code in enumerate(lines, 1): |
| 730 | cell = "" |
| 731 | if blame is not None: |
| 732 | b = blame[i - 1] if i - 1 < len(blame) else None |
| 733 | starts = b and (i == 1 or blame[i - 2] is None or blame[i - 2]["sha"] != b["sha"]) |
| 734 | cell = f'<td class="blame">{b["html"] if starts else ""}</td>' |
| 735 | run = ' class="run"' if starts and i > 1 else "" |
| 736 | rows.append(f'<tr id="L{i}"{run}>{cell}' |
| 737 | f'<td class="ln"><a href="#L{i}">{i}</a></td><td class="code">{code or " "}</td></tr>') |
| 738 | else: |
| 739 | rows.append(f'<tr id="L{i}"><td class="ln"><a href="#L{i}">{i}</a></td><td class="code">{code or " "}</td></tr>') |
| 740 | return f'<table class="source{" blamed" if blame is not None else ""}">{"".join(rows)}</table>' |
| 741 | |
| 742 | |
| 743 | # ---------------------------------------------------------------- repo pages |
| 744 | |
| 745 | def tree_entries(repo, ref): |
| 746 | """Files at `ref` as {path: (mode, sha, size)}; submodules get size -1.""" |
| 747 | out = {} |
| 748 | data = git(repo, "ls-tree", "-r", "-l", "-z", ref) |
| 749 | for rec in data.split(b"\0"): |
| 750 | if not rec: |
| 751 | continue |
| 752 | meta, path = rec.split(b"\t", 1) |
| 753 | mode, typ, sha, size = meta.decode().split() |
| 754 | p = path.decode("utf-8", errors="replace") |
| 755 | if not safe_path(p): |
| 756 | continue |
| 757 | out[p] = (mode, sha, -1 if typ == "commit" else int(size)) |
| 758 | return out |
| 759 | |
| 760 | |
| 761 | def all_dirs(files): |
| 762 | dirs = set() |
| 763 | for p in files: |
| 764 | d = posixpath.dirname(p) |
| 765 | while d: |
| 766 | dirs.add(d) |
| 767 | d = posixpath.dirname(d) |
| 768 | return dirs |
| 769 | |
| 770 | |
| 771 | def branch_note(view): |
| 772 | if view.is_default: |
| 773 | return "" |
| 774 | return (f'<p class="branch-note">Branch <code>{esc(view.branch)}</code> · ' |
| 775 | f'<a href="{url(view.name, "tree")}/">default branch</a> · <a href="{url(view.name, "refs")}/">all branches</a></p>') |
| 776 | |
| 777 | |
| 778 | def crumbs_for(view, path): |
| 779 | crumbs = [f'<a href="{url(view.name)}/">{esc(view.name)}</a>'] |
| 780 | if not view.is_default: |
| 781 | crumbs.append(f"<code>{esc(view.branch)}</code>") |
| 782 | crumbs.append(f'<a href="{view.url("tree")}/">files</a>') |
| 783 | parts = path.split("/") if path else [] |
| 784 | for i, part in enumerate(parts): |
| 785 | sub = "/".join(parts[: i + 1]) |
| 786 | if i == len(parts) - 1: |
| 787 | crumbs.append(f"<strong>{esc(part)}</strong>") |
| 788 | else: |
| 789 | crumbs.append(f'<a href="{view.url("tree", sub)}/">{esc(part)}</a>') |
| 790 | return crumbs |
| 791 | |
| 792 | |
| 793 | def build_tree_page(view, d, files, dirs): |
| 794 | prefix = d + "/" if d else "" |
| 795 | subdirs = sorted({p[len(prefix):].split("/")[0] for p in dirs if p.startswith(prefix) and p != d and "/" not in p[len(prefix):]}) |
| 796 | here = sorted(p for p in files if posixpath.dirname(p) == d) |
| 797 | rows = [] |
| 798 | if d: |
| 799 | parent = posixpath.dirname(d) |
| 800 | rows.append(f'<tr><td><a href="{view.url("tree", parent)}/">..</a></td><td></td></tr>') |
| 801 | for s in subdirs: |
| 802 | rows.append(f'<tr><td class="dir"><a href="{view.url("tree", prefix + s)}/">{esc(s)}/</a></td><td></td></tr>') |
| 803 | for p in here: |
| 804 | size = files[p][2] |
| 805 | label = "submodule" if size < 0 else human_size(size) |
| 806 | rows.append(f'<tr><td><a href="{view.url("blob", p)}.html">{esc(posixpath.basename(p))}</a></td><td class="num">{label}</td></tr>') |
| 807 | title = d or (view.name if view.is_default else view.branch) |
| 808 | body = f'{branch_note(view)}<h1>{esc(title)}</h1>\n<table class="files">{"".join(rows)}</table>' |
| 809 | write(view.path("tree", d, "index.html"), |
| 810 | page(f"{view.name}/{d}" if d else f"{view.name} files", body, crumbs_for(view, d) if d or not view.is_default else None, view=view)) |
| 811 | |
| 812 | |
| 813 | def blame_lines(repo, view, path, commit_set): |
| 814 | """[{sha, html}] per line of the file at the view's branch, from `git blame --porcelain`.""" |
| 815 | data = git(repo, "blame", "--porcelain", view.ref, "--", path, check=False) |
| 816 | info, out, cur = {}, [], None |
| 817 | for raw in data.split(b"\n"): |
| 818 | if raw.startswith(b"\t"): |
| 819 | out.append(cur) |
| 820 | continue |
| 821 | head = raw.split(b" ", 1)[0] |
| 822 | if len(head) == 40 and re.fullmatch(rb"[0-9a-f]{40}", head): |
| 823 | cur = head.decode() |
| 824 | info.setdefault(cur, {}) |
| 825 | elif cur and raw: |
| 826 | k, _, v = raw.decode("utf-8", errors="replace").partition(" ") |
| 827 | info[cur].setdefault(k, v) |
| 828 | cells = {} |
| 829 | for sha, i in info.items(): |
| 830 | when = time.strftime("%Y-%m-%d", time.gmtime(int(i.get("author-time", "0") or 0))) |
| 831 | short = f'<code>{sha[:8]}</code>' |
| 832 | link = f'<a href="{url(view.name, "commit", sha)}.html" title="{esc(i.get("summary", ""), quote=True)}">{short}</a>' if sha in commit_set else short |
| 833 | cells[sha] = {"sha": sha, "html": f'{link} <span class="when">{when}</span> <span class="who">{esc(i.get("author", ""))}</span>'} |
| 834 | return [cells.get(s) for s in out] |
| 835 | |
| 836 | |
| 837 | def build_blob_page(repo, view, path, entry, blobs, ctx, raw_set, commit_set): |
| 838 | """The file page, plus its blame page for text files. Returns True if a blame page was written.""" |
| 839 | mode, sha, size = entry |
| 840 | title = f"{view.name}/{path}" |
| 841 | crumbs = crumbs_for(view, path) |
| 842 | blamed = False |
| 843 | links = f' · <a href="{view.url("history", path)}.html">history</a>' |
| 844 | if size < 0: |
| 845 | body = f"<h1>{esc(posixpath.basename(path))}</h1><p>Git submodule at commit <code>{esc(sha)}</code>.</p>" |
| 846 | else: |
| 847 | data = blobs.read(sha) |
| 848 | if path in raw_set: |
| 849 | links += f' · <a href="{view.url("raw", path)}">raw</a>' |
| 850 | ext = posixpath.splitext(path)[1].lower() |
| 851 | text = None |
| 852 | if path in raw_set: |
| 853 | content = f'<p><img class="blob" src="{view.url("raw", path)}" alt="{esc(posixpath.basename(path))}"></p>' |
| 854 | elif is_binary(data): |
| 855 | content = "<p>Binary file, not shown. Clone the repository to get it.</p>" |
| 856 | elif size > MAX_TEXT_BYTES: |
| 857 | content = "<p>File too large to show here. Clone the repository to get it.</p>" |
| 858 | else: |
| 859 | text = data.decode("utf-8", errors="replace") |
| 860 | lines = highlight_lines(path, text) |
| 861 | source = source_table(lines) |
| 862 | if ext in (".md", ".markdown") and markdown: |
| 863 | ctx.base_dir = posixpath.dirname(path) |
| 864 | content = (f'<article class="markdown">{render_markdown(text, ctx)}</article>' |
| 865 | f"<details><summary>Source</summary>{source}</details>") |
| 866 | else: |
| 867 | content = source |
| 868 | if size <= MAX_BLAME_BYTES: |
| 869 | blame = blame_lines(repo, view, path, commit_set) |
| 870 | if blame: |
| 871 | links += f' · <a href="{view.url("blame", path)}.html">blame</a>' |
| 872 | bbody = (f'{branch_note(view)}<h1>{esc(posixpath.basename(path))}</h1>' |
| 873 | f'<p class="meta">Blame: the commit that last changed each line · ' |
| 874 | f'<a href="{view.url("blob", path)}.html">file</a> · ' |
| 875 | f'<a href="{view.url("history", path)}.html">history</a></p>{source_table(lines, blame)}') |
| 876 | write(view.path("blame", path + ".html"), page(f"{title} blame", bbody, crumbs, view=view)) |
| 877 | blamed = True |
| 878 | meta = f'<p class="meta">{human_size(size)}{links}</p>' |
| 879 | body = f"{branch_note(view)}<h1>{esc(posixpath.basename(path))}</h1>{meta}{content}" |
| 880 | if not blamed: |
| 881 | remove(view.path("blame", path + ".html")) |
| 882 | write(view.path("blob", path + ".html"), page(title, body, crumbs, view=view)) |
| 883 | return blamed |
| 884 | |
| 885 | |
| 886 | COMMIT_FORMAT = "%H%x1f%h%x1f%an%x1f%ad%x1f%s" |
| 887 | |
| 888 | |
| 889 | def commit_list(repo, ref, limit): |
| 890 | raw = git_text(repo, "log", f"-n{limit}", "--date=short", f"--format={COMMIT_FORMAT}", ref, check=False) |
| 891 | return [line.split("\x1f") for line in raw.splitlines() if line.count("\x1f") == 4] |
| 892 | |
| 893 | |
| 894 | def file_histories(repo, ref, paths): |
| 895 | """{path: [commit, ...]} newest first, at most HISTORY_LIMIT each, from one `git log` pass.""" |
| 896 | raw = git(repo, "-c", "core.quotePath=false", "log", "--no-renames", "--name-only", "--date=short", |
| 897 | f"-n{COMMIT_LIMIT}", f"--format=%x1e{COMMIT_FORMAT}", ref, check=False).decode("utf-8", errors="replace") |
| 898 | hist = {} |
| 899 | for chunk in raw.split("\x1e"): |
| 900 | lines = chunk.split("\n") |
| 901 | commit = lines[0].split("\x1f") |
| 902 | if len(commit) != 5: |
| 903 | continue |
| 904 | for p in lines[1:]: |
| 905 | if p in paths: |
| 906 | h = hist.setdefault(p, []) |
| 907 | if len(h) < HISTORY_LIMIT: |
| 908 | h.append(commit) |
| 909 | return hist |
| 910 | |
| 911 | |
| 912 | # Commits written with Claude Code carry the session's running token tally as git trailers: |
| 913 | # FW-Tokens-In: 10638 |
| 914 | # FW-Tokens-Out: 101300 |
| 915 | # FW-Tokens-Cached: 5110043 |
| 916 | # FW-Tokens-Method: 2 (counting method; absent = 1, which counted duplicate transcript lines and |
| 917 | # missed subagents, so those tallies run roughly 1.5-2x high) |
| 918 | # FW-Model-Tokens: claude-opus-5-5 in=40 out=20311 read=4100233 write5m=0 write1h=88120 (one per model) |
| 919 | # FW-Session: <session id> (provenance only: a resumed session may change it while the tally goes on) |
| 920 | # Older commits have one rounded line instead: "Usage: 10.6K in / 101.3K out / 5.1M cached (session tally)". |
| 921 | # A commit's own cost is the step from the previous commit's tally. The tally falls when a new |
| 922 | # session starts, and that commit's cost is its whole tally. Dollar cost needs the per-model lines and |
| 923 | # prices.json; without them the cost is unknown, never zero. |
| 924 | USAGE_KEYS = ("FW-Tokens-In", "FW-Tokens-Out", "FW-Tokens-Cached") |
| 925 | USAGE_RE = re.compile(r"^Usage:\s*([\d.]+)([KMG]?) in / ([\d.]+)([KMG]?) out / ([\d.]+)([KMG]?) cached", re.M) |
| 926 | MODEL_TOKENS_RE = re.compile(r"^FW-Model-Tokens:\s*(\S+)\s+in=(\d+)\s+out=(\d+)\s+read=(\d+)\s+write5m=(\d+)\s+write1h=(\d+)\s*$", re.M) |
| 927 | AI_COAUTHOR_RE = re.compile(r"^Co-Authored-By:.*(noreply@anthropic\.com|\bClaude\b|Copilot|ChatGPT|openai)", re.M | re.I) |
| 928 | USAGE_UNITS = {"": 1, "K": 10**3, "M": 10**6, "G": 10**9} |
| 929 | USAGE_HOT = 2 # a commit with this many times the median output is highlighted |
| 930 | OVER_100K = "@over100k" # model-key suffix for requests whose prompt exceeded 100,000 tokens |
| 931 | MAX_WEEKS = 104 # a longer history is charted by month |
| 932 | |
| 933 | |
| 934 | def parse_usage(message): |
| 935 | """([in, out, cached], [slack, ...]) from a message's usage trailers, or None. |
| 936 | |
| 937 | Trailer counts are exact. The old Usage: line is rounded to a tenth of its unit, so each count |
| 938 | is only known to within half a tenth: the slack. |
| 939 | """ |
| 940 | counts = [re.search(rf"^{key}:\s*(\d+)\s*$", message, re.M) for key in USAGE_KEYS] |
| 941 | if all(counts): |
| 942 | return [float(m.group(1)) for m in counts], [0, 0, 0] |
| 943 | m = USAGE_RE.search(message) |
| 944 | if not m: |
| 945 | return None |
| 946 | units = [USAGE_UNITS[m.group(i + 1)] for i in (1, 3, 5)] |
| 947 | return [float(m.group(i)) * u for i, u in zip((1, 3, 5), units)], [u / 20 for u in units] |
| 948 | |
| 949 | |
| 950 | def parse_model_usage(message): |
| 951 | """{model: [in, out, read, write5m, write1h]} from FW-Model-Tokens trailers, or None (older commits).""" |
| 952 | found = MODEL_TOKENS_RE.findall(message) |
| 953 | if not found: |
| 954 | return None |
| 955 | models = {} |
| 956 | for m in found: |
| 957 | cur = models.setdefault(m[0], [0] * 5) |
| 958 | models[m[0]] = [a + int(b) for a, b in zip(cur, m[1:])] |
| 959 | return models |
| 960 | |
| 961 | |
| 962 | def usage_method(message): |
| 963 | """The FW-Tokens-Method trailer as an int; commits without it used method 1.""" |
| 964 | m = re.search(r"^FW-Tokens-Method:\s*(\d+)\s*$", message, re.M) |
| 965 | return int(m.group(1)) if m else 1 |
| 966 | |
| 967 | |
| 968 | _prices = None |
| 969 | |
| 970 | |
| 971 | def load_prices(): |
| 972 | """server/prices.json beside this file (USD per 1M tokens), read once; {} if missing or broken.""" |
| 973 | global _prices |
| 974 | if _prices is None: |
| 975 | try: |
| 976 | with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "prices.json")) as f: |
| 977 | _prices = json.load(f) |
| 978 | except (OSError, ValueError): |
| 979 | _prices = {} |
| 980 | return _prices |
| 981 | |
| 982 | |
| 983 | def commit_cost(model_delta, prices): |
| 984 | """(dollars, unknown_models): the list-price estimate of a per-model delta, or (None, [...]) when it is |
| 985 | unknown: no model data, or a model with tokens that the price table lacks. Keys ending '@over100k' |
| 986 | use that model's above_100k rates.""" |
| 987 | if not model_delta: |
| 988 | return None, [] |
| 989 | models = prices.get("models", {}) |
| 990 | total, unknown = 0.0, [] |
| 991 | for key, (inp, out, read, w5, w1) in model_delta.items(): |
| 992 | if not any((inp, out, read, w5, w1)): |
| 993 | continue |
| 994 | base = key[:-len(OVER_100K)] if key.endswith(OVER_100K) else key |
| 995 | rates = models.get(base, {}) |
| 996 | if key.endswith(OVER_100K): |
| 997 | rates = rates.get("above_100k") |
| 998 | if not rates: |
| 999 | unknown.append(key) |
| 1000 | continue |
| 1001 | total += (inp * rates["input"] + out * rates["output"] + read * rates["cache_read"] |
| 1002 | + w5 * rates["cache_write_5m"] + w1 * rates["cache_write_1h"]) / 1e6 |
| 1003 | return (None, unknown) if unknown else (total, []) |
| 1004 | |
| 1005 | |
| 1006 | def fmt_tokens(n): |
| 1007 | """10638 -> '10.6K': the units the trailer is written in.""" |
| 1008 | for unit, suffix in ((10**9, "G"), (10**6, "M"), (10**3, "K")): |
| 1009 | if n >= unit: |
| 1010 | return f"{n / unit:.1f}{suffix}" |
| 1011 | return str(round(n)) |
| 1012 | |
| 1013 | |
| 1014 | def approx_tokens(n, slack): |
| 1015 | return "≈0" if n <= slack else ("~" if slack else "") + fmt_tokens(n) |
| 1016 | |
| 1017 | |
| 1018 | def usage_costs(repo, ref): |
| 1019 | """(costs, total, commits) for `ref`'s first-parent chain: see compute_costs.""" |
| 1020 | raw = git_text(repo, "log", "--first-parent", "--reverse", "--date=short", |
| 1021 | f"--format={COMMIT_FORMAT}%x1f%B%x1e", ref, check=False) |
| 1022 | parsed = [] |
| 1023 | for chunk in raw.split("\x1e"): |
| 1024 | fields = chunk.strip("\n").split("\x1f", 5) |
| 1025 | if len(fields) == 6: |
| 1026 | parsed.append((fields[:5], fields[5])) |
| 1027 | return compute_costs(parsed) |
| 1028 | |
| 1029 | |
| 1030 | def models_restarted(prev, cur): |
| 1031 | """True when per-model tallies cannot continue one session: a counter fell, a model with tokens vanished |
| 1032 | (counters never disappear within a session), or model data appears where the previous commit had none.""" |
| 1033 | if prev is None: |
| 1034 | return cur is not None |
| 1035 | if cur is None: |
| 1036 | return False |
| 1037 | for m, v in prev.items(): |
| 1038 | now = cur.get(m) |
| 1039 | if now is None: |
| 1040 | if any(v): |
| 1041 | return True |
| 1042 | elif any(n < o for n, o in zip(now, v)): |
| 1043 | return True |
| 1044 | return False |
| 1045 | |
| 1046 | |
| 1047 | def compute_costs(commits, prices=None): |
| 1048 | """commits: [(fields, message)] oldest first, fields = COMMIT_FORMAT's five values (sha, short, author, |
| 1049 | date, subject). Returns (costs, total, allc): |
| 1050 | costs {sha: {"row", "tally", "delta", "slack", "session", "new", "method", "models", "cost", "unknown"}} |
| 1051 | for each commit with a usage trailer. `models` is the per-model delta (None for old commits), `cost` |
| 1052 | its list-price estimate in dollars (None = unknown), `unknown` the models that could not be priced. |
| 1053 | total the number of commits |
| 1054 | allc [{"sha", "date", "ai", "has_tokens"}] for every commit, so trailer-less ones can be bucketed. |
| 1055 | A new session starts (delta = whole tally) at the first trailer commit, when the counting method |
| 1056 | changes, when the output or cached tally falls, when any per-model counter falls or a model with tokens |
| 1057 | vanishes, or when model data appears after commits that had none.""" |
| 1058 | prices = load_prices() if prices is None else prices |
| 1059 | costs, allc, prev, session = {}, [], None, 0 |
| 1060 | for fields, message in commits: |
| 1061 | parsed = parse_usage(message) |
| 1062 | allc.append({"sha": fields[0], "date": fields[3], "has_tokens": bool(parsed), |
| 1063 | "ai": bool(parsed) or bool(AI_COAUTHOR_RE.search(message))}) |
| 1064 | if not parsed: |
| 1065 | continue |
| 1066 | tally, slack = parsed |
| 1067 | method, tmodels = usage_method(message), parse_model_usage(message) |
| 1068 | new = (prev is None or method != prev["method"] or tally[1] < prev["tally"][1] or tally[2] < prev["tally"][2] |
| 1069 | or models_restarted(prev["models"], tmodels)) |
| 1070 | if new: |
| 1071 | session += 1 |
| 1072 | delta, delta_slack = tally, slack |
| 1073 | mdelta = tmodels |
| 1074 | else: |
| 1075 | delta = [t - p for t, p in zip(tally, prev["tally"])] |
| 1076 | delta_slack = [s + p for s, p in zip(slack, prev["slack"])] |
| 1077 | mdelta = ({m: [a - b for a, b in zip(v, (prev["models"] or {}).get(m, [0] * 5))] for m, v in tmodels.items()} |
| 1078 | if tmodels is not None else None) |
| 1079 | cost, unknown = commit_cost(mdelta, prices) |
| 1080 | costs[fields[0]] = {"row": fields, "tally": tally, "delta": delta, "slack": delta_slack, |
| 1081 | "session": session, "new": new, "method": method, "models": mdelta, |
| 1082 | "cost": cost, "unknown": unknown} |
| 1083 | prev = {"tally": tally, "slack": slack, "method": method, "models": tmodels} |
| 1084 | return costs, len(allc), allc |
| 1085 | |
| 1086 | |
| 1087 | def week_buckets(allc, costs): |
| 1088 | """Commits grouped by Monday-started week of their author date, empty weeks included; by month |
| 1089 | when the history is longer than MAX_WEEKS weeks. Per bucket: label, commits, ai, with_tokens, |
| 1090 | with_cost, out, read, cost (sum over commits whose cost is known), nodata (commits without usage data).""" |
| 1091 | if not allc: |
| 1092 | return [] |
| 1093 | dates = [datetime.date.fromisoformat(c["date"]) for c in allc] |
| 1094 | first, last = min(dates), max(dates) |
| 1095 | monday = lambda d: d - datetime.timedelta(days=d.weekday()) |
| 1096 | by_month = (monday(last) - monday(first)).days // 7 + 1 > MAX_WEEKS |
| 1097 | if by_month: |
| 1098 | key = lambda d: (d.year, d.month) |
| 1099 | keys, y, m = [], first.year, first.month |
| 1100 | while (y, m) <= key(last): |
| 1101 | keys.append((y, m)) |
| 1102 | y, m = (y + 1, 1) if m == 12 else (y, m + 1) |
| 1103 | labels = [f"month of {y}-{m:02d}" for y, m in keys] |
| 1104 | else: |
| 1105 | key = monday |
| 1106 | keys = [monday(first) + datetime.timedelta(weeks=i) for i in range((monday(last) - monday(first)).days // 7 + 1)] |
| 1107 | labels = [f"week of {k.isoformat()}" for k in keys] |
| 1108 | buckets = [{"label": lb, "commits": 0, "ai": 0, "with_tokens": 0, "with_cost": 0, "out": 0, "read": 0, |
| 1109 | "cost": 0.0, "nodata": 0} for lb in labels] |
| 1110 | index = {k: i for i, k in enumerate(keys)} |
| 1111 | for c, d in zip(allc, dates): |
| 1112 | b = buckets[index[key(d)]] |
| 1113 | b["commits"] += 1 |
| 1114 | b["ai"] += c["ai"] |
| 1115 | e = costs.get(c["sha"]) |
| 1116 | if not e: |
| 1117 | b["nodata"] += 1 |
| 1118 | continue |
| 1119 | b["with_tokens"] += 1 |
| 1120 | b["out"] += e["delta"][1] |
| 1121 | b["read"] += e["delta"][2] |
| 1122 | if e["cost"] is not None: |
| 1123 | b["with_cost"] += 1 |
| 1124 | b["cost"] += e["cost"] |
| 1125 | return buckets |
| 1126 | |
| 1127 | |
| 1128 | def bucket_unit(buckets): |
| 1129 | return "month" if buckets and buckets[0]["label"].startswith("month") else "week" |
| 1130 | |
| 1131 | |
| 1132 | def line_chart_svg(labels, values, *, fmt, unknown, second=None, legend=None, aria, axis_fmt=fmt_tokens, |
| 1133 | pattern_id="hatch", tips=None, hatch_note="hatched = commits without usage data (unknown, not zero)"): |
| 1134 | """An inline SVG line chart without script. labels: one text per bucket, ending in its date (the axis shows |
| 1135 | the first and last); values: one number per bucket, or None where the value is unknown: the line breaks |
| 1136 | there (a gap), it never drops to zero, and a single isolated point is only a dot; fmt(value) -> text for the |
| 1137 | tooltip; unknown: per bucket (commits lacking data, commits), drawn as a hatched strip under the axis; |
| 1138 | second: optional second series (drawn in the accent colour, the first then grey); legend: [(text, "muted"|"")] |
| 1139 | drawn under the chart; axis_fmt formats the top axis label; pattern_id keeps the hatch pattern's id unique when |
| 1140 | a page has several charts; tips: optional full tooltip text per bucket instead of the default; hatch_note: the |
| 1141 | line under the axis ("" for none). Every bucket has a transparent hover target with a <title>, every known |
| 1142 | point a dot with the same <title>. Colours come from CSS classes only.""" |
| 1143 | if not labels: |
| 1144 | return "" |
| 1145 | left, right, top, base = 44, 716, 12, 140 |
| 1146 | n = len(values) |
| 1147 | known = [v for v in list(values) + list(second or []) if v is not None] |
| 1148 | vmax = max(known, default=0) or 1 |
| 1149 | step = (right - left) / max(n, 1) |
| 1150 | x = lambda i: left + (i + 0.5) * step |
| 1151 | y = lambda v: base - (base - top) * v / vmax |
| 1152 | tip_of = [] |
| 1153 | for i, v in enumerate(values): |
| 1154 | bad, total = unknown[i] |
| 1155 | tip_of.append(tips[i] if tips else ( |
| 1156 | f"{labels[i]}: " + ("no data" if v is None else fmt(v)) |
| 1157 | + (f", {total - bad} of {total} commits with data" if total else ", no commits"))) |
| 1158 | hits = "".join(f'<rect class="hit" x="{left + i * step:.1f}" y="{top}" width="{step:.1f}" height="{base - top}">' |
| 1159 | f'<title>{esc(tip_of[i])}</title></rect>' for i in range(n)) |
| 1160 | |
| 1161 | def series(vals, muted): |
| 1162 | cls = " muted" if muted else "" |
| 1163 | segments, cur = [], [] |
| 1164 | for i, v in enumerate(vals): |
| 1165 | if v is None: |
| 1166 | if cur: |
| 1167 | segments.append(cur) |
| 1168 | cur = [] |
| 1169 | else: |
| 1170 | cur.append((i, v)) |
| 1171 | if cur: |
| 1172 | segments.append(cur) |
| 1173 | out = [] |
| 1174 | for seg in segments: |
| 1175 | if len(seg) > 1: |
| 1176 | pts = " L".join(f"{x(i):.1f} {y(v):.1f}" for i, v in seg) |
| 1177 | out.append(f'<path class="area{cls}" d="M{x(seg[0][0]):.1f} {base} L{pts} L{x(seg[-1][0]):.1f} {base} Z"/>') |
| 1178 | out.append(f'<path class="line{cls}" d="M{pts}"/>') |
| 1179 | out += [f'<circle class="dot{cls}" cx="{x(i):.1f}" cy="{y(v):.1f}" r="3"><title>{esc(tip_of[i])}</title></circle>' |
| 1180 | for seg in segments for i, v in seg] |
| 1181 | return "".join(out) |
| 1182 | |
| 1183 | parts = [series(values, second is not None)] + ([series(second, False)] if second is not None else []) |
| 1184 | bad_strips = "".join( |
| 1185 | f'<rect class="unk" x="{left + i * step:.1f}" y="{base + 4}" width="{max(step - 1, 1):.1f}" height="6" ' |
| 1186 | f'fill="url(#{pattern_id})"><title>{esc(tip_of[i])}</title></rect>' for i in range(n) if unknown[i][0]) |
| 1187 | first, last = labels[0].rsplit(" ", 1)[-1], labels[-1].rsplit(" ", 1)[-1] |
| 1188 | key = "" |
| 1189 | for j, (text, cls) in enumerate(legend or []): |
| 1190 | kx = left + j * 260 |
| 1191 | key += (f'<line class="line {cls}" x1="{kx}" y1="189" x2="{kx + 28}" y2="189"/>' |
| 1192 | f'<text x="{kx + 34}" y="194" text-anchor="start">{esc(text)}</text>') |
| 1193 | return (f'<svg class="chart" viewBox="0 0 720 200" role="img" aria-label="{esc(aria)}">' |
| 1194 | f'<defs><pattern id="{pattern_id}" width="6" height="6" patternUnits="userSpaceOnUse">' |
| 1195 | '<path class="hatchline" d="M0 6L6 0"/></pattern></defs>' |
| 1196 | f'<line class="axis" x1="{left}" y1="{base}" x2="{right}" y2="{base}"/>' |
| 1197 | + "".join(parts) + bad_strips + hits |
| 1198 | + f'<text x="{left - 4}" y="{base}" text-anchor="end">0</text>' |
| 1199 | f'<text x="{left - 4}" y="{top + 8}" text-anchor="end">{esc(axis_fmt(vmax))}</text>' |
| 1200 | f'<text x="{left}" y="{base + 28}" text-anchor="start">{esc(first)}</text>' |
| 1201 | f'<text x="{right}" y="{base + 28}" text-anchor="end">{esc(last)}</text>' |
| 1202 | + (f'<text x="{left}" y="194" text-anchor="start">{esc(hatch_note)}</text>' if hatch_note else "") |
| 1203 | + key + '</svg>') |
| 1204 | |
| 1205 | |
| 1206 | COST_NOTE = "estimated API list-price equivalent, not what was paid" |
| 1207 | |
| 1208 | |
| 1209 | def usage_chart(title, caption, labels, values, *, pattern_id, empty, **kw): |
| 1210 | """One chart section: heading, one-line caption, and the SVG; or `empty` as a message when no bucket has a |
| 1211 | known value, so a chart never shows an empty axis. A value of None means unknown (a gap in the line); a known |
| 1212 | 0 is a real value and still gets a chart. The one function every Usage chart goes through.""" |
| 1213 | head = f"<h2>{esc(title)}</h2><p class=\"meta\">{caption}</p>" |
| 1214 | if not labels or all(v is None for v in values): |
| 1215 | return head + f'<p class="meta">{esc(empty)}</p>' |
| 1216 | return head + line_chart_svg(labels, values, pattern_id=pattern_id, **kw) |
| 1217 | |
| 1218 | |
| 1219 | def usage_charts(buckets, prefix): |
| 1220 | """The Usage chart sections (output tokens, est. cost, cached reads, cost per commit, AI-marked commits) for |
| 1221 | `buckets` from week_buckets, as HTML. `prefix` keeps the SVG pattern ids unique per page.""" |
| 1222 | unit = bucket_unit(buckets) |
| 1223 | labels = [b["label"] for b in buckets] |
| 1224 | nodata = [(b["nodata"], b["commits"]) for b in buckets] |
| 1225 | nocost = [(b["commits"] - b["with_cost"], b["commits"]) for b in buckets] |
| 1226 | money = lambda v: f"${v:,.2f}" |
| 1227 | cost_note = "hatched = commits without a known cost (unknown, not zero)" |
| 1228 | per_commit = [b["cost"] / b["with_cost"] if b["with_cost"] else None for b in buckets] |
| 1229 | known = lambda b, key, n: b[key] if b[n] else None # None = no commit has the data: unknown, not zero |
| 1230 | out = usage_chart( |
| 1231 | f"Output tokens per {unit}", f"Output tokens of commits with usage trailers; the main measure of work done.", |
| 1232 | labels, [known(b, "out", "with_tokens") for b in buckets], pattern_id=prefix + "-out", empty="No output tokens recorded yet.", |
| 1233 | fmt=lambda v: f"{fmt_tokens(v)} output tokens", unknown=nodata, |
| 1234 | aria=f"Output tokens per {unit}, from commits with usage trailers; hatched marks commits without usage data") |
| 1235 | cost = usage_chart( |
| 1236 | f"Estimated cost per {unit}", f"Sum of known per-commit costs: {COST_NOTE}. Commits without a known cost are " |
| 1237 | "left out, not counted as zero.", labels, [known(b, "cost", "with_cost") for b in buckets], pattern_id=prefix + "-cost", |
| 1238 | empty="No commit has a known cost yet (cost unknown, not zero).", fmt=lambda v: f"est. {money(v)} ({COST_NOTE})", |
| 1239 | tips=[f"{b['label']}: est. {money(b['cost'])} ({COST_NOTE}), {b['with_cost']} of {b['commits']} commits with a known cost" |
| 1240 | if b["with_cost"] else f"{b['label']}: " + ("cost unknown, 0 commits with a known cost" if b["commits"] else "no commits") |
| 1241 | for b in buckets], |
| 1242 | unknown=nocost, axis_fmt=money, hatch_note=cost_note, |
| 1243 | aria=f"Estimated cost per {unit}, {COST_NOTE}; hatched marks commits whose cost is unknown") |
| 1244 | cached = usage_chart( |
| 1245 | f"Cached reads per {unit}", "Tokens read from the prompt cache. They grow with context length and turns, so " |
| 1246 | "they track session length more than work done.", labels, [known(b, "read", "with_tokens") for b in buckets], |
| 1247 | pattern_id=prefix + "-read", empty="No cached reads recorded yet.", |
| 1248 | fmt=lambda v: f"{fmt_tokens(v)} cached reads", unknown=nodata, |
| 1249 | aria=f"Cached reads per {unit}, from commits with usage trailers; hatched marks commits without usage data") |
| 1250 | tips = [f"{b['label']}: est. {money(v)} per commit ({COST_NOTE}), n={plural(b['with_cost'], 'commit')} with a known cost" |
| 1251 | if b["with_cost"] else f"{b['label']}: " + ("cost unknown, n=0 commits with a known cost" if b["commits"] else "no commits") |
| 1252 | for b, v in zip(buckets, per_commit)] |
| 1253 | each = usage_chart( |
| 1254 | f"Cost per commit, average per {unit}", f"Known cost divided by the commits with a known cost that {unit} " |
| 1255 | f"(n is in the tooltip): {COST_NOTE}. A commit's cost includes dead ends since the previous trailer.", |
| 1256 | labels, per_commit, pattern_id=prefix + "-each", empty="No commit has a known cost yet (cost unknown, not zero).", |
| 1257 | fmt=lambda v: money(v), unknown=nocost, axis_fmt=money, hatch_note=cost_note, tips=tips, |
| 1258 | aria=f"Average estimated cost per commit per {unit}, {COST_NOTE}; hatched marks commits whose cost is unknown") |
| 1259 | ai_tips = [f"{b['label']}: {plural(b['commits'], 'commit')}, {b['ai']} AI-marked, {b['commits'] - b['ai']} with no AI marker" |
| 1260 | for b in buckets] |
| 1261 | ai = usage_chart( |
| 1262 | f"AI-marked commits per {unit}", "Grey line: all commits. Blue line: commits with an AI marker (an AI " |
| 1263 | "Co-Authored-By trailer, or a usage trailer). The rest have no AI marker, which does not mean a human " |
| 1264 | "wrote them: a marker is only there when the tool added it.", labels, [b["commits"] for b in buckets], |
| 1265 | pattern_id=prefix + "-ai", empty="No commits yet.", fmt=lambda v: f"{v} commits", |
| 1266 | unknown=[(0, b["commits"]) for b in buckets], second=[b["ai"] for b in buckets], |
| 1267 | legend=[("All commits", "muted"), ("AI-marked commits", "")], hatch_note="", tips=ai_tips, |
| 1268 | aria=f"Commits per {unit}: one line for all commits, one for the AI-marked ones; the rest have no AI marker") |
| 1269 | return out + cost + cached + each + ai |
| 1270 | |
| 1271 | |
| 1272 | _usage_cache = {} |
| 1273 | |
| 1274 | |
| 1275 | def repo_usage(name): |
| 1276 | """{"costs", "commits", "all", "median", "max", "dollars", "priced"} for a repo's default branch: |
| 1277 | output tokens per commit, and the summed list-price estimate over the `priced` commits whose cost is known.""" |
| 1278 | if name not in _usage_cache: |
| 1279 | costs, total, allc = usage_costs(os.path.join(GIT_ROOT, name + ".git"), "HEAD") |
| 1280 | outs = sorted(c["delta"][1] for c in costs.values()) |
| 1281 | known = [c["cost"] for c in costs.values() if c["cost"] is not None] |
| 1282 | _usage_cache[name] = {"costs": costs, "commits": total, "all": allc, |
| 1283 | "median": outs[len(outs) // 2] if outs else 0, "max": outs[-1] if outs else 0, |
| 1284 | "dollars": sum(known), "priced": len(known)} |
| 1285 | return _usage_cache[name] |
| 1286 | |
| 1287 | |
| 1288 | def cost_text(c): |
| 1289 | """The estimated dollar cost with its full label, or why the cost is unknown.""" |
| 1290 | if c["cost"] is not None: |
| 1291 | return (f"est. ${c['cost']:.2f} (API list-price equivalent at {load_prices().get('as_of', '?')} rates, " |
| 1292 | "not what was paid)") |
| 1293 | if c["unknown"]: |
| 1294 | return f"cost unknown (no price for {', '.join(c['unknown'])})" |
| 1295 | return "cost unknown (no per-model trailer)" |
| 1296 | |
| 1297 | |
| 1298 | def usage_cell(usage, sha): |
| 1299 | c = usage["costs"].get(sha) |
| 1300 | if not c: |
| 1301 | return '<td class="cost"></td>' |
| 1302 | out, slack = c["delta"][1], c["slack"][1] |
| 1303 | median = usage["median"] |
| 1304 | real = out > slack |
| 1305 | hot = real and median and out >= USAGE_HOT * median |
| 1306 | title = (f"{approx_tokens(out, slack)} output tokens, {approx_tokens(c['delta'][2], c['slack'][2])} cached reads" |
| 1307 | + (f" ({out / median:.1f}× the median commit)" if real and median else "") |
| 1308 | + f" · {cost_text(c)}" |
| 1309 | + (" · first commit of a session: counts everything since it began" if c["new"] else "")) |
| 1310 | width = 100 * out / usage["max"] if usage["max"] else 0 |
| 1311 | return (f'<td class="cost" title="{esc(title)}"><span class="track"><span class="bar{" hot" if hot else ""}" ' |
| 1312 | f'style="width:{width:.1f}%"></span></span>{approx_tokens(out, slack)}</td>') |
| 1313 | |
| 1314 | |
| 1315 | def usage_panel(name, sha): |
| 1316 | """Usage line for a commit page. Only facts fixed by the commit's own history: no rank, so the page never goes stale.""" |
| 1317 | c = repo_usage(name)["costs"].get(sha) |
| 1318 | if not c: |
| 1319 | return "" |
| 1320 | d, s, t = c["delta"], c["slack"], c["tally"] |
| 1321 | cost = f" · {esc(cost_text(c))}" |
| 1322 | return ('<p class="meta usage">Claude Code usage: this commit ' |
| 1323 | f'{approx_tokens(d[1], s[1])} out / {approx_tokens(d[2], s[2])} cached' |
| 1324 | + (" (first commit of its session: everything since it began)" if c["new"] else "") |
| 1325 | + cost |
| 1326 | + f' · session {c["session"]} tally {fmt_tokens(t[0])} in / {fmt_tokens(t[1])} out / {fmt_tokens(t[2])} cached' |
| 1327 | f' · <a href="{url(name, "usage")}/">usage by commit and session</a></p>') |
| 1328 | |
| 1329 | |
| 1330 | def commits_table(name, commits, commit_set=None, usage=None): |
| 1331 | def sha_cell(h, s): |
| 1332 | if commit_set is not None and h not in commit_set: |
| 1333 | return f"<code>{esc(s)}</code>" |
| 1334 | return f'<a href="{url(name, "commit", h)}.html"><code>{esc(s)}</code></a>' |
| 1335 | ci = ci_pipelines(name) |
| 1336 | rows = "".join(f'<tr><td class="date">{esc(d)}</td><td>{sha_cell(h, s)}</td>' |
| 1337 | f'<td>{ci_mark(name, h, ci.get(h))}{esc(subj)}</td><td class="author">{esc(a)}</td>' |
| 1338 | f'{usage_cell(usage, h) if usage else ""}</tr>' |
| 1339 | for h, s, a, d, subj in commits) |
| 1340 | return f'<table class="log">{rows}</table>' |
| 1341 | |
| 1342 | |
| 1343 | def plural(n, word): |
| 1344 | return f"{n} {word}" + ("" if n == 1 else "s") |
| 1345 | |
| 1346 | |
| 1347 | def cost_stat(dollars, priced, as_of): |
| 1348 | """('Estimated cost', html) for usage_stats: the summed estimate over the `priced` commits with a known cost, |
| 1349 | with the full label in a small note, or "unknown" (never $0) when no commit has one.""" |
| 1350 | if not priced: |
| 1351 | return "Estimated cost", "unknown, not zero (no commit has per-model trailers)" |
| 1352 | return "Estimated cost", (f'est. ${dollars:,.2f} over the {plural(priced, "commit")} with a known cost' |
| 1353 | f'<small>estimated API list-price equivalent at prices of {esc(as_of)}, not what was paid</small>') |
| 1354 | |
| 1355 | |
| 1356 | def usage_stats(items): |
| 1357 | """A compact list of facts, one per line: [(label, html value)]. Values are built by the caller (already escaped).""" |
| 1358 | return '<dl class="stats">' + "".join(f"<div><dt>{esc(k)}</dt><dd>{v}</dd></div>" for k, v in items) + "</dl>" |
| 1359 | |
| 1360 | |
| 1361 | def session_cost(cs): |
| 1362 | """Summed estimated cost of a session's commits, or None unless every commit has a known cost.""" |
| 1363 | return None if any(c["cost"] is None for c in cs) else sum(c["cost"] for c in cs) |
| 1364 | |
| 1365 | |
| 1366 | def session_cost_cell(cs): |
| 1367 | cost = session_cost(cs) |
| 1368 | if cost is None: |
| 1369 | return '<td class="num" title="at least one commit in this session has no known cost">unknown</td>' |
| 1370 | return (f'<td class="num" title="estimated API list-price equivalent at {esc(load_prices().get("as_of", "?"))} ' |
| 1371 | f'rates, not what was paid">est. ${cost:,.2f}</td>') |
| 1372 | |
| 1373 | |
| 1374 | def build_usage_page(view, commit_set): |
| 1375 | """/<repo>/usage/: commits by output tokens, and a table of sessions. Removed when no commit has a trailer.""" |
| 1376 | name = view.name |
| 1377 | usage = repo_usage(name) |
| 1378 | costs = usage["costs"] |
| 1379 | path = view.path("usage", "index.html") |
| 1380 | if not costs: |
| 1381 | remove(path) |
| 1382 | return |
| 1383 | sessions = {} |
| 1384 | for c in costs.values(): |
| 1385 | sessions.setdefault(c["session"], []).append(c) |
| 1386 | total_out = sum(c["delta"][1] for c in costs.values()) or 1 |
| 1387 | total_cached = sum(c["delta"][2] for c in costs.values()) |
| 1388 | session_rows = "".join( |
| 1389 | f'<tr><td class="num">{n}</td><td class="date">{esc(cs[0]["row"][3])}' |
| 1390 | + (f' – {esc(cs[-1]["row"][3])}' if cs[-1]["row"][3] != cs[0]["row"][3] else "") + "</td>" |
| 1391 | f'<td class="num">{len(cs)}</td><td class="num">{fmt_tokens(sum(c["delta"][1] for c in cs))}</td>' |
| 1392 | f'<td class="num">{fmt_tokens(sum(c["delta"][2] for c in cs))}</td>' |
| 1393 | f'<td class="num">{100 * sum(c["delta"][1] for c in cs) / total_out:.0f}%</td>' |
| 1394 | f'{session_cost_cell(cs)}</tr>' |
| 1395 | for n, cs in sessions.items()) |
| 1396 | ranked = sorted(costs.values(), key=lambda c: -c["delta"][1]) |
| 1397 | buckets = week_buckets(usage["all"], costs) |
| 1398 | charts = usage_charts(buckets, "u") |
| 1399 | as_of = load_prices().get("as_of", "?") |
| 1400 | stats = usage_stats([ |
| 1401 | ("Commits with usage data", f'{len(costs)} of {usage["commits"]}'), |
| 1402 | ("Sessions", str(len(sessions))), |
| 1403 | ("Output tokens", f"~{fmt_tokens(total_out)}"), |
| 1404 | ("Cached reads", f"~{fmt_tokens(total_cached)}"), |
| 1405 | cost_stat(usage["dollars"], usage["priced"], as_of)]) |
| 1406 | body = ('<h1>Claude Code usage</h1><p class="meta">Read from the <code>FW-Tokens-*</code> trailers in commit ' |
| 1407 | 'messages (or older <code>Usage:</code> lines).</p>' |
| 1408 | '<p class="meta"><a href="/usage/">Usage across all repositories</a></p>' |
| 1409 | + stats + |
| 1410 | f'{charts}' |
| 1411 | f'<h2>Commits by output tokens</h2>' |
| 1412 | f'{commits_table(name, [c["row"] for c in ranked[:30]], commit_set, usage)}' |
| 1413 | f'<h2>Sessions</h2><table class="log"><tr><th>#</th><th>Dates</th><th>Commits</th><th>Output</th>' |
| 1414 | f'<th>Cached</th><th>Share of output</th><th>Est. cost (not what was paid)</th></tr>{session_rows}</table>' |
| 1415 | '<h2>How this is measured</h2><ul>' |
| 1416 | '<li>A trailer is the running token tally of one Claude Code session. A commit\'s cost is the step ' |
| 1417 | 'from the previous commit\'s tally. A new session starts, and its first commit gets the whole tally, ' |
| 1418 | 'at the first trailer, when the counting method changes, when the output or cached total falls, ' |
| 1419 | 'when any model\'s counter falls, when a model that had tokens disappears, or when per-model ' |
| 1420 | 'lines appear after commits that had none.</li>' |
| 1421 | '<li>The bars and rankings in the commit table use output tokens. Cached reads grow with how long the context is and how many ' |
| 1422 | 'turns ran, so they track session length more than the work; input tokens are negligible. These are ' |
| 1423 | 'tokens; dollar figures are estimated API list-price equivalents, not what was paid.</li>' |
| 1424 | '<li>Costs need the per-model trailers (<code>FW-Model-Tokens</code>) and are estimates at list prices ' |
| 1425 | f'as of {esc(as_of)}. Commits without them show cost unknown, and unknown is not zero: it is left out of ' |
| 1426 | 'the cost total and drawn hatched in the chart.</li>' |
| 1427 | '<li>Trailers without <code>FW-Tokens-Method: 2</code> counted duplicate transcript lines ' |
| 1428 | '(roughly 1.5-2× too high) and missed subagents; they are shown as written.</li>' |
| 1429 | '<li>Tallies are rounded to a tenth of K, M or G, so a step is only known to within about two ' |
| 1430 | 'such tenths. Steps at or under that show as ≈0.</li>' |
| 1431 | '<li>A commit with no trailer shows no bar, and the work before it is counted in the next commit ' |
| 1432 | 'that has one. The cost is of everything done since the previous trailer, including dead ends, ' |
| 1433 | 'not only the code in the commit.</li>' |
| 1434 | f'<li>Highlighted bars in the commit table: at least {USAGE_HOT}× the median commit.</li></ul>') |
| 1435 | write(path, page(f"{name} usage", body, [f'<a href="{url(name)}/">{esc(name)}</a>', "usage"], view=view)) |
| 1436 | |
| 1437 | |
| 1438 | def build_history_page(view, path, commits, commit_set): |
| 1439 | body = (f'{branch_note(view)}<h1>{esc(posixpath.basename(path))}</h1>' |
| 1440 | f'<p class="meta">Commits that changed this file, newest first · ' |
| 1441 | f'<a href="{view.url("blob", path)}.html">file</a></p>' |
| 1442 | + (commits_table(view.name, commits, commit_set) if commits else |
| 1443 | "<p>No recent commits changed this file (it may have been renamed or only changed long ago).</p>")) |
| 1444 | write(view.path("history", path + ".html"), page(f"{view.name}/{path} history", body, crumbs_for(view, path), view=view)) |
| 1445 | |
| 1446 | |
| 1447 | SHORT_SHA_LENGTHS = range(7, 13) |
| 1448 | |
| 1449 | |
| 1450 | def short_sha_aliases(commits): |
| 1451 | """{alias: full sha} for 7–12 character prefixes, skipping any prefix two commits share. |
| 1452 | |
| 1453 | Apps link to `commit/<git rev-parse --short>.html`, and the short length varies by clone. |
| 1454 | """ |
| 1455 | seen = {} |
| 1456 | for full in commits: |
| 1457 | for n in SHORT_SHA_LENGTHS: |
| 1458 | seen.setdefault(full[:n], set()).add(full) |
| 1459 | return {alias: fulls.pop() for alias, fulls in seen.items() if len(fulls) == 1} |
| 1460 | |
| 1461 | |
| 1462 | def write_short_sha_alias(out, view, alias, full): |
| 1463 | target = f"{full}.html" |
| 1464 | body = f'<h1>Commit {esc(alias)}</h1><p>See <a href="{esc(target)}"><code>{esc(full)}</code></a>.</p>' |
| 1465 | html_out = page(f"{view.name} {alias}", body, view=view).replace( |
| 1466 | "<head>\n", f'<head>\n<meta http-equiv="refresh" content="0; url={esc(target)}">\n', 1) |
| 1467 | write(os.path.join(out, "commit", alias + ".html"), html_out) |
| 1468 | |
| 1469 | |
| 1470 | def build_commit_page(repo, out, view, sha, force=False): |
| 1471 | path = os.path.join(out, "commit", sha + ".html") |
| 1472 | if os.path.exists(path) and not force: |
| 1473 | return # commits never change |
| 1474 | name = view.name |
| 1475 | head = git_text(repo, "show", "-s", "--format=%H%n%an <%ae>%n%ad%n%P%n%B", "--date=iso", sha) |
| 1476 | full, author, date, parents, *msg = head.split("\n") |
| 1477 | author = author.split(" <")[0] # don't publish email addresses |
| 1478 | diff = git(repo, "show", "--format=", "--stat", "--patch", "-M", "--diff-merges=first-parent", sha) |
| 1479 | truncated = len(diff) > MAX_DIFF_BYTES |
| 1480 | text = diff[:MAX_DIFF_BYTES].decode("utf-8", errors="replace") |
| 1481 | lines = [] |
| 1482 | for l in text.split("\n"): |
| 1483 | cls = "" |
| 1484 | if l.startswith("+") and not l.startswith("+++"): |
| 1485 | cls = "add" |
| 1486 | elif l.startswith("-") and not l.startswith("---"): |
| 1487 | cls = "del" |
| 1488 | elif l.startswith("@@"): |
| 1489 | cls = "hunk" |
| 1490 | elif l.startswith("diff --git"): |
| 1491 | cls = "file" |
| 1492 | lines.append(f'<span class="{cls}">{esc(l)}</span>' if cls else esc(l)) |
| 1493 | parent_links = " ".join(f'<a href="{url(name, "commit", p)}.html"><code>{p[:12]}</code></a>' for p in parents.split()) |
| 1494 | body = (f"<h1>{esc(msg[0] if msg else full)}</h1>" |
| 1495 | f'<p class="meta"><code>{full}</code> · {esc(author)} · {esc(date)}' |
| 1496 | f'{" · parents " + parent_links if parent_links else ""}</p>' |
| 1497 | f'{usage_panel(name, full)}' |
| 1498 | f'<pre class="message">{esc(chr(10).join(msg).strip())}</pre>' |
| 1499 | f'<pre class="diff">{chr(10).join(lines)}</pre>' |
| 1500 | + ("<p>Diff truncated. Clone the repository to see all of it.</p>" if truncated else "")) |
| 1501 | write(path, page(f"{name} {full[:12]}", body, [f'<a href="{url(name)}/">{esc(name)}</a>', "commit"], view=view)) |
| 1502 | |
| 1503 | |
| 1504 | def refs_rows(repo, pattern, datefield): |
| 1505 | fmt = f"%(refname:short)%1f%({datefield}:short)%1f%(subject)%1f%(objectname)" |
| 1506 | raw = git_text(repo, "for-each-ref", f"--sort=-{datefield}", f"--format={fmt}", pattern, check=False) |
| 1507 | return [l.split("\x1f") for l in raw.splitlines() if l.count("\x1f") == 3] |
| 1508 | |
| 1509 | |
| 1510 | def build_refs_page(repo, view, branch_views, commit_set): |
| 1511 | branches = refs_rows(repo, "refs/heads", "committerdate") |
| 1512 | tags = refs_rows(repo, "refs/tags", "creatordate") |
| 1513 | def branch_cell(n, sha): |
| 1514 | v = branch_views.get(n) |
| 1515 | label = f"<code>{esc(n)}</code>" |
| 1516 | if v: |
| 1517 | return f'<a href="{v.url("tree")}/">{label}</a>{" <em>(default)</em>" if v.is_default else ""}' |
| 1518 | if sha in commit_set: # merged into the default branch: no pages of its own |
| 1519 | return f'{label} <em>(merged, <a href="{url(view.name, "commit", sha)}.html">last commit</a>)</em>' |
| 1520 | return label |
| 1521 | def table(rows, cell): |
| 1522 | if not rows: |
| 1523 | return "<p>None.</p>" |
| 1524 | return '<table class="log">' + "".join( |
| 1525 | f'<tr><td>{cell(n, h)}</td><td class="date">{esc(d)}</td><td>{esc(s)}</td></tr>' for n, d, s, h in rows) + "</table>" |
| 1526 | body = (f"<h1>Branches</h1>{table(branches, branch_cell)}<h1>Tags</h1>{table(tags, lambda n, h: f'<code>{esc(n)}</code>')}" |
| 1527 | f"<p>Tags can be cloned: <code>git clone --branch <tag> {BASE_URL}/{esc(view.name)}.git</code></p>") |
| 1528 | write(os.path.join(SITE, view.name, "refs", "index.html"), page(f"{view.name} branches & tags", body, view=view)) |
| 1529 | |
| 1530 | |
| 1531 | def app_shots(app, repo=None): |
| 1532 | """[(url, label)]: the screenshots a CI runner took of the app; otherwise (public repos) |
| 1533 | the images in the repo's screenshots/ folder.""" |
| 1534 | try: |
| 1535 | with open(os.path.join(APPS, SHOTS, app, "shots.json")) as f: |
| 1536 | listed = json.load(f).get("shots", []) |
| 1537 | found = [(f"{APPS_URL}/{SHOTS}/{urllib.parse.quote(app)}/{urllib.parse.quote(s['file'])}", s.get("label", "")) |
| 1538 | for s in listed if isinstance(s, dict) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*\.png", s.get("file", "")) |
| 1539 | and os.path.exists(os.path.join(APPS, SHOTS, app, s["file"]))] |
| 1540 | if found: |
| 1541 | return found |
| 1542 | except (OSError, ValueError, AttributeError): |
| 1543 | pass |
| 1544 | d = os.path.join(SITE, repo or "", "raw", "screenshots") |
| 1545 | if repo and os.path.isdir(d): |
| 1546 | return [(f"{BASE_URL}{url(repo, 'raw', 'screenshots', f)}", os.path.splitext(f)[0]) |
| 1547 | for f in sorted(os.listdir(d)) if f.lower().endswith(SHOT_EXTENSIONS)][:4] |
| 1548 | return [] |
| 1549 | |
| 1550 | |
| 1551 | def app_url(name): |
| 1552 | return f"{APPS_URL}/{name}/" if os.path.exists(os.path.join(APPS, name, "index.html")) else None |
| 1553 | |
| 1554 | |
| 1555 | LICENSE_NAMES = ["license", "license.md", "license.txt", "licence", "licence.md", "licence.txt", "copying", "copying.md", "copying.txt"] |
| 1556 | # (SPDX id, phrases that must all appear), checked in order; first match wins. |
| 1557 | LICENSE_PATTERNS = [ |
| 1558 | ("AGPL-3.0", ["gnu affero general public license"]), |
| 1559 | ("LGPL-3.0", ["gnu lesser general public license", "version 3"]), |
| 1560 | ("LGPL-2.1", ["gnu lesser general public license", "version 2.1"]), |
| 1561 | ("GPL-3.0", ["gnu general public license", "version 3"]), |
| 1562 | ("GPL-2.0", ["gnu general public license", "version 2"]), |
| 1563 | ("Apache-2.0", ["apache license", "version 2.0"]), |
| 1564 | ("MPL-2.0", ["mozilla public license", "2.0"]), |
| 1565 | ("MIT", ["permission is hereby granted, free of charge"]), |
| 1566 | ("ISC", ["permission to use, copy, modify, and/or distribute this software for any purpose"]), |
| 1567 | ("BSD-3-Clause", ["redistribution and use in source and binary forms", "neither the name"]), |
| 1568 | ("BSD-2-Clause", ["redistribution and use in source and binary forms"]), |
| 1569 | ("Unlicense", ["this is free and unencumbered software released into the public domain"]), |
| 1570 | ("CC0-1.0", ["cc0 1.0 universal"]), |
| 1571 | ] |
| 1572 | |
| 1573 | |
| 1574 | def detect_license(files, blobs): |
| 1575 | """(license name, path of the license file) from a root LICENSE/COPYING file or package.json, or None.""" |
| 1576 | lower = {p.lower(): p for p in files if "/" not in p} |
| 1577 | for cand in LICENSE_NAMES: |
| 1578 | if cand in lower: |
| 1579 | p = lower[cand] |
| 1580 | text = " ".join(blobs.read(files[p][1])[:20000].decode("utf-8", errors="replace").lower().split()) |
| 1581 | for spdx, phrases in LICENSE_PATTERNS: |
| 1582 | if all(ph in text for ph in phrases): |
| 1583 | return spdx, p |
| 1584 | return "see license file", p |
| 1585 | if "package.json" in files: |
| 1586 | try: |
| 1587 | lic = json.loads(blobs.read(files["package.json"][1])).get("license") |
| 1588 | except (ValueError, AttributeError): |
| 1589 | lic = None |
| 1590 | if isinstance(lic, str) and lic.strip() and lic.strip().upper() != "UNLICENSED": |
| 1591 | return lic.strip(), "package.json" |
| 1592 | return None |
| 1593 | |
| 1594 | |
| 1595 | def build_summary(repo, view, desc, files, ctx, commits, empty, commit_set): |
| 1596 | name = view.name |
| 1597 | clone_url = f"{BASE_URL}/{name}.git" |
| 1598 | app = app_url(name) |
| 1599 | links = f'<p class="meta">App: <a href="{app}">{esc(app)}</a></p>' if app else "" |
| 1600 | contrib, bugs = f"contrib-68uq+{name}@fuzzyworld.net", f"bugs-k7q4+{name}@fuzzyworld.net" |
| 1601 | links += (f'<p class="meta">Contribute: <a href="mailto:{esc(contrib)}">{esc(contrib)}</a>' |
| 1602 | f' (<a href="/contribute/">how</a>) · Report a bug: <a href="mailto:{esc(bugs)}">{esc(bugs)}</a>' |
| 1603 | + (f' · <a href="{url(name, "issues")}/">Issues</a>' if os.path.isfile(os.path.join(SITE, name, "issues", "index.html")) else "") |
| 1604 | + "</p>") |
| 1605 | readme_html, license_html = "", "" |
| 1606 | if not empty: |
| 1607 | blobs = BlobReader(repo) |
| 1608 | lic = detect_license(files, blobs) |
| 1609 | if lic: |
| 1610 | license_html = f' · License: <a href="{view.url("blob", lic[1])}.html">{esc(lic[0])}</a>' |
| 1611 | lower = {p.lower(): p for p in files if "/" not in p} |
| 1612 | for cand in README_NAMES: |
| 1613 | if cand in lower: |
| 1614 | p = lower[cand] |
| 1615 | text = blobs.read(files[p][1]).decode("utf-8", errors="replace") |
| 1616 | ctx.base_dir = "" |
| 1617 | rendered = render_markdown(text, ctx) if p.lower().endswith((".md", ".markdown")) else f"<pre>{esc(text)}</pre>" |
| 1618 | readme_html = f'<article class="markdown readme">{rendered}</article>' |
| 1619 | break |
| 1620 | blobs.close() |
| 1621 | # "Open in VS Code" clones into the visitor's own VS Code; "in the browser" opens our static VS Code web build. |
| 1622 | vscode = f'<a href="vscode://vscode.git/clone?url={urllib.parse.quote(clone_url, safe="")}" class="vscode-link">Open in VS Code</a>' |
| 1623 | if not empty and os.path.exists(os.path.join(APPS, VSCODE_APP, "index.html")): |
| 1624 | vscode += f' · <a href="{APPS_URL}/{VSCODE_APP}/?repo={urllib.parse.quote(name)}" class="vscode-link">Open in VS Code in the browser</a>' |
| 1625 | recent = commits_table(name, commits[:5], commit_set) if commits else "" |
| 1626 | shots = app_shots(name, name) if app else [] |
| 1627 | shots_html = ('<p class="shots">' + "".join(f'<a href="{app}"><img src="{esc(u)}" alt="{esc(l)}" loading="lazy"></a>' for u, l in shots[:4]) |
| 1628 | + "</p>") if shots else "" |
| 1629 | body = (f"<h1>{esc(name)}</h1>" |
| 1630 | + (f'<p class="desc">{esc(desc)}</p>' if desc else "") |
| 1631 | + links + shots_html |
| 1632 | + f'<pre class="clone">git clone {esc(clone_url)}</pre>' |
| 1633 | + f'<p class="meta">{vscode}{license_html}{ci_badge(name, commits[0][0]) if commits else ""}</p>' |
| 1634 | + ("<p>This repository is empty.</p>" if empty else f"<h2>Recent commits</h2>{recent}<p><a href=\"{url(name, 'log')}/\">Full log</a></p>") |
| 1635 | + readme_html) |
| 1636 | write(os.path.join(SITE, name, "index.html"), page(name, body, view=None if empty else view)) |
| 1637 | |
| 1638 | |
| 1639 | def is_merged(repo, branch): |
| 1640 | r = subprocess.run(["git", "-C", repo, "merge-base", "--is-ancestor", "refs/heads/" + branch, "HEAD"], capture_output=True) |
| 1641 | return r.returncode == 0 |
| 1642 | |
| 1643 | |
| 1644 | def build_view(repo, view, vstate, commit_set): |
| 1645 | """Tree, file, history, blame and log pages for one branch. Returns the view's new state.""" |
| 1646 | files = tree_entries(repo, view.ref) |
| 1647 | dirs = all_dirs(files) |
| 1648 | raw_set = {p for p, (m, s, size) in files.items() |
| 1649 | if 0 <= size <= MAX_RAW_BYTES and posixpath.splitext(p)[1].lower() in RAW_EXTENSIONS} |
| 1650 | ctx = LinkContext(view, "", set(files), dirs, raw_set) |
| 1651 | commits = commit_list(repo, view.ref, COMMIT_LIMIT) |
| 1652 | head = commits[0][0] if commits else None |
| 1653 | head_changed = vstate.get("head") != head |
| 1654 | |
| 1655 | old_files = vstate.get("files", {}) |
| 1656 | for p in set(old_files) - set(files): |
| 1657 | for kind, suffix in (("blob", ".html"), ("blame", ".html"), ("history", ".html"), ("raw", "")): |
| 1658 | remove(view.path(kind, p + suffix)) |
| 1659 | for d in set(vstate.get("dirs", [])) - dirs: |
| 1660 | remove(view.path("tree", d, "index.html")) |
| 1661 | |
| 1662 | # File and blame pages and raw copies: only where the blob changed. Markdown pages are |
| 1663 | # rebuilt whenever the branch moves, because their links depend on the whole tree. |
| 1664 | blobs = BlobReader(repo) |
| 1665 | written = 0 |
| 1666 | for p, entry in files.items(): |
| 1667 | is_md = p.lower().endswith((".md", ".markdown")) |
| 1668 | if old_files.get(p) == entry[1] and not (is_md and head_changed): |
| 1669 | continue |
| 1670 | try: |
| 1671 | build_blob_page(repo, view, p, entry, blobs, ctx, raw_set, commit_set) |
| 1672 | if p in raw_set: |
| 1673 | write(view.path("raw", p), blobs.read(entry[1])) |
| 1674 | else: |
| 1675 | remove(view.path("raw", p)) |
| 1676 | written += 1 |
| 1677 | except OSError as e: # e.g. a path component too long for the filesystem |
| 1678 | print(f"skip {p}: {e}", file=sys.stderr) |
| 1679 | blobs.close() |
| 1680 | |
| 1681 | # History pages: only where the newest commit touching the file changed. |
| 1682 | hist = file_histories(repo, view.ref, set(files)) if head_changed or not vstate.get("hist") else None |
| 1683 | new_hist = vstate.get("hist", {}) if hist is None else {} |
| 1684 | if hist is not None: |
| 1685 | old_hist = vstate.get("hist", {}) |
| 1686 | for p in files: |
| 1687 | commits_p = hist.get(p, []) |
| 1688 | newest = commits_p[0][0] if commits_p else "" |
| 1689 | new_hist[p] = newest |
| 1690 | if old_hist.get(p) != newest or not os.path.exists(view.path("history", p + ".html")): |
| 1691 | try: |
| 1692 | build_history_page(view, p, commits_p, commit_set) |
| 1693 | except OSError as e: |
| 1694 | print(f"skip history of {p}: {e}", file=sys.stderr) |
| 1695 | |
| 1696 | # For VS Code in the browser (vscode-web/): every file's blob SHA and size. The contents are |
| 1697 | # objects/<sha>, written by build_repo for files up to MAX_TEXT_BYTES. |
| 1698 | write(view.path("files.json"), json.dumps({"repo": view.name, "branch": view.branch, "commit": head, |
| 1699 | "maxBytes": MAX_TEXT_BYTES, |
| 1700 | "files": {p: [e[1], e[2]] for p, e in sorted(files.items())}})) |
| 1701 | if files: |
| 1702 | build_tree_page(view, "", files, dirs) |
| 1703 | for d in dirs: |
| 1704 | build_tree_page(view, d, files, dirs) |
| 1705 | log_title = f"Last {len(commits)} commits" + ("" if view.is_default else f" on {esc(view.branch)}") |
| 1706 | usage = repo_usage(view.name) |
| 1707 | usage_note = (f'<p class="meta">Bars: Claude Code output tokens per commit, highlighted at {USAGE_HOT}× the median or more · ' |
| 1708 | f'<a href="{url(view.name, "usage")}/">details</a></p>' if usage["costs"] else "") |
| 1709 | write(view.path("log", "index.html"), |
| 1710 | page(f"{view.name} log", f"{branch_note(view)}<h1>{log_title}</h1>{usage_note}" |
| 1711 | f"{commits_table(view.name, commits, commit_set, usage if usage['costs'] else None)}", view=view)) |
| 1712 | print(f"{view.name} [{view.branch}]: {len(files)} files, {written} file pages written, {len(commits)} commits") |
| 1713 | return {"head": head, "files": {p: e[1] for p, e in files.items()}, "dirs": sorted(dirs), "hist": new_hist, |
| 1714 | "_files": files, "_ctx": ctx, "_commits": commits} |
| 1715 | |
| 1716 | |
| 1717 | def build_repo(name): |
| 1718 | repo = os.path.join(GIT_ROOT, name + ".git") |
| 1719 | if not os.path.isdir(repo): |
| 1720 | raise SystemExit(f"no such repo: {name}") |
| 1721 | if not is_public(repo): |
| 1722 | raise SystemExit(f"{name} is not public; refusing to publish") |
| 1723 | out = os.path.join(SITE, name) |
| 1724 | os.makedirs(STATE, exist_ok=True) |
| 1725 | state_path = os.path.join(STATE, name + ".json") |
| 1726 | try: |
| 1727 | with open(state_path) as f: |
| 1728 | state = json.load(f) |
| 1729 | except (FileNotFoundError, ValueError): |
| 1730 | state = {} |
| 1731 | stale = state.get("version") != gen_key() |
| 1732 | if stale: |
| 1733 | state = {} |
| 1734 | |
| 1735 | desc = repo_config(repo, "fuzzyworld.description") |
| 1736 | default_branch = git_text(repo, "symbolic-ref", "--short", "HEAD", check=False).strip() |
| 1737 | branch_names = [b for b in git_text(repo, "for-each-ref", "--format=%(refname:short)", "refs/heads", check=False).split() |
| 1738 | if safe_path(b)] |
| 1739 | empty = default_branch not in branch_names |
| 1740 | # Browse pages for the default branch and for branches not yet merged into it. A merged |
| 1741 | # branch adds nothing new to read, and would duplicate every page. |
| 1742 | views = {b: View(name, b, b == default_branch) for b in branch_names |
| 1743 | if b == default_branch or empty or not is_merged(repo, b)} |
| 1744 | default_view = views.get(default_branch) or View(name, default_branch or "main", True) |
| 1745 | |
| 1746 | # Commit pages for every commit on every branch (up to COMMIT_LIMIT), shared by all branches. |
| 1747 | commit_shas = git_text(repo, "rev-list", f"-n{COMMIT_LIMIT}", "--branches", check=False).split() |
| 1748 | commit_set = set(commit_shas) |
| 1749 | for h in set(state.get("commits", [])) - commit_set: |
| 1750 | remove(os.path.join(out, "commit", h + ".html")) |
| 1751 | aliases = short_sha_aliases(commit_shas) |
| 1752 | for a in set(state.get("aliases", [])) - set(aliases): |
| 1753 | remove(os.path.join(out, "commit", a + ".html")) |
| 1754 | for h in commit_shas: |
| 1755 | build_commit_page(repo, out, default_view, h, force=stale) |
| 1756 | for alias, full in aliases.items(): |
| 1757 | write_short_sha_alias(out, default_view, alias, full) |
| 1758 | |
| 1759 | # Branches that are gone lose their whole directory. |
| 1760 | old_views = state.get("views", {}) |
| 1761 | keep = {v.key for v in views.values()} |
| 1762 | branch_root = os.path.join(out, "branch") |
| 1763 | if os.path.isdir(branch_root): |
| 1764 | for d in os.listdir(branch_root): |
| 1765 | if f"branch/{d}" not in keep: |
| 1766 | shutil.rmtree(os.path.join(branch_root, d), ignore_errors=True) |
| 1767 | if empty: |
| 1768 | remove(os.path.join(out, "files.json")) |
| 1769 | for kind in ("tree", "blob", "blame", "history", "raw", "log", "objects"): |
| 1770 | shutil.rmtree(os.path.join(out, kind), ignore_errors=True) |
| 1771 | |
| 1772 | new_views, built = {}, {} |
| 1773 | for b, v in views.items(): |
| 1774 | built[b] = build_view(repo, v, old_views.get(v.key, {}), commit_set) |
| 1775 | new_views[v.key] = {k: val for k, val in built[b].items() if not k.startswith("_")} |
| 1776 | |
| 1777 | # objects/<sha>: file contents by blob SHA (served as downloads, never as pages), for VS Code in the browser. |
| 1778 | wanted = {e[1] for b in built.values() for e in b["_files"].values() if 0 <= e[2] <= MAX_TEXT_BYTES} |
| 1779 | objects = os.path.join(out, "objects") |
| 1780 | have = set(os.listdir(objects)) if os.path.isdir(objects) else set() |
| 1781 | for sha in have - wanted: |
| 1782 | remove(os.path.join(objects, sha)) |
| 1783 | if wanted - have: |
| 1784 | blobs = BlobReader(repo) |
| 1785 | for sha in wanted - have: |
| 1786 | write(os.path.join(objects, sha), blobs.read(sha)) |
| 1787 | blobs.close() |
| 1788 | |
| 1789 | d = built.get(default_branch, {}) |
| 1790 | if not empty: |
| 1791 | build_refs_page(repo, default_view, views, commit_set) |
| 1792 | build_usage_page(default_view, commit_set) |
| 1793 | if is_public(repo): |
| 1794 | usage_summary(name, save=True) # the site-wide /usage/ page reads this instead of re-running git log |
| 1795 | build_ci_pages(default_view) |
| 1796 | build_summary(repo, default_view, desc, d.get("_files", {}), d.get("_ctx"), d.get("_commits", []), empty, commit_set) |
| 1797 | |
| 1798 | state = {"version": gen_key(), "views": new_views, "commits": commit_shas, |
| 1799 | "aliases": sorted(aliases), "built": int(time.time())} |
| 1800 | write(state_path, json.dumps(state)) |
| 1801 | print(f"{name}: {len(views)} branches, {len(commit_shas)} commits") |
| 1802 | |
| 1803 | |
| 1804 | # Top-level paths of the code site that are not repos. A repo with one of these names would be overwritten by |
| 1805 | # (or overwrite) that page, so the roll-up and index skip it. (lib/admin.py RESERVED_REPOS is for `fw issues`: a different list.) |
| 1806 | SITE_RESERVED = {"usage", "ci", "projects", "why-i-left-gitlab", "my-ai-use", "contribute", "why", "principles"} |
| 1807 | |
| 1808 | |
| 1809 | def published_public_repos(): |
| 1810 | """[(name, git dir)] of public repos that already have a published summary page, sorted by name. |
| 1811 | Reserved top-level names (SITE_RESERVED) are never repos.""" |
| 1812 | found = [] |
| 1813 | for entry in sorted(os.listdir(GIT_ROOT)): |
| 1814 | if not entry.endswith(".git") or entry[:-4] in SITE_RESERVED: |
| 1815 | continue |
| 1816 | repo = os.path.join(GIT_ROOT, entry) |
| 1817 | if is_public(repo) and os.path.exists(os.path.join(SITE, entry[:-4], "index.html")): |
| 1818 | found.append((entry[:-4], repo)) |
| 1819 | return found |
| 1820 | |
| 1821 | |
| 1822 | def usage_summary_rows(u): |
| 1823 | """Compact per-commit rows [date, ai, out, read, cost] (out/read None = no usage data, cost None = unknown) |
| 1824 | from a repo_usage result: all that the site-wide roll-up needs.""" |
| 1825 | rows = [] |
| 1826 | for c in u["all"]: |
| 1827 | e = u["costs"].get(c["sha"]) |
| 1828 | rows.append([c["date"], int(c["ai"]), e["delta"][1] if e else None, e["delta"][2] if e else None, |
| 1829 | e["cost"] if e else None]) |
| 1830 | return rows |
| 1831 | |
| 1832 | |
| 1833 | def usage_summary(name, save=False): |
| 1834 | """{"head", "key", "rows"} for a repo: read from STATE/<name>.usage.json when its HEAD sha and gen_key() still |
| 1835 | match, else computed from git (via repo_usage) and, when `save` or the file was missing or stale, written. |
| 1836 | A missing, corrupt or stale file is simply recomputed.""" |
| 1837 | repo = os.path.join(GIT_ROOT, name + ".git") |
| 1838 | head = git_text(repo, "rev-parse", "HEAD", check=False).strip() |
| 1839 | path = os.path.join(STATE, name + ".usage.json") |
| 1840 | if not save: |
| 1841 | try: |
| 1842 | with open(path) as f: |
| 1843 | cached = json.load(f) |
| 1844 | if (head and cached["head"] == head and cached["key"] == gen_key() |
| 1845 | and all(isinstance(r, list) and len(r) == 5 for r in cached["rows"])): |
| 1846 | return cached |
| 1847 | except (OSError, ValueError, KeyError, TypeError): |
| 1848 | pass |
| 1849 | summary = {"head": head, "key": gen_key(), "rows": usage_summary_rows(repo_usage(name))} |
| 1850 | try: |
| 1851 | os.makedirs(STATE, exist_ok=True) |
| 1852 | write(path, json.dumps(summary)) |
| 1853 | except OSError: |
| 1854 | pass |
| 1855 | return summary |
| 1856 | |
| 1857 | |
| 1858 | def site_usage_totals(names, today=None): |
| 1859 | """(buckets, rows, start) over the named repos. buckets: week_buckets of every repo's commits together, from |
| 1860 | `start`, the first commit with an AI marker or usage data (dates before 2000 or after tomorrow are ignored |
| 1861 | entirely; start is None if there is none). rows: per repo name, commits, with_data, ai, out, dollars, priced, |
| 1862 | has_page; sorted by output tokens, then name.""" |
| 1863 | today = today or datetime.date.today() |
| 1864 | lo, hi = datetime.date(2000, 1, 1).isoformat(), (today + datetime.timedelta(days=1)).isoformat() |
| 1865 | allc, costs, rows = [], {}, [] |
| 1866 | for name in names: |
| 1867 | data = usage_summary(name)["rows"] |
| 1868 | with_data = [r for r in data if r[2] is not None] |
| 1869 | known = [r[4] for r in data if r[4] is not None] |
| 1870 | for i, (date, ai, out, read, cost) in enumerate(data): |
| 1871 | if not lo <= date <= hi: |
| 1872 | continue |
| 1873 | sha = f"{name}:{i}" |
| 1874 | allc.append({"sha": sha, "date": date, "ai": bool(ai), "has_tokens": out is not None}) |
| 1875 | if out is not None: |
| 1876 | costs[sha] = {"delta": [0, out, read], "cost": cost} |
| 1877 | rows.append({"name": name, "commits": len(data), "with_data": len(with_data), "ai": sum(r[1] for r in data), |
| 1878 | "out": sum(r[2] for r in with_data), "dollars": sum(known), "priced": len(known), |
| 1879 | "has_page": bool(with_data)}) |
| 1880 | rows.sort(key=lambda r: (-r["out"], r["name"])) |
| 1881 | start = min((c["date"] for c in allc if c["ai"]), default=None) |
| 1882 | allc = [c for c in allc if start and c["date"] >= start] |
| 1883 | return week_buckets(allc, costs), rows, start |
| 1884 | |
| 1885 | |
| 1886 | def build_site_usage(): |
| 1887 | """code.fuzzyworld.net/usage/: the Usage charts summed over public published repos, and a table per repo.""" |
| 1888 | repos = published_public_repos() |
| 1889 | buckets, rows, start = site_usage_totals([n for n, _ in repos]) |
| 1890 | def cost_cell(r): |
| 1891 | if not r["priced"]: |
| 1892 | return '<td class="num" title="no commit in this repo has a known cost">unknown</td>' |
| 1893 | part = "" if r["priced"] == r["with_data"] else f" ({r['priced']} of {r['with_data']} commits)" |
| 1894 | return (f'<td class="num" title="estimated API list-price equivalent, not what was paid">' |
| 1895 | f'est. ${r["dollars"]:,.2f}{part}</td>') |
| 1896 | table_rows = "".join( |
| 1897 | '<tr><td>' + (f'<a href="{url(r["name"], "usage")}/">{esc(r["name"])}</a>' if r["has_page"] else esc(r["name"])) |
| 1898 | + f'</td><td class="num">{r["commits"]}</td><td class="num">{r["with_data"]}</td>' |
| 1899 | + (f'<td class="num">{100 * r["ai"] / r["commits"]:.0f}%</td>' if r["commits"] else '<td class="num">n/a</td>') |
| 1900 | + f'<td class="num">{fmt_tokens(r["out"]) if r["with_data"] else "unknown"}</td>{cost_cell(r)}</tr>' |
| 1901 | for r in rows) |
| 1902 | if rows: |
| 1903 | table = ('<h2>By repository</h2><table class="log"><tr><th>Repository</th><th>Commits</th>' |
| 1904 | '<th>Commits with usage data</th><th>AI-marked commits</th><th>Output tokens</th>' |
| 1905 | f'<th>Est. cost (not what was paid)</th></tr>{table_rows}</table>') |
| 1906 | else: |
| 1907 | table = '<p class="meta">No public repository is published yet.</p>' |
| 1908 | with_data = sum(r["with_data"] for r in rows) |
| 1909 | stats = usage_stats([ |
| 1910 | ("Repositories", str(len(rows))), |
| 1911 | ("Commits with usage data", f'{with_data} of {sum(r["commits"] for r in rows)}'), |
| 1912 | ("Output tokens", f'~{fmt_tokens(sum(r["out"] for r in rows))}' if any(r["with_data"] for r in rows) else "unknown, not zero (no commit has usage data)"), |
| 1913 | cost_stat(sum(r["dollars"] for r in rows), sum(r["priced"] for r in rows), load_prices().get("as_of", "?"))]) |
| 1914 | body = ('<h1>Claude Code usage, all repositories</h1><p class="meta">Summed over the public repositories, read ' |
| 1915 | 'from the <code>FW-Tokens-*</code> trailers in their commit messages (or older <code>Usage:</code> lines). ' |
| 1916 | 'Unknown is never counted as zero; each repository\'s own usage page has the per-commit detail.</p>' |
| 1917 | + stats |
| 1918 | + (f'<p class="meta">Starts at the first AI-marked commit, {esc(start)}.</p>' if start else "") |
| 1919 | + usage_charts(buckets, "s") + table) |
| 1920 | write(os.path.join(SITE, "usage", "index.html"), |
| 1921 | page("Claude Code usage, all repositories", body, ["usage"], section="usage")) |
| 1922 | print(f"site usage: {len(rows)} repos, {len(buckets)} buckets") |
| 1923 | |
| 1924 | |
| 1925 | def last_commit_time(repo): |
| 1926 | """Unix time of the newest commit on the repo's HEAD, or 0 (empty or unreadable repo).""" |
| 1927 | last = git_text(repo, "log", "-1", "--format=%ct", "HEAD", check=False).strip() |
| 1928 | return int(last) if last.isdigit() else 0 |
| 1929 | |
| 1930 | |
| 1931 | def build_index(): |
| 1932 | """code.fuzzyworld.net/index.html: a table of public repos, most recently committed first.""" |
| 1933 | build_ci_dashboard() |
| 1934 | rows = [(last_commit_time(repo), name, repo_config(repo, "fuzzyworld.description"), app_url(name)) |
| 1935 | for name, repo in published_public_repos()] |
| 1936 | rows.sort(key=lambda r: (-r[0], r[1])) |
| 1937 | items = [] |
| 1938 | for t, n, d, a in rows: |
| 1939 | app = f'<a href="{esc(a)}" class="icon-text"><span class="icon icon-app"></span>app</a>' if a else "" |
| 1940 | when = time.strftime("%Y-%m-%d", time.gmtime(t)) if t else "never (empty)" |
| 1941 | items.append(f'<tr><td><a class="name" href="{url(n)}/">{esc(n)}</a></td><td>{esc(d or "")}</td>' |
| 1942 | f'<td class="num">{when}</td><td>{app}</td></tr>') |
| 1943 | body = (f"<h1>Pavel's code</h1>" |
| 1944 | f"<p>Browse, or clone with <code>git clone {BASE_URL}/<repo>.git</code>. No account or sign-in needed. " |
| 1945 | f'Small web apps are at <a href="{APPS_URL}/">apps.fuzzyworld.net</a>. <br /> <br />' |
| 1946 | f'<a href="/usage/">Claude Code usage across all repositories</a><br /> <br />' |
| 1947 | f'<a href="/why-i-left-gitlab/">Why aren\'t those GitHub/GitLab/Codeberg links? (Why I left GitLab)</a></p>' |
| 1948 | f'<p class="meta">Newest commit first.</p>' |
| 1949 | f'<table class="log repos"><tr><th>Repository</th><th>Description</th><th>Last commit</th><th>App</th></tr>' |
| 1950 | f'{"".join(items)}</table>') |
| 1951 | write(os.path.join(SITE, "index.html"), page("Repositories", body)) |
| 1952 | build_site_usage() |
| 1953 | print(f"index: {len(rows)} public repos") |
| 1954 | |
| 1955 | |
| 1956 | def build_apps_index(): |
| 1957 | """apps.fuzzyworld.net/index.html from each deployed app's .fuzzyworld.json, newest commit first.""" |
| 1958 | apps = [] |
| 1959 | for entry in sorted(os.listdir(APPS)): |
| 1960 | meta_path = os.path.join(APPS, entry, ".fuzzyworld.json") |
| 1961 | if not os.path.isfile(meta_path): |
| 1962 | continue |
| 1963 | try: |
| 1964 | with open(meta_path) as f: |
| 1965 | meta = json.load(f) |
| 1966 | except ValueError: |
| 1967 | continue |
| 1968 | apps.append({"app": entry, "href": f"/{entry}/", "title": meta.get("title") or entry, |
| 1969 | "description": meta.get("description", ""), "repo": meta.get("repo", entry), |
| 1970 | "commit": meta.get("commit", ""), "deployed": meta.get("deployed", "")}) |
| 1971 | for a in apps: |
| 1972 | # Use commit date from the repo instead of deploy timestamp for consistent sorting |
| 1973 | repo = os.path.join(GIT_ROOT, a["repo"] + ".git") |
| 1974 | if a["repo"] and os.path.isdir(repo): |
| 1975 | a["time"] = a["commit_time"] = last_commit_time(repo) |
| 1976 | else: |
| 1977 | # Fallback to deploy timestamp if repo is not accessible |
| 1978 | try: |
| 1979 | a["time"] = datetime.datetime.fromisoformat(a["deployed"].replace("Z", "+00:00")).timestamp() |
| 1980 | except ValueError: |
| 1981 | a["time"] = 0 |
| 1982 | # Apps hosted elsewhere (e.g. engye on Cloudflare Pages), listed in server/site-apps/extra.json. |
| 1983 | # They have no deploy record, so their repo's last commit stands in for the update time. |
| 1984 | extra = os.path.join(os.path.dirname(os.path.abspath(__file__)), "site-apps", "extra.json") |
| 1985 | if os.path.isfile(extra): |
| 1986 | with open(extra) as f: |
| 1987 | for a in json.load(f): |
| 1988 | repo = os.path.join(GIT_ROOT, a.get("repo", "") + ".git") |
| 1989 | t = last_commit_time(repo) if a.get("repo") and os.path.isdir(repo) else 0 |
| 1990 | apps.append({**a, "time": t, "commit_time": t}) |
| 1991 | apps.sort(key=lambda a: (-a["time"], a["title"].lower())) # newest code commit first, not newest deploy |
| 1992 | items = [] |
| 1993 | for a in apps: |
| 1994 | repo = os.path.join(GIT_ROOT, a.get("repo", "") + ".git") |
| 1995 | public = a.get("repo") and os.path.isdir(repo) and is_public(repo) |
| 1996 | src = f'<a href="{BASE_URL}/{urllib.parse.quote(a["repo"])}/" class="icon-text"><span class="icon icon-code"></span>source</a>' if public else "" |
| 1997 | # "Last commit" is blank when the repo is unreadable: "time" then holds the deploy time, only for ordering. |
| 1998 | when = time.strftime("%Y-%m-%d", time.gmtime(a["commit_time"])) if a.get("commit_time") else "" |
| 1999 | deployed = esc((a.get("deployed") or "")[:10]) |
| 2000 | shots = app_shots(a["app"], a["repo"] if public else None) if a.get("app") else [] |
| 2001 | thumb = (f'<a class="thumb" href="{esc(a["href"])}"><img src="{esc(shots[0][0])}" alt="" loading="lazy"></a>' |
| 2002 | if shots else "") |
| 2003 | items.append(f'<tr><td class="shot">{thumb}</td><td><a class="name" href="{esc(a["href"])}">{esc(a["title"])}</a></td>' |
| 2004 | f'<td>{esc(a.get("description") or "")}</td><td class="num">{when}</td>' |
| 2005 | f'<td class="num">{deployed}</td><td>{src}</td></tr>') |
| 2006 | body = (f"<h1>Pavel's apps</h1>" |
| 2007 | f'<p>Small web apps and games. No accounts, no sign-in. Source code is at ' |
| 2008 | f'<a href="{BASE_URL}/">code.fuzzyworld.net</a>.</p>' |
| 2009 | f'<p class="meta">Ordered by the newest code commit, not by deploy date: an app can be redeployed ' |
| 2010 | f'often without its code changing.</p>' |
| 2011 | f'<table class="log repos"><tr><th></th><th>App</th><th>Description</th><th>Last commit</th>' |
| 2012 | f'<th>Deployed</th><th>Source</th></tr>{"".join(items)}</table>') |
| 2013 | write(os.path.join(APPS, "index.html"), page("Apps", body, site="apps")) |
| 2014 | print(f"apps index: {len(apps)} apps") |
| 2015 | |
| 2016 | |
| 2017 | def main(argv): |
| 2018 | if len(argv) == 3 and argv[1] == "repo": |
| 2019 | build_repo(argv[2]) |
| 2020 | elif len(argv) == 2 and argv[1] == "index": |
| 2021 | build_index() |
| 2022 | elif len(argv) == 2 and argv[1] == "apps": |
| 2023 | build_apps_index() |
| 2024 | elif len(argv) == 4 and argv[1] == "landing": |
| 2025 | build_landing(argv[2], argv[3]) |
| 2026 | elif len(argv) == 2 and argv[1] == "content": |
| 2027 | build_content() |
| 2028 | elif len(argv) == 3 and argv[1] == "ci": |
| 2029 | name = argv[2] |
| 2030 | if public_repo(name): |
| 2031 | branch = git_text(os.path.join(GIT_ROOT, name + ".git"), "symbolic-ref", "--short", "HEAD", check=False).strip() |
| 2032 | build_ci_pages(View(name, branch or "main", True)) |
| 2033 | build_ci_dashboard() |
| 2034 | elif len(argv) == 2 and argv[1] == "ci-dashboard": |
| 2035 | build_ci_dashboard() |
| 2036 | elif len(argv) == 2 and argv[1] == "ci-status": |
| 2037 | print(ci_status_text()) |
| 2038 | else: |
| 2039 | raise SystemExit(__doc__) |
| 2040 | |
| 2041 | |
| 2042 | if __name__ == "__main__": |
| 2043 | main(sys.argv) |