fuzzyworld-tools / files / server / issues.py
issues.py
| 1 | #!/usr/bin/env python3 |
| 2 | """Bug reports by email become issue pages on code.fuzzyworld.net. Python standard library only. |
| 3 | |
| 4 | Usage (on the server: `fw issues <cmd>`; from the laptop: `bin/fw issues <cmd>`): |
| 5 | |
| 6 | list [repo] [--json] issues (repo, number, state, whether the page is public, title); --json (for `fw admin`): |
| 7 | {"repos": {repo: {mode, public_repo}}, "issues": [...with record, messages, shown, waiting]} |
| 8 | show <repo> <n> [--json] one issue in full (no addresses are stored, so none are shown); --json (for `fw admin`): |
| 9 | the full text of every message, shown or not, labels, details, prev/next, page and watch URLs |
| 10 | record <repo> <n> record.json as stored, even when the issue is not published (error when there is none) |
| 11 | approve <repo> <n> mark the issue public (what `reviewed` mode waits for) |
| 12 | hide <repo> <n> take the issue off the site and keep it off (even in `public` mode) |
| 13 | close <repo> <n> / reopen <repo> <n> |
| 14 | publish <repo> rewrite <repo>/issues/ on the site from the stored issues (--locked: the caller holds the repo lock) |
| 15 | publish-all the same for every repo that has issues |
| 16 | ingest read new mail from the bugs mailbox (IMAP if imap.json exists, else Maildir) |
| 17 | imap-check connect to the IMAP mailbox and print what it holds (header names only) |
| 18 | fw issues <repo> public|reviewed|private set the repo's mode (`git config fuzzyworld.issues`; anything else = private) |
| 19 | |
| 20 | Modes. `private` (also the default and any unknown word): issues are stored, never published. `reviewed`: an issue |
| 21 | page appears only after `approve`. `public`: an issue whose first message passed authentication appears at once. |
| 22 | A private repo (fuzzyworld.visibility) never publishes issues, whatever its mode. `hide` beats both modes. |
| 23 | |
| 24 | Mail arrives at bugs-k7q4+<repo>@fuzzyworld.net; <repo> must be a repo that exists under ~/git, else the mail goes to |
| 25 | the bad folder. A reply (In-Reply-To / References that match an earlier Message-ID) is added to its issue. |
| 26 | |
| 27 | State, outside every web root (override the directory with FW_ISSUES_DIR): |
| 28 | ~/fw/issues/imap.json IMAP login (mode 600), same format as comments' imap.json; finished mail goes to |
| 29 | <prefix>issues-done / <prefix>issues-bad. Without it: Maildir ~/Maildir/.bugs |
| 30 | (override with FW_ISSUES_MAILDIR), with .bugs.done / .bugs.bad beside it. |
| 31 | ~/fw/issues/secret random salt for sender hashes (mode 600) |
| 32 | ~/fw/issues/msgids.json sha256(Message-ID) -> [repo, n, message_index], so replies find their issue; never published |
| 33 | ~/fw/issues/keys.json sha256(raw mail) -> [repo, n, message_index], so a mail is never stored twice; an entry |
| 34 | counts only if that message holds the same mkey (crash recovery) |
| 35 | ~/fw/issues/rejected.log "id date reason" per rejected mail (rotated at 1 MB) |
| 36 | ~/fw/issues/lock, ingest.lock held for every change to the store / for a whole ingest run |
| 37 | ~/fw/issues/repos/<repo>/<n>/issue.json {n, repo, title, created, state, approved, hidden, record, record_msg, |
| 38 | messages: [{date, author, text, sender_hash, authenticated, approved, mkey, mid}]}; |
| 39 | numbers count up per repo; record_msg is the message that carried the record |
| 40 | ~/fw/issues/repos/<repo>/<n>/record.json the run record, only when one passed the checks (see below) |
| 41 | Published (when visible): <site>/<repo>/issues/index.html and <site>/<repo>/issues/<n>/index.html, plus |
| 42 | <n>/record.json. The whole issues/ folder is rebuilt on every publish, so a hidden issue disappears. |
| 43 | |
| 44 | Privacy. No address, name, Message-ID or sender hash is ever written to a page: the author is always "Reporter", |
| 45 | every address-shaped string in the title and text is replaced by "[address hidden]", lines starting "Device:" or |
| 46 | "Run record:" are left out of the text, and all text is HTML-escaped inside <pre class="issue-text">. |
| 47 | Authentication is the same as for comments (comments.authenticated, topmost Authentication-Results from the |
| 48 | "authserv_id" in sites.json). Unauthenticated mail is stored with authenticated:false and never shown by `public` |
| 49 | mode alone; it needs `approve`. Each sender may start or continue at most 5 issues per 24 hours in a `public` or `reviewed` repo. |
| 50 | |
| 51 | Run records. A line "Run record: {json}" in the text part is read from the FULL decoded part (before the text is cut |
| 52 | to 4000 characters), because a record is ~9 KB. The first line that passes the checks wins (a later "Run record: |
| 53 | too long for an email link" footer is ignored): at most 64 KB, valid JSON, an object with integer seed and |
| 54 | simVersion and a list "actions". It is stored re-serialized (never as mail text). Only the first record of an issue is |
| 55 | kept. A record from an unauthenticated reply is ignored. record.json is published only while the issue is visible. |
| 56 | Only the keys simVersion, seed, startLevel, startCharge, options, actions are kept, and a record with an address-like |
| 57 | string is refused. `approve` marks the messages present at that time; later ones need another approve (or, in `public` |
| 58 | mode, an authenticated sender). The bugs mailbox must not be the comments mailbox (same host, user and folder is refused). |
| 59 | The issue page links to the app's replay viewer when the app's .fuzzyworld.json has a `replay` template. |
| 60 | """ |
| 61 | |
| 62 | import hashlib |
| 63 | import json |
| 64 | import os |
| 65 | import quopri |
| 66 | import re |
| 67 | import secrets |
| 68 | import shutil |
| 69 | import subprocess |
| 70 | import sys |
| 71 | import time |
| 72 | import unicodedata |
| 73 | from contextlib import contextmanager |
| 74 | from email import message_from_bytes, policy |
| 75 | from email.header import decode_header, make_header |
| 76 | from email.utils import getaddresses |
| 77 | from urllib.parse import quote |
| 78 | |
| 79 | sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) |
| 80 | import comments # noqa: E402 |
| 81 | import gen # noqa: E402 |
| 82 | |
| 83 | try: |
| 84 | import fcntl |
| 85 | except ImportError: # not on Windows servers; only the lock is lost |
| 86 | fcntl = None |
| 87 | |
| 88 | Error = comments.CommentError |
| 89 | REPO_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") |
| 90 | BUGS_RE = re.compile(r"^bugs-k7q4\+([A-Za-z0-9][A-Za-z0-9._-]*)@fuzzyworld\.net$", re.I) |
| 91 | ADDRESS_RE = re.compile(r"[^\s@<>]+@[^\s@<>]+") |
| 92 | RECORD_RE = re.compile(r"^Run record:[ \t]*\n?[ \t]*(\{.*\})\s*$", re.M) # the JSON may sit on the next line: some mail clients break the line after the label |
| 93 | RECORD_MAX = 64 * 1024 |
| 94 | RECORD_DEPTH = 32 |
| 95 | RECORD_KEYS = ("simVersion", "seed", "startLevel", "startCharge", "options", "actions") # every key of the real records; the rest is dropped |
| 96 | MAX_MAIL = 5 * 1024 * 1024 |
| 97 | MAX_TITLE = 120 |
| 98 | SENDER_LIMIT = 5 # issues or replies per sender per SENDER_WINDOW seconds, in `public` and `reviewed` repos |
| 99 | SENDER_WINDOW = 24 * 3600 |
| 100 | MODES = ("public", "reviewed", "private") |
| 101 | HIDDEN = "[address hidden]" |
| 102 | |
| 103 | |
| 104 | # ---- configuration and paths ---- |
| 105 | |
| 106 | def state_dir(): |
| 107 | return os.path.expanduser(os.environ.get("FW_ISSUES_DIR") or "~/fw/issues") |
| 108 | |
| 109 | |
| 110 | def imap_path(): |
| 111 | return os.path.join(state_dir(), "imap.json") |
| 112 | |
| 113 | |
| 114 | def maildir_path(): |
| 115 | return os.path.expanduser(os.environ.get("FW_ISSUES_MAILDIR") or "~/Maildir/.bugs") |
| 116 | |
| 117 | |
| 118 | def check_repo(repo): |
| 119 | if not (isinstance(repo, str) and REPO_RE.match(repo) and not repo.endswith(".git")): |
| 120 | raise Error(f"invalid repo name: {repo!r}") |
| 121 | return repo |
| 122 | |
| 123 | |
| 124 | def check_n(n): |
| 125 | if not (isinstance(n, str) and n.isascii() and n.isdigit() and 0 < int(n) < 10**9): |
| 126 | raise Error(f"invalid issue number: {n!r}") |
| 127 | return int(n) |
| 128 | |
| 129 | |
| 130 | def repo_dir(repo): |
| 131 | return os.path.join(gen.GIT_ROOT, check_repo(repo) + ".git") |
| 132 | |
| 133 | |
| 134 | def issue_dir(repo, n): |
| 135 | return os.path.join(state_dir(), "repos", check_repo(repo), str(n)) |
| 136 | |
| 137 | |
| 138 | def log_rejection(cid, reason): |
| 139 | os.makedirs(state_dir(), exist_ok=True) |
| 140 | reason = comments.clean(reason).replace("\n", " ") |
| 141 | log = os.path.join(state_dir(), "rejected.log") |
| 142 | try: |
| 143 | if os.path.getsize(log) > 1_000_000: |
| 144 | os.replace(log, log + ".1") |
| 145 | except FileNotFoundError: |
| 146 | pass |
| 147 | with open(log, "a") as f: |
| 148 | f.write(f"{cid} {comments.now_iso()} {reason}\n") |
| 149 | |
| 150 | |
| 151 | @contextmanager |
| 152 | def _lock(name): |
| 153 | os.makedirs(state_dir(), exist_ok=True) |
| 154 | with open(os.path.join(state_dir(), name), "w") as f: |
| 155 | if fcntl: |
| 156 | fcntl.flock(f, fcntl.LOCK_EX) |
| 157 | yield |
| 158 | |
| 159 | |
| 160 | def locked(): |
| 161 | return _lock("lock") |
| 162 | |
| 163 | |
| 164 | def ingest_lock(): |
| 165 | """Job-wide: one ingest at a time (a second one waits).""" |
| 166 | return _lock("ingest.lock") |
| 167 | |
| 168 | |
| 169 | @contextmanager |
| 170 | def repo_lock(repo): |
| 171 | """The lock `fw` takes around publishing one repo (STATE/locks/<repo>.lock).""" |
| 172 | d = os.path.join(gen.STATE, "locks") |
| 173 | os.makedirs(d, exist_ok=True) |
| 174 | with open(os.path.join(d, check_repo(repo) + ".lock"), "w") as f: |
| 175 | if fcntl: |
| 176 | fcntl.flock(f, fcntl.LOCK_EX) |
| 177 | yield |
| 178 | |
| 179 | |
| 180 | def secret(): |
| 181 | path = os.path.join(state_dir(), "secret") |
| 182 | try: |
| 183 | with open(path) as f: |
| 184 | value = f.read().strip() |
| 185 | except FileNotFoundError: |
| 186 | value = "" |
| 187 | if not value: |
| 188 | value = secrets.token_hex(32) |
| 189 | comments.write_atomic(path, value + "\n") |
| 190 | return value |
| 191 | |
| 192 | |
| 193 | def sender_hash(address): |
| 194 | return hashlib.sha256((secret() + address.strip().lower()).encode()).hexdigest() |
| 195 | |
| 196 | |
| 197 | def sha(text): |
| 198 | return hashlib.sha256(text.encode() if isinstance(text, str) else text).hexdigest() |
| 199 | |
| 200 | |
| 201 | # ---- store ---- |
| 202 | |
| 203 | def issue_numbers(repo): |
| 204 | d = os.path.join(state_dir(), "repos", check_repo(repo)) |
| 205 | # a folder without issue.json is the leftover of a crash (a record written before the issue): not an issue |
| 206 | return sorted(int(x) for x in os.listdir(d) if x.isdigit() and os.path.isfile(os.path.join(d, x, "issue.json"))) if os.path.isdir(d) else [] |
| 207 | |
| 208 | |
| 209 | def stored_repos(): |
| 210 | d = os.path.join(state_dir(), "repos") |
| 211 | return sorted(x for x in os.listdir(d) if REPO_RE.match(x)) if os.path.isdir(d) else [] |
| 212 | |
| 213 | |
| 214 | def load_issue(repo, n): |
| 215 | issue = comments.read_json(os.path.join(issue_dir(repo, n), "issue.json"), None) |
| 216 | if issue is None: |
| 217 | raise Error(f"no issue {repo}#{n}") |
| 218 | return issue |
| 219 | |
| 220 | |
| 221 | def save_issue(issue): |
| 222 | comments.write_json(os.path.join(issue_dir(issue["repo"], issue["n"]), "issue.json"), issue) |
| 223 | |
| 224 | |
| 225 | def load_issues(repo): |
| 226 | out = [] |
| 227 | for n in issue_numbers(repo): |
| 228 | issue = comments.read_json(os.path.join(issue_dir(repo, n), "issue.json"), None) |
| 229 | if issue: |
| 230 | out.append(issue) |
| 231 | return out |
| 232 | |
| 233 | |
| 234 | def record_path(repo, n): |
| 235 | return os.path.join(issue_dir(repo, n), "record.json") |
| 236 | |
| 237 | |
| 238 | def recent_index(): |
| 239 | """sender hash -> dates of that sender's stored messages (read once per run).""" |
| 240 | index = {} |
| 241 | for repo in stored_repos(): |
| 242 | for issue in load_issues(repo): |
| 243 | for m in issue["messages"]: |
| 244 | if m.get("sender_hash"): |
| 245 | index.setdefault(m["sender_hash"], []).append(m.get("date", "")) |
| 246 | return index |
| 247 | |
| 248 | |
| 249 | # ---- visibility and publishing ---- |
| 250 | |
| 251 | def issue_mode(repo_path): |
| 252 | """public | reviewed | private; anything else (a typo, an empty value) is private.""" |
| 253 | mode = gen.repo_config(repo_path, "fuzzyworld.issues") |
| 254 | return mode if mode in ("public", "reviewed") else "private" |
| 255 | |
| 256 | |
| 257 | def visible(repo_path, issue, mode=None): |
| 258 | mode = mode or issue_mode(repo_path) |
| 259 | if mode == "private" or not gen.is_public(repo_path) or issue.get("hidden"): |
| 260 | return False |
| 261 | first = issue["messages"][0] if issue.get("messages") else {} |
| 262 | return bool(issue.get("approved")) or (mode == "public" and bool(first.get("authenticated"))) |
| 263 | |
| 264 | |
| 265 | def replay_ok(t): |
| 266 | """The same rule as checkReplay in lib/deploy.mjs: a relative page of the app, containing {record}.""" |
| 267 | head = t.split("?")[0] if isinstance(t, str) else "" |
| 268 | return (isinstance(t, str) and not re.search(r"[%\s\x00-\x1f\x7f]", head) and len(t) <= 200 and "{record}" in t and not t.startswith("/") and "//" not in t |
| 269 | and ".." not in t and "\\" not in t and ":" not in head) |
| 270 | |
| 271 | |
| 272 | def replay_href(repo, n): |
| 273 | """The URL of the repo's app replaying issue n's record, or "" when no deployed app has a `replay` template.""" |
| 274 | found = [] |
| 275 | try: |
| 276 | entries = sorted(os.listdir(gen.APPS)) |
| 277 | except OSError: |
| 278 | return "" |
| 279 | for app in entries: |
| 280 | try: |
| 281 | with open(os.path.join(gen.APPS, app, ".fuzzyworld.json")) as f: |
| 282 | meta = json.load(f) |
| 283 | except (OSError, ValueError): |
| 284 | continue |
| 285 | if (isinstance(meta, dict) and meta.get("repo", app) == repo and replay_ok(meta.get("replay")) |
| 286 | and os.path.isfile(os.path.join(gen.APPS, app, "index.html"))): |
| 287 | found.append((app != repo, len(app), app, meta["replay"])) # the app named like the repo first (not a -next copy) |
| 288 | if not found: |
| 289 | return "" |
| 290 | _, _, app, replay = min(found) |
| 291 | record_url = f"{gen.BASE_URL}/{repo}/issues/{n}/record.json" |
| 292 | return f"{gen.APPS_URL}/{quote(app)}/" + replay.replace("{record}", quote(record_url, safe="")) |
| 293 | |
| 294 | |
| 295 | def replay_link(repo, n): |
| 296 | """An <a> to the repo's app replaying issue n's record, or "" when no deployed app has a `replay` template.""" |
| 297 | href = replay_href(repo, n) |
| 298 | return f'<a href="{gen.esc(href)}" rel="noopener">Watch what happened</a>' if href else "" |
| 299 | |
| 300 | |
| 301 | def shown_messages(issue, mode): |
| 302 | """Messages `approve` has marked, plus (in public mode only) authenticated ones. A message that arrives after |
| 303 | an approve is not marked, so it needs another approve unless the repo is public and its sender authenticated.""" |
| 304 | return [m for m in issue["messages"] if m.get("approved") or (mode == "public" and m.get("authenticated"))] |
| 305 | |
| 306 | |
| 307 | def record_shown(issue, mode): |
| 308 | """A run record is as public as the message that carried it (issue["record_msg"], 0 for the opening report).""" |
| 309 | if not issue.get("record"): |
| 310 | return False |
| 311 | i = issue.get("record_msg", 0) |
| 312 | return 0 <= i < len(issue["messages"]) and issue["messages"][i] in shown_messages(issue, mode) |
| 313 | |
| 314 | |
| 315 | VERSION_RE = re.compile(r"Version (\S+) \(rules (\d+)\), commit ([0-9a-f]{7,40})") |
| 316 | SEED_RE = re.compile(r"Seed (\d+), level (\d+), (\d+) turns taken, (\d+) lives, ([a-z ]+); modes: (.*)") |
| 317 | SEPARATOR_RE = re.compile(r"^\s*-{3,}\s*$", re.M) |
| 318 | DETAIL_MAX = 40 |
| 319 | |
| 320 | |
| 321 | def badge(state): |
| 322 | return f'<span class="badge{" badge-success" if state == "open" else ""}">{gen.esc(state)}</span>' |
| 323 | |
| 324 | |
| 325 | def details_rows(text): |
| 326 | """(label, value) pairs for the game's own report lines (strict patterns); [] when none matched.""" |
| 327 | rows = [] |
| 328 | v, d = VERSION_RE.search(text), SEED_RE.search(text) |
| 329 | if v: |
| 330 | rows += list(zip(("Version", "Rules", "Commit"), v.groups())) |
| 331 | if d: |
| 332 | rows += list(zip(("Seed", "Level", "Turns", "Lives", "Status", "Modes"), d.groups())) |
| 333 | return [(k, val.strip()[:DETAIL_MAX]) for k, val in rows] |
| 334 | |
| 335 | |
| 336 | def details_table(text): |
| 337 | """A small table of the game's own report lines (strict patterns); "" when none matched.""" |
| 338 | rows = details_rows(text) |
| 339 | if not rows: |
| 340 | return "" |
| 341 | cells = "".join(f"<tr><th>{k}</th><td>{gen.esc(val)}</td></tr>" for k, val in rows) |
| 342 | return f'<table class="issue-details">{cells}</table>' |
| 343 | |
| 344 | |
| 345 | def text_blocks(text): |
| 346 | """The message as <pre> blocks; a line of three or more dashes becomes a thin rule between them.""" |
| 347 | blocks = [b.strip("\n") for b in SEPARATOR_RE.split(text)] |
| 348 | return '<hr class="issue-sep">'.join(f'<pre class="issue-text">{gen.esc(b)}</pre>' for b in blocks if b.strip()) |
| 349 | |
| 350 | |
| 351 | def reporter_labels(messages): |
| 352 | """"Reporter A", "Reporter B", ... by first appearance of each sender within this one issue (hashes never written).""" |
| 353 | seen, labels = {}, [] |
| 354 | for m in messages: |
| 355 | h = m.get("sender_hash") |
| 356 | if not h: |
| 357 | labels.append("Reporter") |
| 358 | continue |
| 359 | if h not in seen: |
| 360 | seen[h] = len(seen) |
| 361 | i = seen[h] |
| 362 | labels.append("Reporter " + (chr(ord("A") + i) if i < 26 else str(i + 1))) |
| 363 | return labels |
| 364 | |
| 365 | |
| 366 | def issue_page(repo, issue, with_record, mode, prev=None, next_=None): |
| 367 | n = issue["n"] |
| 368 | parts = [f'<h1>#{n} {gen.esc(issue["title"])} {badge(issue["state"])}</h1>'] |
| 369 | meta = f'Opened {gen.esc(issue["created"][:10])}' |
| 370 | if with_record: |
| 371 | link = replay_link(repo, n) |
| 372 | meta += (" · " + link if link else "") |
| 373 | parts.append(f'<p class="meta">{meta}</p>') |
| 374 | shown = shown_messages(issue, mode) |
| 375 | if shown: |
| 376 | parts.append(details_table(shown[0]["text"])) |
| 377 | for label, m in zip(reporter_labels(shown), shown): |
| 378 | parts.append(f'<p class="meta">{gen.esc(label)} · {gen.esc(m["date"][:10])}</p>{text_blocks(m["text"])}') |
| 379 | nav = (f'<a href="{gen.url(repo, "issues", str(prev))}/">\u2190 #{prev}</a>' if prev else "<span></span>") + \ |
| 380 | (f'<a href="{gen.url(repo, "issues", str(next_))}/">#{next_} \u2192</a>' if next_ else "<span></span>") |
| 381 | if prev or next_: |
| 382 | parts.append(f'<p class="issue-nav">{nav}</p>') |
| 383 | crumbs = [f'<a href="{gen.url(repo)}/">{gen.esc(repo)}</a>', f'<a href="{gen.url(repo, "issues")}/">Issues</a>', f"#{n}"] |
| 384 | return gen.page(f"{repo} issue #{n}", "".join(parts), crumbs=crumbs) |
| 385 | |
| 386 | |
| 387 | def index_page(repo, issues): |
| 388 | rows = "".join( |
| 389 | f'<tr><td class="num"><a href="{gen.url(repo, "issues", str(i["n"]))}/">#{i["n"]}</a></td>' |
| 390 | f'<td>{gen.esc(i["title"])}</td><td class="date">{gen.esc(i["created"][:10])}</td><td>{badge(i["state"])}</td></tr>' |
| 391 | for i in sorted(issues, key=lambda i: -i["n"])) |
| 392 | body = f'<h1>{gen.esc(repo)} issues</h1><table><tr><th>#</th><th>Title</th><th>Opened</th><th>State</th></tr>{rows}</table>' |
| 393 | return gen.page(f"{repo} issues", body, crumbs=[f'<a href="{gen.url(repo)}/">{gen.esc(repo)}</a>', "Issues"]) |
| 394 | |
| 395 | |
| 396 | def refresh_summary(repo): |
| 397 | """Rebuild the repo's pages so the summary page shows (or drops) its Issues link.""" |
| 398 | subprocess.run([sys.executable, os.path.join(os.path.dirname(os.path.abspath(__file__)), "gen.py"), "repo", repo], |
| 399 | check=False) |
| 400 | |
| 401 | |
| 402 | def write_site(repo): |
| 403 | """Rebuild <site>/<repo>/issues/ from the store. Returns (whether index.html existed before, whether it does now).""" |
| 404 | path = repo_dir(repo) |
| 405 | with locked(): |
| 406 | issues = load_issues(repo) |
| 407 | site_repo = os.path.join(gen.SITE, repo) |
| 408 | final = os.path.join(site_repo, "issues") |
| 409 | had = os.path.isfile(os.path.join(final, "index.html")) |
| 410 | mode = issue_mode(path) if os.path.isdir(path) else "private" |
| 411 | shown = [i for i in issues if os.path.isdir(path) and visible(path, i, mode)] |
| 412 | if not shown: |
| 413 | shutil.rmtree(final, ignore_errors=True) |
| 414 | return had, False |
| 415 | tmp = os.path.join(site_repo, ".issues-new") |
| 416 | shutil.rmtree(tmp, ignore_errors=True) |
| 417 | numbers = sorted(i["n"] for i in shown) |
| 418 | for i in shown: |
| 419 | before = [x for x in numbers if x < i["n"]] |
| 420 | after = [x for x in numbers if x > i["n"]] |
| 421 | has_record = record_shown(i, mode) and os.path.isfile(record_path(repo, i["n"])) |
| 422 | gen.write(os.path.join(tmp, str(i["n"]), "index.html"), issue_page(repo, i, has_record, mode, before[-1] if before else None, after[0] if after else None)) |
| 423 | if has_record: |
| 424 | with open(record_path(repo, i["n"])) as f: |
| 425 | gen.write(os.path.join(tmp, str(i["n"]), "record.json"), f.read()) |
| 426 | gen.write(os.path.join(tmp, "index.html"), index_page(repo, shown)) |
| 427 | shutil.rmtree(final, ignore_errors=True) |
| 428 | os.replace(tmp, final) |
| 429 | return had, True |
| 430 | |
| 431 | |
| 432 | def publish(repo, caller_holds_lock=False): |
| 433 | """Rewrite the repo's issue pages. `fw publish` takes the repo lock itself, so it passes caller_holds_lock |
| 434 | (and runs gen.py repo afterwards); otherwise this takes the lock and refreshes the summary link when needed.""" |
| 435 | check_repo(repo) |
| 436 | if caller_holds_lock: |
| 437 | write_site(repo) |
| 438 | return |
| 439 | with repo_lock(repo): |
| 440 | had, has = write_site(repo) |
| 441 | if had != has and gen.is_public(repo_dir(repo)): |
| 442 | refresh_summary(repo) |
| 443 | |
| 444 | |
| 445 | # ---- email intake ---- |
| 446 | |
| 447 | def scrub(s): |
| 448 | return ADDRESS_RE.sub(HIDDEN, s) |
| 449 | |
| 450 | |
| 451 | def repo_from_mail(msg): |
| 452 | """(repo, None) from the bugs-k7q4+<repo>@fuzzyworld.net address, else (None, reason).""" |
| 453 | values = [str(v) for h in ("To", "Cc", "Delivered-To", "X-Original-To") for v in (msg.get_all(h) or [])] |
| 454 | names = {m.group(1) for _, a in getaddresses(values) if (m := BUGS_RE.match(a.strip()))} |
| 455 | if not names: |
| 456 | return None, "no bugs-k7q4+<repo>@ address in To, Cc, Delivered-To or X-Original-To" |
| 457 | if len(names) > 1: |
| 458 | return None, "mail addressed to more than one repo" |
| 459 | name = names.pop() |
| 460 | if name.endswith(".git") or not os.path.isdir(repo_dir(name)): |
| 461 | return None, f"no such repo {name[:60]!r}" |
| 462 | return name, None |
| 463 | |
| 464 | |
| 465 | def message_ids(value): |
| 466 | return re.findall(r"<[^<>\s]+>", str(value or "")) |
| 467 | |
| 468 | |
| 469 | def thread_of(msg, repo=None): |
| 470 | """(repo, n) of the issue this message replies to (via In-Reply-To / References), else None.""" |
| 471 | ids = message_ids(msg.get("In-Reply-To")) + message_ids(msg.get("References"))[::-1] |
| 472 | known = comments.read_json(os.path.join(state_dir(), "msgids.json"), {}) |
| 473 | for mid in ids: |
| 474 | hit = known.get(sha(mid)) |
| 475 | if hit and (repo is None or hit[0] == repo) and os.path.isfile(os.path.join(issue_dir(hit[0], hit[1]), "issue.json")): |
| 476 | return hit[0], hit[1] |
| 477 | return None |
| 478 | |
| 479 | |
| 480 | def strings_in(value): |
| 481 | """Every string (and dict key) inside a JSON value, iteratively.""" |
| 482 | todo = [value] |
| 483 | while todo: |
| 484 | v = todo.pop() |
| 485 | if isinstance(v, str): |
| 486 | yield v |
| 487 | elif isinstance(v, dict): |
| 488 | todo.extend(v.keys()) |
| 489 | todo.extend(v.values()) |
| 490 | elif isinstance(v, list): |
| 491 | todo.extend(v) |
| 492 | |
| 493 | |
| 494 | def check_record(s): |
| 495 | """(record dict, None) or (None, reason). The checks the replay viewer relies on. Only RECORD_KEYS survive.""" |
| 496 | if len(s.encode("utf-8", "replace")) > RECORD_MAX: |
| 497 | return None, f"run record over {RECORD_MAX // 1024} KB" |
| 498 | |
| 499 | def refuse(name): # NaN and Infinity are not JSON, and the viewer would choke on them |
| 500 | raise ValueError(name) |
| 501 | |
| 502 | def number(text): # 1e400 reads as infinity |
| 503 | x = float(text) |
| 504 | if x != x or x in (float("inf"), float("-inf")): |
| 505 | raise ValueError(text) |
| 506 | return x |
| 507 | try: |
| 508 | rec = json.loads(s, parse_constant=refuse, parse_float=number) |
| 509 | except (ValueError, RecursionError): |
| 510 | return None, "run record is not valid JSON" |
| 511 | if not isinstance(rec, dict): |
| 512 | return None, "run record is not an object" |
| 513 | depth, level = 0, [rec] |
| 514 | while level: # iterative, so a deeply nested record cannot exhaust the stack (here or in the viewer) |
| 515 | depth += 1 |
| 516 | if depth > RECORD_DEPTH: |
| 517 | return None, f"run record nested deeper than {RECORD_DEPTH}" |
| 518 | level = [c for v in level for c in (v.values() if isinstance(v, dict) else v if isinstance(v, list) else ())] |
| 519 | rec = {k: rec[k] for k in RECORD_KEYS if k in rec} |
| 520 | for key in ("seed", "simVersion"): |
| 521 | v = rec.get(key) |
| 522 | if not isinstance(v, int) or isinstance(v, bool) or abs(v) > 2**53: |
| 523 | return None, f"run record needs an integer {key}" |
| 524 | if not isinstance(rec.get("actions"), list): |
| 525 | return None, "run record needs a list of actions" |
| 526 | if any(ADDRESS_RE.search(unicodedata.normalize("NFKC", t)) for t in strings_in(rec)): |
| 527 | return None, "run record holds an address-like string" |
| 528 | return rec, None |
| 529 | |
| 530 | |
| 531 | def find_record(text, raw=b""): |
| 532 | """(record, None), or (None, "") when the text holds no "Run record:" line, or (None, reason) when none passes. |
| 533 | Every matching line is tried and the first valid one wins: the mail repeats the label in a footer.""" |
| 534 | why = "" |
| 535 | sources = [text] |
| 536 | if b"quoted-printable" in raw.lower(): # fallback: decode the whole raw mail (the rule of Explodorama's report-replay.mjs) |
| 537 | sources.append(quopri.decodestring(raw).decode("utf-8", "replace")) |
| 538 | for src in sources: |
| 539 | for m in RECORD_RE.finditer(src.replace("\r\n", "\n")): |
| 540 | rec, why = check_record(m.group(1)) |
| 541 | if rec: |
| 542 | return rec, None |
| 543 | return None, why |
| 544 | |
| 545 | |
| 546 | WROTE_RE = re.compile(r"\b(wrote|schrieb|a écrit|escribió|schreef|skrev|scrisse|написал)\s?[:：]$", re.I) |
| 547 | COLON_RE = re.compile(r"\s?[:：]$") |
| 548 | ORIGINAL_RE = re.compile(r"^\s*[-_=]{2,}\s*(Original Message|Forwarded message|Urspr\w+ Nachricht)\s*[-_=]*\s*$", re.I) |
| 549 | HEADERISH_RE = re.compile(r"^(Sent|Date|To|Cc|Subject|Reply-To):", re.I) |
| 550 | UNDERLINE_RE = re.compile(r"^\s*[_=-]{5,}\s*$") |
| 551 | # the labels of the Explodorama report template (src/main.ts bugDetails) and the like, or any short "Label:" line |
| 552 | KNOWN_LINE_RE = re.compile(r"^(Seed|Version|Errors caught|Run record|Rules|Level|Turn|Modes|What happened)(?=[ :])|^[A-Z][A-Za-z ]{0,20}:") # "Version/17.0" is a user agent, not a label |
| 553 | USER_AGENT_RE = re.compile(r"\S/\S|[();]") # what a wrapped user-agent line looks like; prose rarely has it |
| 554 | |
| 555 | |
| 556 | BARE_VERB_RE = re.compile(r"^((a )?écrit|wrote|schrieb|escribió|schreef|skrev|scrisse|написал)\s?[:：]$", re.I) |
| 557 | VERB_ANY_RE = re.compile(r"\b(wrote|schrieb|écrit|escribió|schreef|skrev|scrisse|написал)\b", re.I) |
| 558 | OPENER_RE = re.compile(r"^(On|Am|Le|El|Op|Il|Den|Em|Dne|W dniu)\b", re.I) |
| 559 | # one "Label: value" line of a mail-header block, in any language ("Von:", "De :", "Envoyé :") |
| 560 | HEADER_LINE_RE = re.compile(r"^([^\W\d_][\w .-]{0,19}?)\s?[:：]\s*\S") |
| 561 | HEADER_LABELS = {"from", "von", "de", "da", "van", "från", "sent", "gesendet", "envoyé", "date", "datum", "verzonden", |
| 562 | "fecha", "enviado", "to", "an", "à", "aan", "para", "cc", "subject", "betreff", "objet", "onderwerp", |
| 563 | "asunto", "reply-to", "antwort an"} |
| 564 | TEMPLATE_LABELS = {"device", "run record", "errors caught", "seed", "version", "rules"} # the game's own report lines |
| 565 | |
| 566 | |
| 567 | def has_address(text): |
| 568 | return bool(ADDRESS_RE.search(unicodedata.normalize("NFKC", text))) |
| 569 | |
| 570 | |
| 571 | def strong_attribution(line, nxt): |
| 572 | """True for a reply's "who wrote this" line in any language: a colon line that holds an address, or that opens like |
| 573 | one (On, Am, Le, ... plus a digit) and says wrote/schrieb/a écrit/..., or says it and is followed by a quoted line; |
| 574 | or an opener/address line wrapped so that only "wrote:" is on the next line. "I pressed <Enter> and wrote:" is prose.""" |
| 575 | t = line.strip() |
| 576 | if line.startswith(">") or not t: |
| 577 | return False |
| 578 | if COLON_RE.search(t): |
| 579 | if has_address(t): |
| 580 | return True |
| 581 | return bool(WROTE_RE.search(t)) and ((OPENER_RE.match(t) and re.search(r"\d", t)) or nxt.startswith(">")) |
| 582 | return bool(BARE_VERB_RE.match(nxt.strip())) and bool(OPENER_RE.match(t) or has_address(t)) |
| 583 | |
| 584 | |
| 585 | def header_run_start(lines, i): |
| 586 | """True when lines[i:] begin a block of >= 3 consecutive "Label: value" lines that is a mail header (it holds an |
| 587 | address, or at least three of its labels are From/To/Subject/Date words in some language).""" |
| 588 | run = [] |
| 589 | for line in lines[i:]: |
| 590 | m = HEADER_LINE_RE.match(line.strip()) |
| 591 | if not m or m.group(1).strip().lower() in TEMPLATE_LABELS or line.startswith(">"): |
| 592 | break |
| 593 | run.append((m.group(1).strip().lower(), line)) |
| 594 | if len(run) < 3: |
| 595 | return False |
| 596 | return any(has_address(l) for _, l in run) or sum(1 for lab, _ in run if lab in HEADER_LABELS) >= 3 |
| 597 | |
| 598 | |
| 599 | def find_cut(lines): |
| 600 | """Index of the first line of the quoted reply, attribution or signature, or None.""" |
| 601 | cuts = [] |
| 602 | n = len(lines) |
| 603 | quoted = lambda l: l.startswith(">") |
| 604 | for q in range(1, n): # a wrapped attribution: up to 3 plain lines ending in a colon, directly followed by a quoted line |
| 605 | if not quoted(lines[q]) or quoted(lines[q - 1]) or not lines[q - 1].strip() or not COLON_RE.search(lines[q - 1].strip()): |
| 606 | continue |
| 607 | lo = q |
| 608 | while lo > max(q - 3, 0) and lines[lo - 1].strip() and not quoted(lines[lo - 1]): |
| 609 | lo -= 1 |
| 610 | for k in range(lo, q): |
| 611 | joined = " ".join(x.strip() for x in lines[k:q]) |
| 612 | if (VERB_ANY_RE.search(joined) or has_address(joined)) and ( |
| 613 | OPENER_RE.match(lines[k].strip()) or has_address(lines[k]) or "<" in lines[k]): |
| 614 | cuts.append(k) |
| 615 | break |
| 616 | for i, line in enumerate(lines): |
| 617 | nxt = lines[i + 1] if i + 1 < n else "" |
| 618 | if line.rstrip() == "--" or ORIGINAL_RE.match(line) or strong_attribution(line, nxt) or header_run_start(lines, i): |
| 619 | cuts.append(i) |
| 620 | break |
| 621 | if (COLON_RE.search(line.strip()) and nxt.startswith(">") and not quoted(line) # a weak match: only when the rest is |
| 622 | and all(not l.strip() or quoted(l) for l in lines[i + 1:]) # quoted, and text came before it |
| 623 | and any(l.strip() and not l.lstrip().startswith((">", "Device:", "Run record:")) for l in lines[:i])): |
| 624 | cuts.append(i) |
| 625 | break |
| 626 | return min(cuts) if cuts else None |
| 627 | |
| 628 | |
| 629 | def unflow(lines, delsp=False): |
| 630 | """RFC 3676 format=flowed: a line ending in a space continues on the next one (quote marks aside). The space is |
| 631 | tested on the physical line, and an empty physical line always ends a paragraph.""" |
| 632 | out, cur = [], None |
| 633 | for raw in lines: |
| 634 | if raw.startswith(">") or raw == "-- ": # quotes and the signature separator are tested on the raw line |
| 635 | if cur is not None: |
| 636 | out.append(cur) |
| 637 | cur = None |
| 638 | out.append(raw) |
| 639 | continue |
| 640 | line = raw[1:] if raw.startswith(" ") else raw # space-stuffing |
| 641 | if line == "": |
| 642 | if cur is not None: |
| 643 | out.append(cur) |
| 644 | cur = None |
| 645 | out.append("") |
| 646 | continue |
| 647 | if cur is not None: |
| 648 | if line.lstrip().startswith(("Device:", "Run record:")): |
| 649 | out.append(cur) # these always start a line of their own |
| 650 | elif cur.lstrip().startswith("Device:") and KNOWN_LINE_RE.match(line): |
| 651 | out.append(cur) # a Device: line never swallows the paragraph after it |
| 652 | else: |
| 653 | line = (cur[:-1] if delsp else cur) + line |
| 654 | cur = None |
| 655 | if raw.endswith(" "): |
| 656 | cur = line |
| 657 | else: |
| 658 | out.append(line) |
| 659 | if cur is not None: |
| 660 | out.append(cur) |
| 661 | return out |
| 662 | |
| 663 | |
| 664 | def report_text(content, flowed=False, delsp=False): |
| 665 | """The reporter's words: quotes, signature, Device: and Run record: lines out, replies cut at their attribution |
| 666 | (any language, wrapped or not) or mail-header block, addresses hidden, cut to 4000. NFKC comes first, so fullwidth |
| 667 | letters and @ count. Device: and the lines that wrap it: following lines are dropped while they look like a user |
| 668 | agent (indented, or containing token/version, parentheses or semicolons); a blank line, a template label ("Seed |
| 669 | ...") or plain prose ends the block, so the reporter's own words after a Device line are kept.""" |
| 670 | lines = unicodedata.normalize("NFKC", comments.clean(content)).split("\n") |
| 671 | if flowed: |
| 672 | lines = unflow(lines, delsp) |
| 673 | cut = find_cut(lines) |
| 674 | kept, in_device, after_label = [], False, False |
| 675 | for line in lines[:cut]: |
| 676 | s = line.lstrip() |
| 677 | if after_label and s.startswith("{"): # the record sat on the line after a bare "Run record:" label |
| 678 | after_label = False |
| 679 | continue |
| 680 | after_label = s.startswith("Run record:") and not s[len("Run record:"):].strip() |
| 681 | if s.startswith("Device:"): |
| 682 | in_device = True |
| 683 | continue |
| 684 | if in_device: |
| 685 | if line.strip() and not KNOWN_LINE_RE.match(s) and (line[0] in " \t" or USER_AGENT_RE.search(line)): |
| 686 | continue |
| 687 | in_device = False |
| 688 | if s.startswith((">", "Run record:")): |
| 689 | continue |
| 690 | kept.append(line) |
| 691 | while cut is not None and kept and (not kept[-1].strip() or UNDERLINE_RE.match(kept[-1])): # Outlook's ____ line |
| 692 | kept.pop() |
| 693 | return scrub("\n".join(kept).strip())[:comments.MAX_TEXT] |
| 694 | |
| 695 | |
| 696 | def parse_report(msg, authserv_id, raw=b""): |
| 697 | """(report dict with the sender "_address", None) or (None, reason). Content problems only.""" |
| 698 | froms = msg.get_all("From") or [] |
| 699 | addrs = [a for a in getaddresses([str(x) for x in froms]) if a[1]] |
| 700 | if len(froms) != 1 or len(addrs) != 1 or "@" not in addrs[0][1]: |
| 701 | return None, "need exactly one From header with one address" |
| 702 | addr = addrs[0][1] |
| 703 | ok = comments.authenticated(msg, addr.rsplit("@", 1)[1].lower(), authserv_id) is None |
| 704 | body = msg.get_body(("plain",)) |
| 705 | content = body.get_content() if body is not None else "" |
| 706 | flowed = body is not None and (body.get_param("format") or "").lower() == "flowed" |
| 707 | delsp = body is not None and (body.get_param("delsp") or "").lower() == "yes" |
| 708 | text = report_text(content, flowed, delsp) |
| 709 | if not text: |
| 710 | return None, "empty text" |
| 711 | subject = str(make_header(decode_header(str(msg.get("Subject", ""))))) |
| 712 | title = " ".join(scrub(unicodedata.normalize("NFKC", comments.clean(subject))).split())[:MAX_TITLE].strip() or "(no subject)" |
| 713 | record, record_why = find_record(content, raw) |
| 714 | return {"title": title, "text": text, "author": "Reporter", "authenticated": ok, "date": comments.now_iso(), |
| 715 | "record": record, "record_why": record_why, "_address": addr}, None |
| 716 | |
| 717 | |
| 718 | def next_number(repo): |
| 719 | nums = issue_numbers(repo) |
| 720 | return (nums[-1] if nums else 0) + 1 |
| 721 | |
| 722 | |
| 723 | def store_report(repo, report, thread, mkey, mid): |
| 724 | """Add the report to its thread or open a new issue. Returns (issue, new?, note).""" |
| 725 | message = {"date": report["date"], "author": report["author"], "text": report["text"], |
| 726 | "sender_hash": report["sender_hash"], "authenticated": report["authenticated"], "approved": False, |
| 727 | "mkey": mkey, "mid": sha(mid) if mid else ""} # mkey and mid: for crash recovery, never published |
| 728 | if thread: |
| 729 | issue, new = load_issue(*thread), False |
| 730 | issue["messages"].append(message) |
| 731 | else: |
| 732 | n = next_number(repo) |
| 733 | issue = {"n": n, "repo": repo, "title": report["title"], "created": report["date"], "state": "open", |
| 734 | "approved": False, "hidden": False, "record": False, "record_msg": 0, "messages": [message]} |
| 735 | new = True |
| 736 | note = "" |
| 737 | index = len(issue["messages"]) - 1 |
| 738 | where = [repo, issue["n"], index] |
| 739 | # Remember the mail first: a crash after this leaves a key without a message, which stored() does not count, |
| 740 | # so the mail is read again (no lost report) and never stored twice (no duplicate issue). |
| 741 | keys = comments.read_json(os.path.join(state_dir(), "keys.json"), {}) |
| 742 | keys[mkey] = where |
| 743 | comments.write_json(os.path.join(state_dir(), "keys.json"), keys) |
| 744 | if mid: |
| 745 | ids = comments.read_json(os.path.join(state_dir(), "msgids.json"), {}) |
| 746 | ids[sha(mid)] = where |
| 747 | comments.write_json(os.path.join(state_dir(), "msgids.json"), ids) |
| 748 | write_record = None |
| 749 | if report["record"]: |
| 750 | if issue["record"]: |
| 751 | note = "second run record ignored" |
| 752 | elif not new and not report["authenticated"]: |
| 753 | note = "run record from an unauthenticated reply ignored" |
| 754 | else: |
| 755 | write_record = json.dumps(report["record"], separators=(",", ":"), allow_nan=False) |
| 756 | issue["record"], issue["record_msg"] = True, index |
| 757 | elif report["record_why"]: |
| 758 | note = report["record_why"] |
| 759 | if write_record: # the file first, so issue.json never claims a record that is missing; issue_numbers ignores a lone folder |
| 760 | comments.write_atomic(record_path(repo, issue["n"]), write_record) |
| 761 | save_issue(issue) |
| 762 | return issue, new, note |
| 763 | |
| 764 | |
| 765 | def stored(where, field, value): |
| 766 | """Whether the message a keys.json / msgids.json entry points at is really in its issue (same mkey / mid).""" |
| 767 | try: |
| 768 | issue = comments.read_json(os.path.join(issue_dir(where[0], where[1]), "issue.json"), None) |
| 769 | return bool(issue) and issue["messages"][where[2]].get(field) == value |
| 770 | except (Error, IndexError, TypeError, KeyError): |
| 771 | return False |
| 772 | |
| 773 | |
| 774 | def ingest_messages(source, authserv_id): |
| 775 | with locked(): |
| 776 | recent = recent_index() |
| 777 | cutoff = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - SENDER_WINDOW)) |
| 778 | touched = [] |
| 779 | for key in sorted(source.keys(), key=lambda k: [int(x) if x.isdigit() else x for x in re.split(r"(\d+)", k)]): # oldest first, so a reply finds its issue |
| 780 | with locked(): # per message, so `fw issues approve` is not starved by a big backlog |
| 781 | try: |
| 782 | raw = source.read(key) |
| 783 | except KeyError: |
| 784 | continue |
| 785 | except comments.SkipMessage as e: # leave it in the inbox and go on with the others |
| 786 | log_rejection(comments.new_id(), f"skipped {key}: {e}") |
| 787 | print(f"skipped {key}: {e}") |
| 788 | continue |
| 789 | mkey = sha(raw) |
| 790 | keys = comments.read_json(os.path.join(state_dir(), "keys.json"), {}) |
| 791 | if mkey in keys and stored(keys[mkey], "mkey", mkey): # stored already: a crash or overlap left the mail behind |
| 792 | log_rejection(comments.new_id(), "duplicate of a stored report; moved to done") |
| 793 | source.finish(key, raw, "done") |
| 794 | continue |
| 795 | repo = report = thread = mid = None |
| 796 | try: |
| 797 | if len(raw) > MAX_MAIL: |
| 798 | why = f"message over {MAX_MAIL // 1024} KB" |
| 799 | else: |
| 800 | msg = message_from_bytes(raw, policy=policy.default) |
| 801 | repo, why = repo_from_mail(msg) |
| 802 | if repo: |
| 803 | report, why = parse_report(msg, authserv_id, raw) |
| 804 | if report: |
| 805 | thread = thread_of(msg, repo) |
| 806 | mid = (message_ids(msg.get("Message-ID")) or [None])[0] |
| 807 | known = comments.read_json(os.path.join(state_dir(), "msgids.json"), {}) |
| 808 | if mid and sha(mid) in known and stored(known[sha(mid)], "mid", sha(mid)): # the same Message-ID again |
| 809 | report, why = None, "duplicate Message-ID of a stored report" |
| 810 | except (Error, OSError): # state problems abort the run and leave the mail in the inbox |
| 811 | raise |
| 812 | except Exception as e: # a malformed message must not stop the rest |
| 813 | report, why = None, f"unreadable message: {e}" |
| 814 | if report: |
| 815 | report["sender_hash"] = h = sender_hash(report.pop("_address")) |
| 816 | if issue_mode(repo_dir(repo)) != "private" and sum(1 for d in recent.get(h, []) if d >= cutoff) >= SENDER_LIMIT: |
| 817 | report, why = None, f"sender limit reached ({SENDER_LIMIT} per 24 hours)" |
| 818 | if report is None: # rejected mail is not remembered: moved back from bad, it is tried again |
| 819 | cid = comments.new_id() |
| 820 | log_rejection(cid, why) |
| 821 | print(f"bad {cid}: {why}") |
| 822 | source.finish(key, raw, "bad") |
| 823 | continue |
| 824 | issue, new, note = store_report(repo, report, thread, mkey, mid) |
| 825 | if note: |
| 826 | log_rejection(comments.new_id(), f"{repo}#{issue['n']}: {note}") |
| 827 | recent.setdefault(report["sender_hash"], []).append(report["date"]) |
| 828 | print(f"{'new issue' if new else 'reply to'} {repo}#{issue['n']}" + (" with run record" if issue["record"] and new else "")) |
| 829 | if repo not in touched: |
| 830 | touched.append(repo) |
| 831 | source.finish(key, raw, "done") |
| 832 | for repo in touched: |
| 833 | try: |
| 834 | publish(repo) |
| 835 | except (Error, OSError) as e: # the data is stored; `publish` later fixes the pages |
| 836 | print(f"issues: could not publish {repo} ({e}); run `fw issues publish {repo}` later", file=sys.stderr) |
| 837 | |
| 838 | |
| 839 | def message_source(): |
| 840 | if os.path.exists(imap_path()): |
| 841 | return comments.IMAPSource(cfg_path=imap_path(), stem="issues", max_size=MAX_MAIL, log=log_rejection) |
| 842 | return comments.MaildirSource(maildir_path()) |
| 843 | |
| 844 | |
| 845 | def check_mailbox_separate(): |
| 846 | """Refuse to share the comments mailbox: the two jobs would move each other's mail to their bad folders.""" |
| 847 | mine, theirs = imap_path(), comments.imap_config_path() |
| 848 | if not (os.path.exists(mine) and os.path.exists(theirs)): |
| 849 | return |
| 850 | a, b = comments.load_imap_config(mine), comments.load_imap_config(theirs) |
| 851 | if tuple(a[k].casefold() for k in ("host", "user", "folder")) == tuple(b[k].casefold() for k in ("host", "user", "folder")): |
| 852 | raise Error(f"{mine} and {theirs} point at the same mailbox ({a['user']}@{a['host']}, folder {a['folder']}); " |
| 853 | "the bugs mailbox needs its own account or its own \"folder\"") |
| 854 | |
| 855 | |
| 856 | def cmd_ingest(): |
| 857 | source = message_source() |
| 858 | if not source.available(): |
| 859 | return |
| 860 | if isinstance(source, comments.IMAPSource): |
| 861 | check_mailbox_separate() |
| 862 | authserv_id = (comments.load_config().get("authserv_id") or "").strip() |
| 863 | if not authserv_id: |
| 864 | raise Error('authserv_id is empty in sites.json; refusing to ingest (set it to the authserv-id ' |
| 865 | 'your mail server puts at the start of its Authentication-Results header)') |
| 866 | try: |
| 867 | with ingest_lock(): |
| 868 | ingest_messages(source, authserv_id) |
| 869 | finally: |
| 870 | source.close() |
| 871 | |
| 872 | |
| 873 | # ---- moderation ---- |
| 874 | |
| 875 | def mode_of(repo): |
| 876 | """(issue mode, whether the repo exists and is public) for a repo name.""" |
| 877 | path = repo_dir(repo) |
| 878 | if not os.path.isdir(path): |
| 879 | return "private", False |
| 880 | return issue_mode(path), gen.is_public(path) |
| 881 | |
| 882 | |
| 883 | def cmd_list_json(repo=None): |
| 884 | """For `fw admin`: every stored issue with its flags, plus each repo's mode. No sender hash, mkey or mid.""" |
| 885 | repos, out = {}, [] |
| 886 | for r in ([check_repo(repo)] if repo else stored_repos()): |
| 887 | mode, is_pub = mode_of(r) |
| 888 | repos[r] = {"mode": mode, "public_repo": is_pub} |
| 889 | path = repo_dir(r) |
| 890 | for i in load_issues(r): |
| 891 | msgs = i.get("messages", []) |
| 892 | shown = len(shown_messages(i, mode)) |
| 893 | seen = bool(msgs) and is_pub and visible(path, i, mode) |
| 894 | out.append({"repo": r, "n": i["n"], "title": i["title"], "created": i["created"], |
| 895 | "updated": msgs[-1]["date"] if msgs else i["created"], "state": i["state"], |
| 896 | "approved": bool(i.get("approved")), "hidden": bool(i.get("hidden")), "public": seen, |
| 897 | "record": bool(i.get("record")), "messages": len(msgs), "shown": shown, |
| 898 | "waiting": not i.get("hidden") and mode in ("public", "reviewed") and is_pub and shown < len(msgs)}) |
| 899 | print(json.dumps({"repos": repos, "issues": out}, ensure_ascii=False)) |
| 900 | |
| 901 | |
| 902 | def cmd_list(repo=None): |
| 903 | for r in ([check_repo(repo)] if repo else stored_repos()): |
| 904 | path = repo_dir(r) |
| 905 | mode = issue_mode(path) if os.path.isdir(path) else "private" |
| 906 | for i in load_issues(r): |
| 907 | flags = ("approved " if i.get("approved") else "") + ("hidden " if i.get("hidden") else "") + ("record " if i.get("record") else "") |
| 908 | shown = "PUBLIC" if os.path.isdir(path) and visible(path, i, mode) else "-" |
| 909 | print(f"{r}#{i['n']} {i['state']:<6} {shown:<6} {mode:<8} {flags}{i['title'][:60]}") |
| 910 | |
| 911 | |
| 912 | def cmd_show(repo, n): |
| 913 | n = check_n(n) |
| 914 | print(json.dumps(load_issue(check_repo(repo), n), indent=2, ensure_ascii=False)) |
| 915 | |
| 916 | |
| 917 | def cmd_show_json(repo, n): |
| 918 | """For `fw admin`: one issue with the full text of every message, shown or not. No sender hash, mkey or mid.""" |
| 919 | n = check_n(n) |
| 920 | issue = load_issue(check_repo(repo), n) |
| 921 | mode, is_pub = mode_of(repo) |
| 922 | msgs = issue.get("messages", []) |
| 923 | shown = shown_messages(issue, mode) |
| 924 | public = bool(msgs) and is_pub and visible(repo_dir(repo), issue, mode) |
| 925 | rec_shown = record_shown(issue, mode) |
| 926 | watch = replay_href(repo, n) if rec_shown else "" |
| 927 | numbers = issue_numbers(repo) |
| 928 | out = {"repo": repo, "n": n, "title": issue["title"], "created": issue["created"], "state": issue["state"], |
| 929 | "approved": bool(issue.get("approved")), "hidden": bool(issue.get("hidden")), "mode": mode, "public_repo": is_pub, |
| 930 | "public": public, "page_url": f"{gen.BASE_URL}/{repo}/issues/{n}/" if public else None, |
| 931 | "record": bool(issue.get("record")), "record_shown": rec_shown, "watch_url": watch or None, |
| 932 | "details": [list(r) for r in details_rows(msgs[0]["text"])] if msgs else [], |
| 933 | "prev": max((x for x in numbers if x < n), default=None), "next": min((x for x in numbers if x > n), default=None), |
| 934 | "messages": [{"label": label, "date": m.get("date", ""), "authenticated": bool(m.get("authenticated")), |
| 935 | "approved": bool(m.get("approved")), "shown": any(m is s for s in shown), "text": m.get("text", "")} |
| 936 | for label, m in zip(reporter_labels(msgs), msgs)]} |
| 937 | print(json.dumps(out, ensure_ascii=False)) |
| 938 | |
| 939 | |
| 940 | def cmd_record(repo, n): |
| 941 | """record.json exactly as stored (for `fw admin`: the record of a report that is not published).""" |
| 942 | n = check_n(n) |
| 943 | try: |
| 944 | with open(record_path(check_repo(repo), n)) as f: |
| 945 | sys.stdout.write(f.read()) |
| 946 | except FileNotFoundError: |
| 947 | raise Error(f"no record for {repo}#{n}") |
| 948 | |
| 949 | |
| 950 | def cmd_change(repo, n, action): |
| 951 | n = check_n(n) |
| 952 | with locked(): |
| 953 | issue = load_issue(check_repo(repo), n) |
| 954 | if action == "approve": |
| 955 | issue["approved"], issue["hidden"] = True, False |
| 956 | for m in issue["messages"]: |
| 957 | m["approved"] = True |
| 958 | elif action == "hide": |
| 959 | issue["approved"], issue["hidden"] = False, True |
| 960 | for m in issue["messages"]: |
| 961 | m["approved"] = False |
| 962 | elif action == "close": |
| 963 | issue["state"] = "closed" |
| 964 | elif action == "reopen": |
| 965 | issue["state"] = "open" |
| 966 | save_issue(issue) |
| 967 | publish(repo) |
| 968 | path = repo_dir(repo) |
| 969 | print(f"{action}: {repo}#{n} ({'public' if os.path.isdir(path) and visible(path, issue) else 'not public'})") |
| 970 | |
| 971 | |
| 972 | def cmd_publish(repo): |
| 973 | publish(repo) |
| 974 | print(f"{repo}: issues published") |
| 975 | |
| 976 | |
| 977 | def cmd_publish_all(): |
| 978 | for r in stored_repos(): |
| 979 | try: |
| 980 | publish(r) |
| 981 | except Error as e: |
| 982 | print(f"issues: {r}: {e}", file=sys.stderr) |
| 983 | print("issues: published") |
| 984 | |
| 985 | |
| 986 | def main(argv): |
| 987 | if not argv or argv[0] in ("-h", "--help", "help"): |
| 988 | print(__doc__) |
| 989 | return 0 |
| 990 | cmd, args = argv[0], argv[1:] |
| 991 | flags = [a for a in args if a.startswith("--")] |
| 992 | args = [a for a in args if not a.startswith("--")] |
| 993 | try: |
| 994 | if cmd == "list" and len(args) <= 1 and set(flags) <= {"--json"}: |
| 995 | (cmd_list_json if "--json" in flags else cmd_list)(args[0] if args else None) |
| 996 | elif cmd == "show" and len(args) == 2 and set(flags) <= {"--json"}: |
| 997 | (cmd_show_json if "--json" in flags else cmd_show)(*args) |
| 998 | elif cmd == "record" and len(args) == 2 and not flags: |
| 999 | cmd_record(*args) |
| 1000 | elif cmd in ("approve", "hide", "close", "reopen") and len(args) == 2: |
| 1001 | cmd_change(args[0], args[1], cmd) |
| 1002 | elif cmd == "publish" and len(args) == 1 and set(flags) <= {"--locked"}: |
| 1003 | if "--locked" in flags: |
| 1004 | publish(args[0], caller_holds_lock=True) |
| 1005 | else: |
| 1006 | cmd_publish(args[0]) |
| 1007 | elif cmd == "publish-all": |
| 1008 | cmd_publish_all() |
| 1009 | elif cmd == "ingest": |
| 1010 | cmd_ingest() |
| 1011 | elif cmd == "imap-check": |
| 1012 | source = comments.IMAPSource(cfg_path=imap_path(), stem="issues", max_size=MAX_MAIL, log=log_rejection) |
| 1013 | if not source.available(): |
| 1014 | raise Error(f"no IMAP config at {imap_path()}") |
| 1015 | check_mailbox_separate() |
| 1016 | try: |
| 1017 | source.check() |
| 1018 | finally: |
| 1019 | source.close() |
| 1020 | else: |
| 1021 | print("usage: issues.py list|show|record|approve|hide|close|reopen|publish|publish-all|ingest|imap-check (try --help)", file=sys.stderr) |
| 1022 | return 2 |
| 1023 | except (Error, OSError) as e: |
| 1024 | print(f"issues: {e}", file=sys.stderr) |
| 1025 | return 1 |
| 1026 | return 0 |
| 1027 | |
| 1028 | |
| 1029 | if __name__ == "__main__": |
| 1030 | sys.exit(main(sys.argv[1:])) |