fuzzyworld-tools / files / server / blog.py

blog.py

Blame: the commit that last changed each line · file · history

`0e603cf7` 2026-10-31 Pavel Safronov	1	#!/usr/bin/env python3
	2	"""Static blog generator (writing.fuzzyworld.net). Runs on the laptop, from Markdown files.
	3
	4	blog.py list <config> every source file: path -> URL, title, date, tags (and whether it is published)
	5	blog.py build <config> --out DIR write the site into DIR (--no-htaccess for a local preview)
	6	blog.py deploy-target <config> print host:dir from the config (used by `fw blog deploy`)
`9bb22e48` 2026-11-03 Pavel Safronov	7	blog.py deploy <config> --out DIR build, audit, ask, rsync (`fw blog deploy`); --dry-run, --yes
`0e603cf7` 2026-10-31 Pavel Safronov	8	blog.py audit DIR refuse (exit 1) if DIR holds anything that must not be uploaded
	9
	10	`fw blog ...` is the usual way in (it also installs Markdown and Pygments into ~/fw/pylib). The config is
	11	a JSON file next to the posts; its keys are listed in CONFIG_KEYS. `source` and `comments` are relative to
	12	the config file (so is the importer's `comments.json`); `pages_dir` and `media` are relative to `source`.
	13
	14	`timezone` (an IANA name, default UTC) says where post dates were written: feed timestamps use it.
	15
	16	A post is a Markdown file `YYYY-MM-DD-name.md`. Optional front matter (a `---` block of `key: value` lines)
	17	and the config's `posts` table can set published, date, slug, title, tags, body_until and wp_id; front matter
	18	wins over the config, which wins over what the file name and its first `# ` heading say. A post is published
	19	only when `published` is exactly true. Text under `## Spelling` and `## Removed writing`, and every HTML comment
	20	except <!--more-->, never reaches the output. A heading that looks like a private one but is not written as
	21	`## Spelling` (no space, an underlined heading, an unclosed code fence) stops the build instead of leaking.
	22
	23	A page (a file in `pages_dir`) is published only with `published: true` in its front matter; it may set slug and title.
	24
	25	Only the standard library is required, plus Python-Markdown (and Pygments for colored code) from ~/fw/pylib.
	26	"""
	27
	28	import argparse
	29	import calendar
	30	import dataclasses
	31	import datetime
`9bb22e48` 2026-11-03 Pavel Safronov	32	import fcntl
`0e603cf7` 2026-10-31 Pavel Safronov	33	import glob
	34	import html
	35	import json
	36	import math
	37	import os
	38	import re
`9bb22e48` 2026-11-03 Pavel Safronov	39	import hashlib
`0e603cf7` 2026-10-31 Pavel Safronov	40	import shutil
`9bb22e48` 2026-11-03 Pavel Safronov	41	import subprocess
`0e603cf7` 2026-10-31 Pavel Safronov	42	import sys
`9bb22e48` 2026-11-03 Pavel Safronov	43	import time
`0e603cf7` 2026-10-31 Pavel Safronov	44	import urllib.parse
	45	import zoneinfo
	46
	47	HERE = os.path.dirname(os.path.abspath(__file__))
	48	sys.path.insert(0, HERE)
	49	import atom # noqa: E402
	50	import gen # noqa: E402 (also puts ~/fw/pylib, or $FW_PYLIB, on sys.path)
	51
	52	markdown = gen.markdown
	53	esc, write = gen.esc, gen.write
	54
	55	SITE_DIR = os.path.join(HERE, "site") # _shared.css, favicon.ico: shared with code.fuzzyworld.net
	56	BLOG_DIR = os.path.join(HERE, "site-blog") # blog.css, blog.js, htaccess, robots.txt
	57	MARKER = ".fw-blog-out" # in an output folder this tool made: the only kind it empties
	58	# Media are copied by an allowlist of extensions: ~/Blog holds much that must never be published (Markdown,
	59	# JSON, notes, dotfiles). The audit applies the same rule again to everything in the build before an upload.
	60	MEDIA_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".pdf", ".epub", ".kpf", ".mp3", ".mp4", ".zip"}
	61	GENERATED_EXTENSIONS = {".html", ".xml", ".css", ".js", ".ico"} # what the generator itself writes
	62	GENERATED_NAMES = {"robots.txt", ".htaccess"} # the only other files allowed in an upload
	63	PRIVATE_SECTIONS = {"spelling", "removed writing"} # never published, wherever they are
	64	SLUG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._%-]*$") # every slug that becomes part of a path or a URL
	65	MORE = "<!--more-->"
	66	DEFAULT_URL = "/{yyyy}/{mm}/{dd}/{slug}/"
	67	POST_FIELDS = ("published", "date", "slug", "title", "tags", "body_until", "wp_id")
	68	PAGE_FIELDS = ("published", "slug", "title")
	69	CONFIG_KEYS = {"title", "tagline", "base_url", "author", "copyright", "source", "include", "body_section",
	70	"line_breaks", "url", "home", "pages_dir", "nav", "tags", "feed", "reading_wpm", "media",
`e9fdc5db` 2026-11-01 Pavel Safronov	71	"rewrite_hosts", "redirects", "wp_ids", "comments", "deploy", "posts", "timezone", "site",
`9bb22e48` 2026-11-03 Pavel Safronov	72	"comments_site", "comments_mailbox", "comments_form", "mirror", "autodeploy"}
`0e603cf7` 2026-10-31 Pavel Safronov	73	REQUIRED_KEYS = ("title", "base_url", "source")
	74	DEFAULTS = {"tagline": "", "author": "", "copyright": "© {years} {author}. All rights reserved.",
	75	"include": ["*.md"], "body_section": "Writing", "line_breaks": "markdown", "url": DEFAULT_URL,
	76	"home": None, "pages_dir": None, "nav": [], "tags": {}, "feed": {"path": "/feed/", "entries": 20},
	77	"reading_wpm": 200, "media": None, "rewrite_hosts": [], "redirects": {}, "wp_ids": {},
`e9fdc5db` 2026-11-01 Pavel Safronov	78	"comments": None, "deploy": None, "posts": {}, "timezone": "UTC", "site": None,
`9bb22e48` 2026-11-03 Pavel Safronov	79	"comments_site": None, "comments_mailbox": None, "comments_form": None,
	80	"mirror": None, "autodeploy": None}
`0e603cf7` 2026-10-31 Pavel Safronov	81
	82
	83	class BlogError(Exception):
	84	"""A problem with the config or a source file, reported without a traceback."""
	85
	86
	87	# ---------------------------------------------------------------- config
	88
	89	def load_config(path):
	90	"""The config as a dict with defaults filled in, plus `_dir` (the config's folder) and `_source` (absolute)."""
	91	path = os.path.abspath(path)
	92	try:
	93	with open(path, encoding="utf-8") as f:
	94	raw = json.load(f)
	95	except OSError as e:
	96	raise BlogError(f"cannot read config {path}: {e.strerror}")
	97	except ValueError as e:
	98	raise BlogError(f"config {path} is not valid JSON: {e}")
	99	if not isinstance(raw, dict):
	100	raise BlogError("config must be a JSON object")
	101	unknown = sorted(set(raw) - CONFIG_KEYS)
	102	if unknown:
	103	raise BlogError("unknown config key(s): " + ", ".join(unknown))
	104	missing = [k for k in REQUIRED_KEYS if not raw.get(k)]
	105	if missing:
	106	raise BlogError("config needs: " + ", ".join(missing))
	107	cfg = {k: (dict(v) if isinstance(v, dict) else list(v) if isinstance(v, list) else v) for k, v in DEFAULTS.items()}
	108	cfg.update(raw)
	109	cfg["feed"] = {DEFAULTS["feed"], (cfg["feed"] or {})}
	110	cfg["base_url"] = cfg["base_url"].rstrip("/")
	111	if cfg["line_breaks"] not in ("br", "markdown"):
	112	raise BlogError('line_breaks must be "br" or "markdown"')
	113	try:
	114	zoneinfo.ZoneInfo(str(cfg["timezone"]))
	115	except (zoneinfo.ZoneInfoNotFoundError, ValueError, OSError):
	116	raise BlogError(f'timezone {cfg["timezone"]!r} is not an IANA time zone name (e.g. "America/Los_Angeles")')
	117	if not isinstance(cfg["posts"], dict) or not all(isinstance(v, dict) for v in cfg["posts"].values()):
	118	raise BlogError("posts must map a file path to an object")
	119	for rel, entry in cfg["posts"].items():
	120	bad = sorted(set(entry) - set(POST_FIELDS))
	121	if bad:
	122	raise BlogError(f"posts[{rel}]: unknown key(s): " + ", ".join(bad))
	123	if not (isinstance(cfg["nav"], list) and all(isinstance(n, list) and len(n) == 2 for n in cfg["nav"])):
	124	raise BlogError("nav must be a list of [label, path] pairs")
`7e1f4430` 2026-11-01 Pavel Safronov	125	site = cfg["site"]
	126	if site is not None:
`d62e3357` 2026-11-01 Pavel Safronov	127	if isinstance(site, dict) and "footer" in site:
	128	raise BlogError('site: "footer" is set by "nav": "gen", not in the config')
`0330e724` 2026-11-01 Pavel Safronov	129	if isinstance(site, dict) and site.get("nav") == "gen":
`d62e3357` 2026-11-01 Pavel Safronov	130	site = cfg["site"] = {**site, "nav": gen.site_nav(), "footer": True} # "gen": the fuzzyworld nav and footer, built in gen.py
	131	if not (isinstance(site, dict) and set(site) <= {"name", "url", "nav", "current", "footer"} and {"name", "url", "nav"} <= set(site)
`7e1f4430` 2026-11-01 Pavel Safronov	132	and isinstance(site["name"], str) and isinstance(site["url"], str) and isinstance(site["nav"], list)
	133	and isinstance(site.get("current", ""), str)
	134	and all(isinstance(n, list) and len(n) == 2 and all(isinstance(x, str) for x in n) for n in site["nav"])):
	135	raise BlogError('site must be {"name", "url", "nav": [[label, url], ...], "current": label (optional)}')
`0e603cf7` 2026-10-31 Pavel Safronov	136	if not (cfg["url"].startswith("/") and cfg["url"].endswith("/") and "{slug}" in cfg["url"]):
	137	raise BlogError("url must start and end with / and contain {slug}")
	138	for old, new in cfg["redirects"].items():
	139	if (not isinstance(new, str) or htaccess_unsafe(old) or htaccess_unsafe(new) or old.strip("/") == ""):
	140	raise BlogError(f"redirects: bad entry {old!r} -> {new!r} (no spaces or control characters, even percent-encoded)")
	141	for key, value in cfg["wp_ids"].items():
	142	if not isinstance(value, str) or htaccess_unsafe(key) or htaccess_unsafe(value):
	143	raise BlogError(f"wp_ids: bad entry {key!r} -> {value!r} (no spaces or control characters, even percent-encoded)")
	144	if cfg["home"]:
	145	if not isinstance(cfg["home"], str):
	146	raise BlogError("home must be a page slug")
	147	check_slug(cfg["home"], "home")
	148	feed_path = cfg["feed"]["path"]
	149	segments = [x for x in str(feed_path).split("/") if x]
	150	if not (isinstance(feed_path, str) and feed_path.startswith("/") and segments) or htaccess_unsafe(feed_path):
	151	raise BlogError(f"feed.path {feed_path!r} must be a path like /feed/")
	152	for x in segments:
	153	check_slug(x, "feed.path")
	154	if not isinstance(cfg["tags"], dict):
	155	raise BlogError("tags must map a slug to a name")
	156	for slug in cfg["tags"]:
	157	check_slug(slug, "tags")
`e9fdc5db` 2026-11-01 Pavel Safronov	158	check_comments_config(cfg)
`9bb22e48` 2026-11-03 Pavel Safronov	159	check_watch_config(cfg)
	160	cfg["_path"] = path
`0e603cf7` 2026-10-31 Pavel Safronov	161	cfg["_dir"] = os.path.dirname(path)
	162	cfg["_source"] = os.path.normpath(os.path.join(cfg["_dir"], cfg["source"]))
	163	return cfg
	164
	165
`e9fdc5db` 2026-11-01 Pavel Safronov	166	def check_comments_config(cfg):
	167	"""Reader comments by email (and, off by default, a web form). Off unless comments_site is set."""
	168	site, mailbox, form = cfg["comments_site"], cfg["comments_mailbox"], cfg["comments_form"]
	169	if site is None:
	170	if mailbox is not None or form is not None:
	171	raise BlogError("comments_mailbox and comments_form need comments_site")
	172	return
	173	if not (isinstance(site, str) and re.fullmatch(r"[A-Za-z0-9_-]+", site)):
	174	raise BlogError('comments_site must be a short name such as "writing" (letters, digits, - and _)')
	175	if not (isinstance(mailbox, str) and re.fullmatch(r"[^@\s<>\"',;:()\[\]\\]+@[^@\s<>\"',;:()\[\]\\]+\.[^@\s<>\"',;:()\[\]\\]+", mailbox)):
	176	raise BlogError("comments_site needs comments_mailbox: the email address that receives comments")
	177	if form is not None:
	178	action = form.get("action") if isinstance(form, dict) else None
	179	parts = urllib.parse.urlsplit(action) if isinstance(action, str) else None
	180	if not (isinstance(form, dict) and set(form) == {"action"} and parts and parts.scheme == "https" and parts.netloc
	181	and not re.search(r"[\s\x00-\x1f\x7f\"'<>]", action)):
	182	raise BlogError('comments_form must be null or {"action": "https://..."} (an https URL without spaces or quotes)')
	183
	184
`9bb22e48` 2026-11-03 Pavel Safronov	185	_BRANCH = re.compile(r"^[A-Za-z0-9._/][A-Za-z0-9._/-]*$")
	186	_REMOTE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
	187
	188
	189	def check_watch_config(cfg):
	190	"""`mirror` and `autodeploy` (both off unless set) drive `fw blog watch` (see blog_watch.py)."""
	191	auto, mirror = cfg["autodeploy"], cfg["mirror"]
	192	if auto is not None:
	193	quiet = auto.get("quiet_seconds", 300) if isinstance(auto, dict) else None
	194	if not (isinstance(auto, dict) and set(auto) <= {"quiet_seconds"}
	195	and isinstance(quiet, int) and not isinstance(quiet, bool) and quiet >= 60):
	196	raise BlogError('autodeploy must be {"quiet_seconds": whole number, at least 60 (optional, default 300)}')
	197	if mirror is None:
	198	return
	199	if not (isinstance(mirror, dict) and set(mirror) <= {"repo", "branch", "remote", "exclude"}):
	200	raise BlogError('mirror must be {"repo", "branch" (optional), "remote" (optional), "exclude" (optional)}')
	201	repo = mirror.get("repo")
	202	if not (isinstance(repo, str) and repo.strip()):
	203	raise BlogError("mirror.repo must be a folder path (non-empty text)")
	204	branch = mirror.get("branch", "dropbox-mirror")
	205	if (not isinstance(branch, str) or not _BRANCH.match(branch) or ".." in branch or "//" in branch
	206	or branch.endswith(("/", ".", ".lock")) or branch in ("main", "master", "HEAD")):
	207	raise BlogError("mirror.branch must be a plain git branch name (letters, digits . _ / -; not starting with -), "
	208	"and not main, master or HEAD")
	209	remote = mirror.get("remote")
	210	if remote is not None and not (isinstance(remote, str) and _REMOTE.match(remote)):
	211	raise BlogError("mirror.remote must be a plain remote name (letters, digits . _ -)")
	212	exclude = mirror.get("exclude", [])
	213	if not (isinstance(exclude, list) and all(isinstance(x, str) and x.strip() and ".." not in x
	214	and not x.startswith("/..") and "\n" not in x
	215	and x[0] not in "!+-:" for x in exclude)):
	216	raise BlogError('mirror.exclude must be a list of rsync patterns (text; no "..", not starting with ! + - :)')
	217
	218
`0e603cf7` 2026-10-31 Pavel Safronov	219	def check_slug(slug, what):
	220	"""The slug, or BlogError: a slug becomes a folder name and part of a URL (and of .htaccess), so it is ASCII
	221	letters, digits and . _ % - only, and it must not decode to `.`, `..` or something with a slash."""
	222	decoded = urllib.parse.unquote(str(slug))
	223	if (not SLUG_RE.match(str(slug)) or decoded in (".", "..") or re.search(r"[/\\\x00-\x1f\x7f]", decoded)):
	224	raise BlogError(f"{what}: bad slug {slug!r} (use letters, digits and . _ % - only)")
	225	return slug
	226
	227
	228	def htaccess_unsafe(value):
	229	"""True when a value that goes into .htaccess has a space or control character, written out or percent-encoded."""
	230	text = str(value)
	231	return any(c.isspace() or ord(c) < 32 or ord(c) == 127 for c in text + urllib.parse.unquote(text))
	232
	233
	234	_DEPLOY_HOST = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._@-]*$")
	235	_DEPLOY_DIR = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-](/[A-Za-z0-9._-]+)$")
	236
	237
	238	def deploy_target(cfg):
	239	"""host:dir for rsync. The dir is relative to the login folder and must look like a site folder (e.g.
	240	writing.fuzzyworld.net): rsync --delete runs against it, so `~`, `.`, `/`, `..` and the like are refused."""
	241	deploy = cfg["deploy"] or {}
	242	host, folder = deploy.get("host"), deploy.get("dir")
	243	if not (isinstance(host, str) and isinstance(folder, str) and host and folder):
	244	raise BlogError("config needs deploy: {host, dir}")
	245	folder = folder.rstrip("/")
	246	if (not _DEPLOY_HOST.match(host) or not _DEPLOY_DIR.match(folder)
	247	or any(part in (".", "..") for part in folder.split("/")) or ".." in folder):
	248	raise BlogError(f"deploy.dir {deploy.get('dir')!r} (or host) is not safe: use a site folder such as "
	249	"writing.fuzzyworld.net, relative to the login folder")
	250	return f"{host}:{folder}"
	251
	252
	253	# ---------------------------------------------------------------- source files
	254
	255	_FM_LINE = re.compile(r"^([A-Za-z_][\w-]):\s(.?)\s$")
	256
	257
	258	def parse_front_matter(text):
	259	"""(meta dict, rest of the text). A leading `---` block counts only if it closes and every line is `key: value`;
	260	otherwise the `---` was a horizontal rule and there is no front matter."""
	261	lines = text.split("\n")
	262	if not lines or lines[0].strip() != "---":
	263	return {}, text
	264	for end in range(1, len(lines)):
	265	if lines[end].strip() == "---":
	266	break
	267	else:
	268	return {}, text
	269	meta = {}
	270	for line in lines[1:end]:
	271	if not line.strip():
	272	continue
	273	m = _FM_LINE.match(line)
	274	if not m:
	275	return {}, text
	276	key, value = m.groups()
	277	if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
	278	value = value[1:-1]
	279	meta[key] = {"true": True, "false": False}.get(value, value)
	280	return meta, "\n".join(lines[end + 1:])
	281
	282
	283	_FENCE = re.compile(r"^ {0,3}(`{3,}\|~{3,})(.*)$")
	284	_H2 = re.compile(r"^ {0,3}##(?!#)\s+(.?)\s#\s$")
	285	_ATX = re.compile(r"^\s#{1,6}(.)$")
	286	_UNDERLINE = re.compile(r"^\s(=+\|-+)\s$")
	287	_TEMPLATE_LINE = re.compile(r"^[ \t]Created by template, on .$", re.M)
	288
	289
	290	class _Fences:
	291	"""Fenced code blocks as CommonMark has them: a fence is 3+ backticks or tildes (a backtick fence has no backtick
	292	in its info string, so an inline ```span``` is not one); it ends at a line of the same character, at least as
	293	long, with nothing after it. feed(line) is True for a line that belongs to a block, the fence lines included."""
	294
	295	def __init__(self):
	296	self.mark = None # (character, length) while inside a block
	297
	298	def feed(self, line):
	299	m = _FENCE.match(line)
	300	if self.mark is None:
	301	if m and not (m.group(1)[0] == "`" and "`" in m.group(2)):
	302	self.mark = (m.group(1)[0], len(m.group(1)))
	303	return True
	304	return False
	305	if m and m.group(1)[0] == self.mark[0] and len(m.group(1)) >= self.mark[1] and not m.group(2).strip():
	306	self.mark = None
	307	return True
	308
	309
	310	def remove_comments(text, name=None):
	311	"""Cuts every HTML comment (private notes must not reach page source) except the <!--more--> marker.
	312	A comment ends at its first `-->`, but a <!--more--> inside a comment belongs to the comment. A comment that
	313	is never closed is cut to the end of the text (with a warning on stderr when `name` says which file)."""
	314	out, i = [], 0
	315	while True:
	316	start = text.find("<!--", i)
	317	if start == -1:
	318	out.append(text[i:])
	319	break
	320	out.append(text[i:start])
	321	if text.startswith(MORE, start):
	322	out.append(MORE)
	323	i = start + len(MORE)
	324	continue
	325	j = start + 4
	326	while True:
	327	end, more = text.find("-->", j), text.find(MORE, j)
	328	if more != -1 and (end == -1 or more < end):
	329	j = more + len(MORE)
	330	continue
	331	break
	332	if end == -1:
	333	if name:
	334	print(f"blog.py: warning: {name}: a `<!--` is never closed; the text from it to the end is left out",
	335	file=sys.stderr)
	336	break
	337	i = end + 3
	338	return "".join(out)
	339
	340
	341	def parse_sections(text, name=None):
	342	"""(title, preamble lines, [(heading, lines)]) of comment-free text. The title is the first `# ` line
	343	(not kept in any lines); each `## ` section's lines start with its heading line. Fenced code is never
	344	mistaken for headings."""
	345	title, pre, sections, fences = None, [], [], _Fences()
	346	for line in remove_comments(text, name).replace("\r\n", "\n").split("\n"):
	347	if not fences.feed(line):
	348	if title is None and line.startswith("# "):
	349	title = line[2:].strip()
	350	continue
	351	h = _H2.match(line)
	352	if h:
	353	sections.append((h.group(1), [line]))
	354	continue
	355	(sections[-1][1] if sections else pre).append(line)
	356	return title, pre, sections
	357
	358
	359	def _private_name(text):
	360	return text.strip(" \t#:*_`").casefold() in PRIVATE_SECTIONS
	361
	362
	363	def check_private_headings(lines, name=None):
	364	"""Fail closed: the body about to be published must hold nothing that looks like a private heading written some
	365	other way than `## Spelling` (`##Spelling`, `Spelling` over a line of dashes, `### Spelling` ...) and no
	366	code fence that is never closed (it would hide what follows from the section parser)."""
	367	fences, prev = _Fences(), None
	368	for n, line in enumerate(lines, 1):
	369	if fences.feed(line):
	370	prev = None
	371	continue
	372	m = _ATX.match(line)
	373	if (m and _private_name(m.group(1))) or (not m and prev is not None and _UNDERLINE.match(line) and _private_name(prev)):
	374	raise BlogError(f"{name or 'text'}: line {n} of the publishable text looks like a private heading "
	375	f"({(prev if not m else line).strip()!r}); write it as `## Spelling` / `## Removed writing` "
	376	"on a line of its own, or reword it")
	377	prev = line
	378	if fences.mark:
	379	raise BlogError(f"{name or 'text'}: a code fence is never closed; fix it so private sections are not hidden by it")
	380
	381
	382	def title_of(text):
	383	return parse_sections(text)[0]
	384
	385
	386	def _collect(sections, until):
	387	"""Lines of the sections up to the one headed `until` (exclusive), skipping private ones."""
	388	out = []
	389	for heading, lines in sections:
	390	if until and heading.casefold() == until.casefold():
	391	break
	392	if heading.casefold() not in PRIVATE_SECTIONS:
	393	out += lines
	394	return out
	395
	396
	397	def extract_body(text, section="Writing", until=None, name=None):
	398	"""The publishable Markdown of a source file.
	399
	400	With a `## <section>` heading: that section, up to the next `## ` heading, or (when `until` names a heading)
	401	through every section up to it. Without one (or with section=None): everything after the `# ` title line.
	402	Private sections, HTML comments (except <!--more-->) and the "Created by template" line are always dropped.
	403	BlogError (naming `name`) if what is left has a private-looking heading in another form, or an unclosed fence.
	404	"""
	405	_, pre, sections = parse_sections(text, name)
	406	target = next((i for i, (h, _) in enumerate(sections) if section and h.casefold() == section.casefold()), None)
	407	if target is not None:
	408	lines = sections[target][1][1:] + (_collect(sections[target + 1:], until) if until else [])
	409	else:
	410	lines = pre + _collect(sections, until)
	411	body = _TEMPLATE_LINE.sub("", "\n".join(lines)).strip("\n").strip()
	412	check_private_headings(body.split("\n"), name)
	413	return body
	414
	415
	416	@dataclasses.dataclass
	417	class Post:
	418	rel: str # path under `source`, with /
	419	text: str # the file without its front matter
	420	published: bool = False
	421	date: datetime.datetime = None
	422	slug: str = ""
	423	title: str = ""
	424	tags: list = dataclasses.field(default_factory=list)
	425	body_until: str = None
	426	wp_id: int = None
	427	path: str = "" # URL path, e.g. /2018/03/02/slug/
	428
	429
	430	def parse_date(value):
	431	"""datetime from "YYYY-MM-DD" or "YYYY-MM-DD[ T]HH:MM[:SS]". A time zone is dropped: dates are WordPress local dates."""
	432	if isinstance(value, datetime.datetime):
	433	return value.replace(tzinfo=None)
	434	try:
	435	return datetime.datetime.fromisoformat(str(value).strip()).replace(tzinfo=None)
	436	except ValueError:
	437	raise BlogError(f"bad date {value!r} (use YYYY-MM-DD)")
	438
	439
	440	def localize(cfg, when):
	441	"""A post's naive date as a time-aware datetime in the config's `timezone` (for feed timestamps)."""
	442	return when.replace(tzinfo=zoneinfo.ZoneInfo(cfg["timezone"])) if when.tzinfo is None else when
	443
	444
	445	def as_list(value):
	446	if isinstance(value, str):
	447	value = value.strip()
	448	if value.startswith("[") and value.endswith("]"):
	449	value = value[1:-1]
	450	return [t for t in (t.strip().strip("\"'").strip() for t in value.split(",")) if t]
	451	return [str(t).strip() for t in (value or []) if str(t).strip()]
	452
	453
	454	_NAME = re.compile(r"^(\d{4}-\d{2}-\d{2})-(.+)\.md$", re.I)
	455
	456
	457	def post_url(cfg, date, slug):
	458	return (cfg["url"].replace("{yyyy}", f"{date.year:04d}").replace("{mm}", f"{date.month:02d}")
	459	.replace("{dd}", f"{date.day:02d}").replace("{slug}", slug))
	460
	461
	462	def source_files(cfg, patterns=None, skip_dir=None):
	463	"""Relative paths of the source files matching the include patterns, sorted. Dropbox's "conflicted copy"
	464	files and hidden folders are left out."""
	465	src, found = cfg["_source"], set()
	466	for pattern in patterns or cfg["include"]:
	467	for p in glob.glob(os.path.join(glob.escape(src), pattern), recursive=True):
	468	rel = os.path.relpath(p, src).replace(os.sep, "/")
	469	parts = rel.split("/")
	470	if (not os.path.isfile(p) or "conflicted copy" in rel.lower() or any(x.startswith(".") for x in parts)
	471	or (skip_dir and rel.startswith(skip_dir + "/"))):
	472	continue
	473	found.add(rel)
	474	return sorted(found)
	475
	476
	477	def read_text(path):
	478	with open(path, encoding="utf-8-sig") as f:
	479	return f.read().replace("\r\n", "\n")
	480
	481
	482	def pages_dir(cfg):
	483	return cfg["pages_dir"].strip("/") if cfg["pages_dir"] else None
	484
	485
	486	def scan_posts(cfg):
	487	"""Every post file as a Post, oldest first. Precedence: front matter > config posts[<path>] > the file."""
	488	posts = []
	489	for rel in source_files(cfg, skip_dir=pages_dir(cfg)):
	490	meta, text = parse_front_matter(read_text(os.path.join(cfg["_source"], rel)))
	491	bad = sorted(set(meta) - set(POST_FIELDS))
	492	if bad:
	493	raise BlogError(f"{rel}: unknown front matter key(s): " + ", ".join(bad))
	494	merged = {cfg["posts"].get(rel, {}), meta}
	495	name = _NAME.match(os.path.basename(rel))
	496	date = merged.get("date") or (name.group(1) if name else None)
	497	stem = name.group(2) if name else os.path.splitext(os.path.basename(rel))[0]
	498	post = Post(rel=rel, text=text,
	499	published=merged.get("published") is True,
	500	date=parse_date(date) if date else None,
	501	slug=str(merged.get("slug") or stem),
	502	title=str(merged.get("title") or title_of(text) or stem),
	503	tags=as_list(merged.get("tags")),
	504	body_until=merged.get("body_until") or None,
	505	wp_id=int(merged["wp_id"]) if str(merged.get("wp_id", "")).isdigit() else None)
	506	if post.published: # a draft's file name may hold anything; it is never written anywhere
	507	check_slug(post.slug, rel)
	508	if post.date:
	509	post.path = post_url(cfg, post.date, post.slug)
	510	posts.append(post)
	511	posts.sort(key=lambda p: (p.date or datetime.datetime.min, p.rel))
	512	return posts
	513
	514
	515	def slugify(text):
	516	return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
	517
	518
	519	def tag_slug(cfg, tag):
	520	"""A post's tag may be written as a configured slug, a configured name or any other text."""
	521	if tag in cfg["tags"]:
	522	return tag
	523	for slug, name in cfg["tags"].items():
	524	if name == tag:
	525	return slug
	526	return slugify(tag) or "tag"
	527
	528
	529	def tag_name(cfg, slug, fallback=None):
	530	return cfg["tags"].get(slug) or fallback or slug
	531
	532
	533	def list_posts(cfg):
	534	lines = []
	535	for p in scan_posts(cfg):
	536	tags = ", ".join(p.tags)
	537	lines.append(f'{"published" if p.published else "draft "} {p.rel} -> {p.path or "(no date)"} '
	538	f'{p.title!r} {p.date.date() if p.date else "no date"}' + (f" tags: {tags}" if tags else ""))
	539	return lines
	540
	541
	542	# ---------------------------------------------------------------- rendering
	543
	544	def redirect_lookup(cfg):
	545	"""{path: target} from the config's redirects and wp_ids, tolerant of a missing or extra trailing slash."""
	546	if "_lookup" not in cfg:
	547	table = {}
	548	for old, new in cfg["redirects"].items():
	549	for key in (old, urllib.parse.unquote(old)):
	550	table[key] = new
	551	table[key.rstrip("/") + "/"] = new
	552	table[key.rstrip("/")] = new
	553	cfg["_lookup"] = table
	554	return cfg["_lookup"]
	555
	556
	557	def rewrite_url(url, cfg):
	558	"""Absolute links to the old host become site-relative; internal links go through `redirects`
	559	(and WordPress's /?p=ID short links through `wp_ids`)."""
	560	for prefix in cfg["rewrite_hosts"]:
	561	p = prefix.rstrip("/")
	562	if url == p or url[len(p):len(p) + 1] in ("/", "?", "#") and url.startswith(p):
	563	url = url[len(p):]
	564	url = url if url.startswith("/") else "/" + url
	565	break
	566	if not url.startswith("/") or url.startswith("//"):
	567	return url
	568	m = re.match(r"([^?#])(.)$", url)
	569	path, tail = m.group(1), m.group(2)
	570	if path == "/":
	571	q = urllib.parse.parse_qs(tail.lstrip("?").split("#")[0])
	572	for key in ("p", "page_id"):
	573	target = cfg["wp_ids"].get((q.get(key) or [None])[0])
	574	if target:
	575	return target
	576	target = redirect_lookup(cfg).get(path)
	577	if target:
	578	return target if re.search(r"[?#]", target) else target + tail
	579	return url
	580
	581
	582	_ATTR = re.compile(r'(<(?:img\|source)\b[^>]?\bsrc\|<a\b[^>]?\bhref)=(["\'])(.*?)\2', re.I \| re.S)
	583
	584
	585	def map_links(html_text, fn):
	586	def fix(m):
	587	return f"{m.group(1)}={m.group(2)}{esc(fn(html.unescape(m.group(3))), quote=True)}{m.group(2)}"
	588	return _ATTR.sub(fix, html_text)
	589
	590
	591	_CODE = re.compile(r'<pre><code(?: class="language-([\w+#.-]+)")?>(.*?)</code></pre>', re.S)
	592
	593
	594	def highlight_blocks(html_text):
	595	"""Fenced code blocks with a language get Pygments token spans (the classes gen.py uses)."""
	596	def fix(m):
	597	lang = m.group(1)
	598	if not lang or not gen.pygments_lexers:
	599	return m.group(0)
	600	try:
	601	lexer = gen.pygments_lexers.get_lexer_by_name(lang, stripnl=False, ensurenl=False)
	602	parts = []
	603	for ttype, value in lexer.get_tokens(html.unescape(m.group(2))):
	604	cls = gen.token_class(ttype)
	605	parts.append(f'<span class="{cls}">{esc(value)}</span>' if cls else esc(value))
	606	except Exception: # unknown language or a lexer bug: leave the block plain
	607	return m.group(0)
	608	return f'<pre class="hl"><code class="language-{lang}">{"".join(parts)}</code></pre>'
	609	return _CODE.sub(fix, html_text)
	610
	611
	612	def render_markdown(text, cfg):
	613	if not markdown:
	614	raise BlogError("Python-Markdown is not installed in " + gen.PYLIB + " (fw blog installs it)")
	615	extensions = ["tables", "fenced_code", "sane_lists"] + (["nl2br"] if cfg["line_breaks"] == "br" else [])
	616	out = markdown.markdown(text, extensions=extensions, output_format="html")
	617	return map_links(highlight_blocks(out), lambda u: rewrite_url(u, cfg))
	618
	619
	620	def plain_text(html_text):
	621	return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", html_text))).strip()
	622
	623
	624	def split_more(text):
	625	"""(excerpt Markdown or None, full Markdown) around the <!--more--> marker."""
	626	if "<!--more-->" not in text:
	627	return None, text
	628	head, tail = text.split("<!--more-->", 1)
	629	return head.strip(), head.rstrip() + "\n\n" + tail.replace("<!--more-->", "").lstrip()
	630
	631
	632	def blocks(html_text):
	633	"""The text blocks of rendered HTML as (kind, escaped-free text) pairs: kind is "p" or "h" (a chapter heading).
	634	<br> is kept as a newline; other markup and images are dropped."""
	635	out = []
	636	for m in re.finditer(r"<(p\|h[1-6])(?:\s[^>])?>(.?)</\1>", html_text, re.S):
	637	marked = re.sub(r"<br\s*/?>", "\x00", m.group(2))
	638	text = html.unescape(re.sub(r"<[^>]+>", "", marked))
	639	text = "\n".join(re.sub(r"[ \t]+", " ", line).strip() for line in text.split("\x00")).strip()
	640	if text:
	641	out.append(("p" if m.group(1) == "p" else "h", text))
	642	return out
	643
	644
	645	def make_excerpt(excerpt_md, body_html, cfg, words=80):
	646	"""(HTML for the lists, whether there is more to read). What is before <!--more-->, else whole blocks from the
	647	start until about `words` words (a first paragraph longer than that is cut at a word, even after headings). Chapter headings are
	648	kept, but never left dangling at the end."""
	649	cut = excerpt_md is not None
	650	found = blocks(render_markdown(excerpt_md, cfg) if cut else body_html)
	651	if cut:
	652	kept, more = found, True
	653	else:
	654	kept, n, more = [], 0, False
	655	for kind, text in found:
	656	w = len(text.split())
	657	if kind == "p" and w > words and not any(k == "p" for k, _ in kept):
	658	text, more = " ".join(text.split(" ")[:words]) + "…", True # the first paragraph, headings or not
	659	elif kept and n + w > words:
	660	break
	661	kept.append((kind, text))
	662	n += w
	663	while kept and kept[-1][0] == "h" and len(kept) < len(found):
	664	kept.pop()
	665	more = more or len(kept) < len(found)
	666	html_out = "".join(("<p>" if k == "p" else '<h5 class="chapter">') + esc(t).replace("\n", "<br>\n") + ("</p>" if k == "p" else "</h5>")
	667	for k, t in kept)
	668	return html_out, bool(more)
	669
	670
	671	def absolutize(html_text, base_url):
	672	"""For the feed: site-relative links become absolute."""
	673	return map_links(html_text, lambda u: base_url + u if u.startswith("/") and not u.startswith("//") else u)
	674
	675
`e9fdc5db` 2026-11-01 Pavel Safronov	676	def comments_src(path):
	677	"""Where the server publishes a post's comments: /2024/05/01/slug/ -> /_comments/2024/05/01/slug.json"""
	678	return "/_comments/" + path.strip("/") + ".json"
	679
	680
	681	def comments_widget(cfg, path):
	682	"""The container blog.js fills, the "Comment by email" link and (only with comments_form) the web form."""
	683	site = cfg["comments_site"]
	684	subject = urllib.parse.quote(f"Comment [{site}:{path}]", safe="")
	685	mailto = f"mailto:{urllib.parse.quote(cfg['comments_mailbox'], safe='@')}?subject={subject}"
	686	html_ = ('<div class="comments-live"></div>'
	687	f'<p class="comment-by-email"><a href="{esc(mailto, quote=True)}">Comment by email</a>. '
	688	'Comments are moderated, and your address is never shown.</p>')
	689	form = cfg["comments_form"]
	690	if form:
	691	html_ += (f'<form class="comment-form" method="post" action="{esc(form["action"], quote=True)}">'
	692	f'<input type="hidden" name="site" value="{esc(site, quote=True)}">'
	693	f'<input type="hidden" name="path" value="{esc(path, quote=True)}">'
	694	'<input type="hidden" name="elapsed" value="0">'
	695	'<div class="hp" aria-hidden="true"><input type="text" name="website" tabindex="-1" autocomplete="off"></div>'
	696	'<p><label>Name <input type="text" name="name" maxlength="60" required></label></p>'
	697	'<p><label>Comment<br><textarea name="text" rows="5" maxlength="4000" required></textarea></label></p>'
	698	'<p><button type="submit">Send comment</button></p></form>')
	699	return html_
	700
	701
	702	def render_comments(comments, widget=None, src=None):
	703	"""Threaded comments: name, date and text only. Everything is escaped; newlines become <br>. Replies nest.
	704	With `src` (comments_site is set) the section is always written: it carries data-src, and `widget` follows the list."""
`0e603cf7` 2026-10-31 Pavel Safronov	705	ids = {c.get("id") for c in comments}
	706	children = {}
	707	for c in sorted(comments, key=lambda c: str(c.get("date", ""))):
	708	parent = c.get("parent") or 0
	709	children.setdefault(parent if parent in ids and parent != c.get("id") else 0, []).append(c)
	710
	711	def tree(parent):
	712	items = []
	713	for c in children.get(parent, []):
	714	text = esc(str(c.get("text", "")).replace("\r\n", "\n")).replace("\n", "<br>\n")
	715	nested = tree(c.get("id")) if c.get("id") in children else ""
	716	items.append(f'<li class="comment" id="comment-{re.sub(r"[^A-Za-z0-9_-]", "", str(c.get("id", "")))}">'
	717	f'<p class="meta"><strong>{esc(str(c.get("author", "")))}</strong> · '
	718	f'{esc(str(c.get("date", ""))[:10])}</p><p>{text}</p>{nested}</li>')
	719	return f'<ul class="comments">{"".join(items)}</ul>' if items else ""
	720
`e9fdc5db` 2026-11-01 Pavel Safronov	721	if src:
	722	return (f'<section class="comments-section" data-src="{esc(src, quote=True)}"><h2>Comments</h2>' + tree(0)
	723	+ (widget or "") + '<p class="meta">Comments © their authors.</p></section>')
`0e603cf7` 2026-10-31 Pavel Safronov	724	if not comments:
	725	return ""
	726	return ('<section class="comments-section"><h2>Comments</h2>' + tree(0)
	727	+ '<p class="meta">Comments © their authors.</p></section>')
	728
	729
`e9fdc5db` 2026-11-01 Pavel Safronov	730	def post_comments(cfg, comments, post):
	731	if not cfg["comments_site"]:
	732	return render_comments(comments.get(post.path, []))
	733	return render_comments(comments.get(post.path, []), comments_widget(cfg, post.path), comments_src(post.path))
	734
	735
`0e603cf7` 2026-10-31 Pavel Safronov	736	def load_comments(cfg):
	737	if not cfg["comments"]:
	738	return {}
	739	path = os.path.join(cfg["_dir"], cfg["comments"])
	740	if not os.path.exists(path):
	741	return {}
	742	try:
	743	with open(path, encoding="utf-8") as f:
	744	data = json.load(f)
	745	except ValueError as e:
	746	raise BlogError(f"comments file {path} is not valid JSON: {e}")
	747	return data if isinstance(data, dict) else {}
	748
	749
	750	def copyright_text(cfg):
	751	return (cfg["copyright"].replace("{years}", cfg.get("_years", str(datetime.date.today().year)))
	752	.replace("{author}", cfg["author"]))
	753
	754
	755	def theme_overrides(shared_css):
	756	"""CSS that lets a reader's choice (<html data-theme="light\|dark">) beat the system setting. _shared.css holds the
	757	light colors in :root and the dark ones in a prefers-color-scheme media query; both blocks are repeated here under
	758	the attribute, taken from _shared.css itself so they cannot drift."""
	759	light = re.search(r":root\s\{([^}]--color-bg-primary[^}]*)\}", shared_css)
	760	dark = re.search(r"@media $prefers-color-scheme: dark$\s\{\s:root\s\{([^}])\}", shared_css)
	761	if not (light and dark):
	762	raise BlogError("_shared.css: cannot find the light and dark color blocks")
	763	def decls(block):
	764	return "\n".join(" " + line.strip() for line in block.splitlines() if line.strip().startswith("--"))
	765	return (f'\n:root[data-theme="light"] {{\n color-scheme: light;\n{decls(light.group(1))}\n}}\n'
	766	f':root[data-theme="dark"] {{\n color-scheme: dark;\n{decls(dark.group(1))}\n}}\n')
	767
	768
	769	def blog_page(cfg, title, body, path):
	770	"""A full page. `path` is the page's own URL path (for the canonical link and the current nav entry)."""
	771	links = "".join(f'<a href="{esc(p, quote=True)}"' + (' aria-current="page"' if p == path else "") + f'>{esc(label)}</a>'
	772	for label, p in cfg["nav"])
`7e1f4430` 2026-11-01 Pavel Safronov	773	tagline = f'<span class="tagline">{esc(cfg["tagline"])}</span>' if cfg["tagline"] else ""
	774	site = cfg["site"]
	775	if site:
	776	# The same bar as on code.fuzzyworld.net and apps.fuzzyworld.net, then the blog's own bar under it.
	777	site_links = "".join(f'<a href="{esc(u, quote=True)}"' + (' aria-current="page"' if label == site.get("current") else "")
	778	+ f'>{esc(label)}</a>' for label, u in site["nav"])
	779	header = (f'<header class="site"><div class="bar"><a class="brand" href="{esc(site["url"], quote=True)}">{esc(site["name"])}</a>'
	780	f'<nav class="site-nav" aria-label="Site">{site_links}</nav></div>\n'
	781	f'<div class="bar sub"><a class="title" href="/">{esc(cfg["title"])}</a>{tagline}'
	782	f'<nav class="site-nav" aria-label="This blog">{links}</nav></div></header>')
	783	else:
	784	header = (f'<header class="site"><div class="bar"><a class="brand" href="/">{esc(cfg["title"])}</a>{tagline}'
	785	f'<nav class="site-nav" aria-label="Site">{links}</nav></div></header>')
`0e603cf7` 2026-10-31 Pavel Safronov	786	feed = cfg["feed"]["path"]
`d62e3357` 2026-11-01 Pavel Safronov	787	legal = (f'<p>{esc(copyright_text(cfg))}</p>\n'
	788	f'<p><a href="{esc(site["url"] if site else "https://fuzzyworld.net/", quote=True)}">{esc(site["name"] if site else "fuzzyworld.net")}</a>'
	789	f' · <a href="{esc(feed, quote=True)}">Feed</a></p>')
	790	if site and site.get("footer"): # the site-wide four-column footer, then the blog's own line under it
	791	footer = f'<footer class="site"><div class="cols">{gen.site_footer("writing")}</div>\n<div class="cols legal">\n{legal}\n</div></footer>'
	792	else:
	793	footer = f'<footer class="site"><div class="cols">\n{legal}\n</div></footer>'
`0e603cf7` 2026-10-31 Pavel Safronov	794	head_title = cfg["title"] if path == "/" else f'{title} · {cfg["title"]}'
	795	return f"""<!doctype html>
	796	<html lang="en">
	797	<head>
	798	<meta charset="utf-8">
	799	<meta name="viewport" content="width=device-width, initial-scale=1">
	800	<title>{esc(head_title)}</title>
	801	<link rel="canonical" href="{esc(cfg["base_url"] + path, quote=True)}">
	802	<link rel="alternate" type="application/atom+xml" title="{esc(cfg["title"], quote=True)}" href="{esc(feed, quote=True)}">
	803	<link rel="icon" href="/favicon.ico">
	804	<link rel="stylesheet" href="/_shared.css">
	805	<link rel="stylesheet" href="/blog.css">
	806	<script src="/blog.js"></script>
	807	</head>
	808	<body>
`7e1f4430` 2026-11-01 Pavel Safronov	809	{header}
`0e603cf7` 2026-10-31 Pavel Safronov	810	<main>
	811	{body}
	812	</main>
`d62e3357` 2026-11-01 Pavel Safronov	813	{footer}
`0e603cf7` 2026-10-31 Pavel Safronov	814	</body>
	815	</html>
	816	"""
	817
	818
	819	def post_meta_line(cfg, post, minutes):
	820	tags = "".join(f' <a class="tag" href="/tag/{esc(s, quote=True)}/">{esc(tag_name(cfg, s, t))}</a>'
	821	for s, t in ((tag_slug(cfg, t), t) for t in post.tags))
	822	return (f'<p class="meta"><time datetime="{post.date.date()}">{post.date.date()}</time> · {minutes} min read'
	823	+ (" ·" + tags if tags else "") + "</p>")
	824
	825
	826	def listing(cfg, items, headings=True):
	827	"""Posts (dicts with post, minutes, excerpt) as a list, newest first; with `headings`, grouped by year and month."""
	828	out, year, month = [], None, None
	829	for it in items:
	830	p = it["post"]
	831	if headings and p.date.year != year:
	832	year, month = p.date.year, None
	833	out.append(f'<h2 id="{year}">{year}</h2>')
	834	if headings and (p.date.year, p.date.month) != month:
	835	month = (p.date.year, p.date.month)
	836	out.append(f'<h3 id="{year}-{p.date.month:02d}">{calendar.month_name[p.date.month]} {year}</h3>')
	837	out.append(f'<article class="entry"><h4><a href="{esc(p.path, quote=True)}">{esc(p.title)}</a></h4>'
	838	f'{post_meta_line(cfg, p, it["minutes"])}'
	839	+ (f'<div class="excerpt">{it["excerpt"][0]}</div>' if it["excerpt"][0] else "")
	840	+ (f'<p class="more"><a href="{esc(p.path, quote=True)}">Continue reading →</a></p>' if it["excerpt"][1] else "")
	841	+ "</article>")
	842	return "\n".join(out)
	843
	844
	845	# ---------------------------------------------------------------- site files
	846
	847	def apache_pattern(path):
	848	"""A RewriteRule pattern (per-directory context: no leading slash) matching exactly this path, with or without a trailing slash."""
	849	text = urllib.parse.unquote(path).strip("/")
	850	text = re.sub(r"([.^$*+?()\[\]{}\|\\])", r"\\\1", text).replace(" ", r"\x20")
	851	return f"^{text}/?$"
	852
	853
	854	def redirect_rules(cfg):
	855	"""Apache rules for the old WordPress URLs, from the config and the fixed WordPress URL shapes."""
	856	rules = ["", "# Redirects from the old WordPress site (generated by blog.py from the config)."]
	857	for old, new in sorted(cfg["redirects"].items()):
	858	rules.append(f"RewriteRule {apache_pattern(old)} {new} [R=301,L]")
	859	for wp_id, path in sorted(cfg["wp_ids"].items(), key=lambda kv: str(kv[0])):
	860	for param in ("p", "page_id"):
	861	rules += [f"RewriteCond %{{QUERY_STRING}} (^\|&){param}={re.escape(str(wp_id))}(&\|$)",
	862	f"RewriteRule ^$ {path}? [R=301,L]"]
	863	if cfg["home"]:
	864	rules.append(f"RewriteRule {apache_pattern(cfg['home'])} / [R=301,L]")
	865	feed = cfg["feed"]["path"]
	866	rules += [f"RewriteRule ^comments/feed/?$ {feed} [R=301,L]",
	867	f"RewriteRule ^feed/(rss2\|rss\|rdf\|atom)/?$ {feed} [R=301,L]",
	868	"RewriteRule ^tag/([^/]+)/feed/?$ /tag/$1/ [R=301,L]"]
	869	if cfg["url"] == DEFAULT_URL:
	870	rules.append(r"RewriteRule ^(\d{4}/\d\d/\d\d/[^/]+)/(feed\|amp\|comment-page-\d+\|attachment/.+)/?$ /$1/ [R=301,L]")
	871	rules += ["RewriteRule ^(category\|author)/.*$ /blog/ [R=301,L]",
	872	r"RewriteRule ^page/\d+/?$ /blog/ [R=301,L]",
	873	r"RewriteRule ^(\d{4})/?$ /blog/#$1 [NE,R=301,L]",
	874	r"RewriteRule ^(\d{4})/(\d\d)/?$ /blog/#$1-$2 [NE,R=301,L]",
	875	r"RewriteRule ^(wp-admin\|wp-login\.php\|xmlrpc\.php) - [R=404,L]"]
	876	return "\n".join(rules) + "\n"
	877
	878
	879	def htaccess_text(cfg):
	880	with open(os.path.join(BLOG_DIR, "htaccess"), encoding="utf-8") as f:
	881	return f.read().rstrip("\n") + "\n" + redirect_rules(cfg)
	882
	883
	884	def copy_media(cfg, out):
	885	"""The media folder's files, as they are (so /wp-content/uploads/... paths keep working). Only file types
	886	a browser shows or plays: nothing the web server could execute. Symbolic links (files or folders) are skipped."""
	887	if not cfg["media"]:
	888	return 0
	889	root, n = os.path.join(cfg["_source"], cfg["media"]), 0
	890	for dirpath, dirs, files in os.walk(root):
	891	dirs[:] = [d for d in dirs if not d.startswith(".") and not os.path.islink(os.path.join(dirpath, d))]
	892	for name in files:
	893	if (name.startswith(".") or os.path.splitext(name)[1].lower() not in MEDIA_EXTENSIONS
	894	or os.path.islink(os.path.join(dirpath, name))): # a link could point at anything on the laptop
	895	continue
	896	dest = os.path.join(out, os.path.relpath(os.path.join(dirpath, name), root))
	897	os.makedirs(os.path.dirname(dest), exist_ok=True)
	898	shutil.copyfile(os.path.join(dirpath, name), dest)
	899	n += 1
	900	return n
	901
	902
	903	def audit_output(out):
	904	"""(file count, total bytes) of what an upload of `out` would send; BlogError if anything must not be published.
	905	Allowed: generated pages, feed, css, icon, robots.txt, .htaccess, and media with an allowlisted extension.
	906	Anything else (a .md, .json or .txt file, any other dotfile, a symlink) is refused. The build's own marker
	907	file is skipped: the upload excludes it."""
	908	if not os.path.isdir(out):
	909	raise BlogError(f"{out} is not a folder")
	910	bad, count, size = [], 0, 0
	911	for dirpath, dirs, files in os.walk(out):
	912	for name in dirs:
	913	if name.startswith(".") or os.path.islink(os.path.join(dirpath, name)):
	914	bad.append(os.path.relpath(os.path.join(dirpath, name), out))
	915	for name in files:
	916	path = os.path.join(dirpath, name)
	917	if name == MARKER and dirpath == out:
	918	continue
	919	ext = os.path.splitext(name)[1].lower()
	920	if (os.path.islink(path) or not (name in GENERATED_NAMES or (not name.startswith(".")
	921	and ext in GENERATED_EXTENSIONS \| MEDIA_EXTENSIONS))):
	922	bad.append(os.path.relpath(path, out))
	923	continue
	924	count += 1
	925	size += os.path.getsize(path)
	926	if bad:
	927	raise BlogError("refusing to upload; not allowed in the output: " + ", ".join(sorted(bad)[:20])
	928	+ (f" (and {len(bad) - 20} more)" if len(bad) > 20 else ""))
	929	return count, size
	930
	931
	932	def safe_join(out, *parts):
	933	"""out/parts..., or BlogError when it would land outside `out` (a bad slug, a symlink in the way)."""
	934	dest = os.path.join(out, *parts)
	935	root, real = os.path.realpath(out), os.path.realpath(dest)
	936	if not real.startswith(root + os.sep):
	937	raise BlogError(f"refusing to write outside the output folder: {os.path.join(*parts)!r}")
	938	return dest
	939
	940
	941	def prepare_out(out, cfg=None):
	942	"""Empty the output folder, but only one this tool made (or an empty one): `--out ~` must not wipe a home folder."""
	943	if cfg:
	944	out_real, src = os.path.realpath(out), os.path.realpath(cfg["_source"])
	945	if out_real == src or out_real.startswith(src + os.sep) or src.startswith(out_real + os.sep):
	946	raise BlogError("--out must be a separate folder, not the source folder or inside it")
	947	if os.path.exists(out):
	948	if os.listdir(out) and not os.path.exists(os.path.join(out, MARKER)):
	949	raise BlogError(f"{out} is not empty and was not made by fw blog; choose another --out")
	950	shutil.rmtree(out)
	951	os.makedirs(out)
	952	write(os.path.join(out, MARKER), "Made by fw blog build. It is emptied on every build.\n")
	953
	954
	955	# ---------------------------------------------------------------- build
	956
	957	def build(cfg, out, htaccess=True):
	958	"""Writes the whole site into `out`. Returns a summary string."""
	959	posts = [p for p in scan_posts(cfg) if p.published]
	960	for p in posts:
	961	if not p.date:
	962	raise BlogError(f"{p.rel}: published, but no date (name the file YYYY-MM-DD-name.md or set date)")
	963	posts.sort(key=lambda p: (p.date, p.rel), reverse=True)
	964	if posts:
	965	first, last = posts[-1].date.year, posts[0].date.year
	966	cfg["_years"] = str(first) if first == last else f"{first}–{last}"
	967	comments = load_comments(cfg)
	968	prepare_out(out, cfg)
	969	written = set()
	970
	971	def emit(path, content):
	972	folder = urllib.parse.unquote(path).strip("/") # the web server decodes %xx before it looks for the file
	973	if folder + path[-1:] in written:
	974	raise BlogError(f"two pages want the URL {path}")
	975	written.add(folder + path[-1:])
	976	write(safe_join(out, folder, "index.html") if path.endswith("/") else safe_join(out, folder), content)
	977
	978	items, tags = [], {}
	979	for p in posts:
	980	excerpt_md, full = split_more(extract_body(p.text, cfg["body_section"], p.body_until, p.rel))
	981	body = render_markdown(full, cfg)
	982	text = plain_text(body)
	983	minutes = max(1, math.ceil(len(text.split()) / cfg["reading_wpm"]))
	984	it = {"post": p, "html": body, "minutes": minutes, "excerpt": make_excerpt(excerpt_md, body, cfg)}
	985	items.append(it)
	986	for t in p.tags:
	987	tags.setdefault(tag_slug(cfg, t), []).append(it)
	988	page_body = (f'<article class="post"><h1>{esc(p.title)}</h1>{post_meta_line(cfg, p, minutes)}'
`e9fdc5db` 2026-11-01 Pavel Safronov	989	f'<div class="post-body">{body}</div></article>{post_comments(cfg, comments, p)}')
`0e603cf7` 2026-10-31 Pavel Safronov	990	emit(p.path, blog_page(cfg, p.title, page_body, p.path))
	991
	992	years = sorted({it["post"].date.year for it in items}, reverse=True)
	993	index = ('<h1>Blog</h1>\n' + ('<p class="years">' + " ".join(f'<a href="#{y}">{y}</a>' for y in years) + "</p>\n"
	994	+ listing(cfg, items) if items else "<p>No posts yet.</p>"))
	995	emit("/blog/", blog_page(cfg, "Blog", index, "/blog/"))
	996	for slug, its in sorted(tags.items()):
	997	name = tag_name(cfg, slug, next((t for t in its[0]["post"].tags if tag_slug(cfg, t) == slug), slug))
	998	emit(f"/tag/{slug}/", blog_page(cfg, name, f'<h1>{esc(name)}</h1>\n{listing(cfg, its, headings=False)}', f"/tag/{slug}/"))
	999
	1000	home_done = False
	1001	pdir = pages_dir(cfg)
	1002	for rel in source_files(cfg, patterns=[pdir + "/*.md"]) if pdir else []:
	1003	meta, text = parse_front_matter(read_text(os.path.join(cfg["_source"], rel)))
	1004	bad = sorted(set(meta) - set(PAGE_FIELDS))
	1005	if bad:
	1006	raise BlogError(f"{rel}: unknown front matter key(s): " + ", ".join(bad))
	1007	if meta.get("published") is not True: # like posts: a page is published only when it says so
	1008	continue
	1009	stem = os.path.splitext(os.path.basename(rel))[0]
	1010	slug = check_slug(str(meta.get("slug") or stem), rel)
	1011	title = str(meta.get("title") or title_of(text) or stem)
	1012	page_html = render_markdown(extract_body(text, None, name=rel), cfg)
	1013	if slug == cfg["home"]:
	1014	emit("/", blog_page(cfg, title, f'<article class="post"><h1>{esc(title)}</h1>{page_html}</article>', "/"))
	1015	home_done = True
	1016	else:
	1017	path = f"/{slug}/"
	1018	emit(path, blog_page(cfg, title, f'<article class="post"><h1>{esc(title)}</h1>{page_html}</article>', path))
	1019	if cfg["home"] and not home_done:
	1020	raise BlogError(f'home page "{cfg["home"]}" not found in pages_dir')
	1021	if not home_done:
	1022	emit("/", blog_page(cfg, cfg["title"], index, "/"))
	1023	emit("/404.html", blog_page(cfg, "Not found", '<h1>Not found</h1><p>There is nothing at this address. '
	1024	'<a href="/blog/">All posts</a></p>', "/404.html"))
	1025
	1026	entries = [{"title": it["post"].title, "url": cfg["base_url"] + it["post"].path, "updated": localize(cfg, it["post"].date),
	1027	"published": localize(cfg, it["post"].date), "content": absolutize(it["html"], cfg["base_url"])}
	1028	for it in items[:cfg["feed"]["entries"]]]
	1029	feed_path = cfg["feed"]["path"].strip("/")
	1030	write(safe_join(out, urllib.parse.unquote(feed_path), "index.xml"),
	1031	atom.atom_feed(cfg["title"], cfg["base_url"] + "/",
	1032	localize(cfg, posts[0].date) if posts else datetime.datetime.now(datetime.timezone.utc),
	1033	entries, copyright_text(cfg), author=cfg["author"] or None,
	1034	feed_url=f'{cfg["base_url"]}/{feed_path}/', subtitle=cfg["tagline"] or None))
	1035
	1036	for name in ("_shared.css", "favicon.ico"):
	1037	shutil.copyfile(os.path.join(SITE_DIR, name), os.path.join(out, name))
	1038	for name in ("blog.js", "robots.txt"):
	1039	shutil.copyfile(os.path.join(BLOG_DIR, name), os.path.join(out, name))
	1040	with open(os.path.join(SITE_DIR, "_shared.css")) as f:
	1041	shared = f.read()
	1042	with open(os.path.join(BLOG_DIR, "blog.css")) as f:
	1043	write(os.path.join(out, "blog.css"), f.read() + theme_overrides(shared))
	1044	media = copy_media(cfg, out)
	1045	if htaccess:
	1046	write(os.path.join(out, ".htaccess"), htaccess_text(cfg))
	1047	return f"{len(posts)} posts, {len(tags)} tags, {len(written)} pages, {media} media files -> {out}"
	1048
	1049
`9bb22e48` 2026-11-03 Pavel Safronov	1050	# ---------------------------------------------------------------- deploy
	1051
	1052	# --delete removes whatever else is in the target folder, except these.
	1053	RSYNC_OPTS = ["-rlptz", "--delete", "--exclude", ".well-known", "--exclude", ".dh-diag", "--exclude", ".fw-blog-out",
	1054	"--exclude", "/_comments/"]
	1055
	1056
	1057	def watch_name(config_path):
	1058	"""<folder name of the config, lowercased>-<first 8 hex of sha256 of its real path>: names the state folder
	1059	and the launchd label, so two configs in same-named folders never share state."""
	1060	real = os.path.realpath(config_path)
	1061	return f"{os.path.basename(os.path.dirname(real)).lower()}-{hashlib.sha256(real.encode()).hexdigest()[:8]}"
	1062
	1063
	1064	def state_dir(config_path):
	1065	"""Where `fw blog watch` and `fw blog deploy` keep their state for one blog: ~/.fw-blog-watch/<watch_name>/
	1066	(the environment variable FW_BLOG_WATCH_DIR replaces the whole path, for tests)."""
	1067	return os.environ.get("FW_BLOG_WATCH_DIR") or os.path.join(os.path.expanduser("~"), ".fw-blog-watch",
	1068	watch_name(config_path))
	1069
	1070
	1071	def acquire_lock(directory, now=time.time):
	1072	"""The exclusive lock of one blog's state folder (a tick of the watcher, or a real manual deploy), as an open
	1073	file to close when done; None when someone holds it. The holder's start time is written into the file."""
	1074	os.makedirs(directory, exist_ok=True)
	1075	f = open(os.path.join(directory, "lock"), "a+")
	1076	try:
	1077	fcntl.flock(f, fcntl.LOCK_EX \| fcntl.LOCK_NB)
	1078	except OSError:
	1079	f.close()
	1080	return None
	1081	f.seek(0)
	1082	f.truncate(0)
	1083	f.write(repr(float(now())))
	1084	f.flush()
	1085	return f
	1086
	1087
	1088	def lock_since(directory):
	1089	"""The start time (epoch seconds) the current lock holder wrote, or None."""
	1090	try:
	1091	with open(os.path.join(directory, "lock")) as f:
	1092	return float(f.read().strip())
	1093	except (OSError, ValueError):
	1094	return None
	1095
	1096
	1097	def generator_hash():
	1098	"""sha256 of everything that shapes the output besides the posts: this generator, the feed code, gen.py (nav and
	1099	footer), the shared CSS/icon and the blog's own CSS/JS/robots/htaccess. `fw blog watch` deploys on its own only
	1100	while this equals the value stored in deployed.json."""
	1101	files = [os.path.join(HERE, n) for n in ("blog.py", "atom.py", "gen.py")]
	1102	files += [os.path.join(SITE_DIR, n) for n in ("_shared.css", "favicon.ico")]
	1103	files += sorted(glob.glob(os.path.join(BLOG_DIR, "*")))
	1104	h = hashlib.sha256()
	1105	for path in files:
	1106	if os.path.isfile(path):
	1107	h.update(os.path.relpath(path, HERE).encode() + b"\0")
	1108	with open(path, "rb") as f:
	1109	h.update(hashlib.sha256(f.read()).digest())
	1110	return h.hexdigest()
	1111
	1112
	1113	def manifest(out):
	1114	"""{relative path: sha256 of the content} of a built folder (the build's marker file is not part of the site)."""
	1115	files = {}
	1116	for dirpath, _, names in os.walk(out):
	1117	for name in names:
	1118	path = os.path.join(dirpath, name)
	1119	rel = os.path.relpath(path, out).replace(os.sep, "/")
	1120	if rel != MARKER:
	1121	with open(path, "rb") as f:
	1122	files[rel] = hashlib.sha256(f.read()).hexdigest()
	1123	return dict(sorted(files.items()))
	1124
	1125
	1126	def config_hash(cfg):
	1127	"""sha256 of the loaded config (keys starting with _ are internal and left out)."""
	1128	public = {k: v for k, v in cfg.items() if not k.startswith("_")}
	1129	return hashlib.sha256(json.dumps(public, sort_keys=True, default=str).encode()).hexdigest()
	1130
	1131
	1132	def write_deployed(directory, files, when=None, generator=None, cfg=None):
	1133	"""deployed.json: what the last real upload sent, the generator hash it was built with and (with `cfg`) the hash of
	1134	the config and the deploy target. `fw blog watch` compares the next build with it."""
	1135	os.makedirs(directory, exist_ok=True)
	1136	when = when or datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
	1137	tmp = os.path.join(directory, "deployed.json.tmp")
	1138	with open(tmp, "w", encoding="utf-8") as f:
	1139	json.dump({"files": files, "time": when, "generator": generator or GENERATOR_HASH,
	1140	**({"config": config_hash(cfg), "target": deploy_target(cfg)} if cfg else {})}, f, indent=1)
	1141	os.replace(tmp, os.path.join(directory, "deployed.json"))
	1142
	1143
	1144	def _read_bytes(path):
	1145	try:
	1146	with open(path, "rb") as f:
	1147	return f.read()
	1148	except OSError:
	1149	return None
	1150
	1151
	1152	def _forget_fingerprint(directory):
	1153	"""After a manual upload the watcher's idea of "handled" may be stale: drop it so the next tick re-evaluates."""
	1154	path = os.path.join(directory, "state.json")
	1155	try:
	1156	with open(path, encoding="utf-8") as f:
	1157	state = json.load(f)
	1158	if not isinstance(state, dict) or not ({"fingerprint", "handled"} & set(state)):
	1159	return
	1160	state.pop("fingerprint", None)
	1161	state.pop("handled", None)
	1162	with open(path + ".tmp", "w", encoding="utf-8") as f:
	1163	json.dump(state, f, indent=1)
	1164	os.replace(path + ".tmp", path)
	1165	except (OSError, ValueError):
	1166	pass
	1167
	1168
	1169	def deploy_site(cfg, out, dry_run=False, confirm=None, run=subprocess.run, prebuilt=False, locked=False, timeout=None):
	1170	"""Uploads ONLY a freshly built, audited output folder, never the source folder. Builds into `out` (unless
	1171	`prebuilt`: the caller just built it), audits it, then (not for a dry run) asks `confirm(target)` if given and
	1172	runs rsync (no time limit unless `timeout`). A real upload takes the state folder's lock around the upload only
	1173	(unless the caller `locked` it already: the watcher), and writes deployed.json. True when something was
	1174	uploaded; BlogError on any failure."""
	1175	target = deploy_target(cfg)
	1176	directory = state_dir(cfg["_path"])
	1177	baseline = _read_bytes(os.path.join(directory, "deployed.json")) # to notice a deploy by the watcher meanwhile
	1178	if not prebuilt:
	1179	print(build(cfg, out))
	1180	count, size = audit_output(out)
	1181	print(f"audit ok: {count} files, {size / 1024:.0f} KB")
	1182	lock = None
	1183	if dry_run:
	1184	cmd = ["rsync", "-n", "-v", *RSYNC_OPTS, out + "/", target + "/"]
	1185	else:
	1186	if confirm and not confirm(target):
	1187	raise BlogError("not uploaded")
	1188	if not locked:
	1189	lock = acquire_lock(directory)
	1190	if lock is None:
	1191	raise BlogError("a watch tick is running, try again in a minute")
	1192	if _read_bytes(os.path.join(directory, "deployed.json")) != baseline:
	1193	lock.close()
	1194	raise BlogError("the watcher deployed while you were answering; run fw blog deploy again")
	1195	cmd = ["rsync", *RSYNC_OPTS, out + "/", target + "/"]
	1196	try:
	1197	try:
	1198	code = run(cmd, timeout=timeout).returncode
	1199	except subprocess.TimeoutExpired:
	1200	raise BlogError(f"rsync timed out after {timeout} s")
	1201	if code:
	1202	raise BlogError(f"rsync failed (exit {code})")
	1203	if not dry_run:
	1204	write_deployed(directory, manifest(out), cfg=cfg)
	1205	if not locked:
	1206	_forget_fingerprint(directory)
	1207	return not dry_run
	1208	finally:
	1209	if lock:
	1210	lock.close()
	1211
	1212
	1213	def ask_to_upload(target):
	1214	"""The prompt of `fw blog deploy` (needs a terminal)."""
	1215	if not sys.stdin.isatty():
	1216	raise BlogError(f"not a terminal; pass --yes to upload to {target}")
	1217	try:
	1218	return input(f"Upload the output above to {target}, deleting other files there? [y/N] ").strip() in ("y", "Y")
	1219	except EOFError:
	1220	return False
	1221	except KeyboardInterrupt:
	1222	print(file=sys.stderr)
	1223	raise SystemExit(1)
	1224
	1225
	1226	GENERATOR_HASH = generator_hash() # computed once, when the module loads: the code that is actually running
	1227
	1228
`0e603cf7` 2026-10-31 Pavel Safronov	1229	# ---------------------------------------------------------------- command line
	1230
	1231	def main(argv):
	1232	ap = argparse.ArgumentParser(prog="blog.py", description=__doc__.split("\n")[0])
	1233	sub = ap.add_subparsers(dest="cmd", required=True)
	1234	sub.add_parser("list").add_argument("config")
	1235	b = sub.add_parser("build")
	1236	b.add_argument("config")
	1237	b.add_argument("--out", required=True)
	1238	b.add_argument("--no-htaccess", action="store_true", help="skip .htaccess (local preview)")
	1239	sub.add_parser("audit").add_argument("out")
	1240	sub.add_parser("deploy-target").add_argument("config")
`9bb22e48` 2026-11-03 Pavel Safronov	1241	d = sub.add_parser("deploy")
	1242	d.add_argument("config")
	1243	d.add_argument("--out", required=True)
	1244	d.add_argument("--dry-run", action="store_true")
	1245	d.add_argument("--yes", action="store_true", help="upload without asking")
`0e603cf7` 2026-10-31 Pavel Safronov	1246	args = ap.parse_args(argv[1:])
	1247	try:
	1248	if args.cmd == "audit":
	1249	count, size = audit_output(args.out)
	1250	print(f"audit ok: {count} files, {size / 1024:.0f} KB")
	1251	return
	1252	cfg = load_config(args.config)
	1253	if args.cmd == "list":
	1254	print("\n".join(list_posts(cfg)))
	1255	elif args.cmd == "build":
	1256	print(build(cfg, os.path.abspath(args.out), htaccess=not args.no_htaccess))
`9bb22e48` 2026-11-03 Pavel Safronov	1257	elif args.cmd == "deploy":
	1258	deploy_site(cfg, os.path.abspath(args.out), dry_run=args.dry_run,
	1259	confirm=None if args.yes else ask_to_upload)
`0e603cf7` 2026-10-31 Pavel Safronov	1260	else:
	1261	print(deploy_target(cfg))
	1262	except BlogError as e:
	1263	raise SystemExit(f"blog.py: {e}")
	1264
	1265
	1266	if __name__ == "__main__":
	1267	main(sys.argv)