fuzzyworld

fuzzyworld-tools / files / server / blog.py

blog.py

61.0 KB · history · blame

1#!/usr/bin/env python3
2"""Static blog generator (writing.fuzzyworld.net). Runs on the laptop, from Markdown files.
3
4 blog.py list <config> every source file: path -> URL, title, date, tags (and whether it is published)
5 blog.py build <config> --out DIR write the site into DIR (--no-htaccess for a local preview)
6 blog.py deploy-target <config> print host:dir from the config (used by `fw blog deploy`)
7 blog.py deploy <config> --out DIR build, audit, ask, rsync (`fw blog deploy`); --dry-run, --yes
8 blog.py audit DIR refuse (exit 1) if DIR holds anything that must not be uploaded
9
10`fw blog ...` is the usual way in (it also installs Markdown and Pygments into ~/fw/pylib). The config is
11a JSON file next to the posts; its keys are listed in CONFIG_KEYS. `source` and `comments` are relative to
12the config file (so is the importer's `comments.json`); `pages_dir` and `media` are relative to `source`.
13
14`timezone` (an IANA name, default UTC) says where post dates were written: feed timestamps use it.
15
16A post is a Markdown file `YYYY-MM-DD-name.md`. Optional front matter (a `---` block of `key: value` lines)
17and the config's `posts` table can set published, date, slug, title, tags, body_until and wp_id; front matter
18wins over the config, which wins over what the file name and its first `# ` heading say. A post is published
19only when `published` is exactly true. Text under `## Spelling` and `## Removed writing`, and every HTML comment
20except <!--more-->, never reaches the output. A heading that looks like a private one but is not written as
21`## Spelling` (no space, an underlined heading, an unclosed code fence) stops the build instead of leaking.
22
23A page (a file in `pages_dir`) is published only with `published: true` in its front matter; it may set slug and title.
24
25Only the standard library is required, plus Python-Markdown (and Pygments for colored code) from ~/fw/pylib.
26"""
27
28import argparse
29import calendar
30import dataclasses
31import datetime
32import fcntl
33import glob
34import html
35import json
36import math
37import os
38import re
39import hashlib
40import shutil
41import subprocess
42import sys
43import time
44import urllib.parse
45import zoneinfo
46
47HERE = os.path.dirname(os.path.abspath(__file__))
48sys.path.insert(0, HERE)
49import atom # noqa: E402
50import gen # noqa: E402 (also puts ~/fw/pylib, or $FW_PYLIB, on sys.path)
51
52markdown = gen.markdown
53esc, write = gen.esc, gen.write
54
55SITE_DIR = os.path.join(HERE, "site") # _shared.css, favicon.ico: shared with code.fuzzyworld.net
56BLOG_DIR = os.path.join(HERE, "site-blog") # blog.css, blog.js, htaccess, robots.txt
57MARKER = ".fw-blog-out" # in an output folder this tool made: the only kind it empties
58# Media are copied by an allowlist of extensions: ~/Blog holds much that must never be published (Markdown,
59# JSON, notes, dotfiles). The audit applies the same rule again to everything in the build before an upload.
60MEDIA_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".pdf", ".epub", ".kpf", ".mp3", ".mp4", ".zip"}
61GENERATED_EXTENSIONS = {".html", ".xml", ".css", ".js", ".ico"} # what the generator itself writes
62GENERATED_NAMES = {"robots.txt", ".htaccess"} # the only other files allowed in an upload
63PRIVATE_SECTIONS = {"spelling", "removed writing"} # never published, wherever they are
64SLUG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._%-]*$") # every slug that becomes part of a path or a URL
65MORE = "<!--more-->"
66DEFAULT_URL = "/{yyyy}/{mm}/{dd}/{slug}/"
67POST_FIELDS = ("published", "date", "slug", "title", "tags", "body_until", "wp_id")
68PAGE_FIELDS = ("published", "slug", "title")
69CONFIG_KEYS = {"title", "tagline", "base_url", "author", "copyright", "source", "include", "body_section",
70 "line_breaks", "url", "home", "pages_dir", "nav", "tags", "feed", "reading_wpm", "media",
71 "rewrite_hosts", "redirects", "wp_ids", "comments", "deploy", "posts", "timezone", "site",
72 "comments_site", "comments_mailbox", "comments_form", "mirror", "autodeploy"}
73REQUIRED_KEYS = ("title", "base_url", "source")
74DEFAULTS = {"tagline": "", "author": "", "copyright": "© {years} {author}. All rights reserved.",
75 "include": ["*.md"], "body_section": "Writing", "line_breaks": "markdown", "url": DEFAULT_URL,
76 "home": None, "pages_dir": None, "nav": [], "tags": {}, "feed": {"path": "/feed/", "entries": 20},
77 "reading_wpm": 200, "media": None, "rewrite_hosts": [], "redirects": {}, "wp_ids": {},
78 "comments": None, "deploy": None, "posts": {}, "timezone": "UTC", "site": None,
79 "comments_site": None, "comments_mailbox": None, "comments_form": None,
80 "mirror": None, "autodeploy": None}
81
82
83class BlogError(Exception):
84 """A problem with the config or a source file, reported without a traceback."""
85
86
87# ---------------------------------------------------------------- config
88
89def load_config(path):
90 """The config as a dict with defaults filled in, plus `_dir` (the config's folder) and `_source` (absolute)."""
91 path = os.path.abspath(path)
92 try:
93 with open(path, encoding="utf-8") as f:
94 raw = json.load(f)
95 except OSError as e:
96 raise BlogError(f"cannot read config {path}: {e.strerror}")
97 except ValueError as e:
98 raise BlogError(f"config {path} is not valid JSON: {e}")
99 if not isinstance(raw, dict):
100 raise BlogError("config must be a JSON object")
101 unknown = sorted(set(raw) - CONFIG_KEYS)
102 if unknown:
103 raise BlogError("unknown config key(s): " + ", ".join(unknown))
104 missing = [k for k in REQUIRED_KEYS if not raw.get(k)]
105 if missing:
106 raise BlogError("config needs: " + ", ".join(missing))
107 cfg = {k: (dict(v) if isinstance(v, dict) else list(v) if isinstance(v, list) else v) for k, v in DEFAULTS.items()}
108 cfg.update(raw)
109 cfg["feed"] = {**DEFAULTS["feed"], **(cfg["feed"] or {})}
110 cfg["base_url"] = cfg["base_url"].rstrip("/")
111 if cfg["line_breaks"] not in ("br", "markdown"):
112 raise BlogError('line_breaks must be "br" or "markdown"')
113 try:
114 zoneinfo.ZoneInfo(str(cfg["timezone"]))
115 except (zoneinfo.ZoneInfoNotFoundError, ValueError, OSError):
116 raise BlogError(f'timezone {cfg["timezone"]!r} is not an IANA time zone name (e.g. "America/Los_Angeles")')
117 if not isinstance(cfg["posts"], dict) or not all(isinstance(v, dict) for v in cfg["posts"].values()):
118 raise BlogError("posts must map a file path to an object")
119 for rel, entry in cfg["posts"].items():
120 bad = sorted(set(entry) - set(POST_FIELDS))
121 if bad:
122 raise BlogError(f"posts[{rel}]: unknown key(s): " + ", ".join(bad))
123 if not (isinstance(cfg["nav"], list) and all(isinstance(n, list) and len(n) == 2 for n in cfg["nav"])):
124 raise BlogError("nav must be a list of [label, path] pairs")
125 site = cfg["site"]
126 if site is not None:
127 if isinstance(site, dict) and "footer" in site:
128 raise BlogError('site: "footer" is set by "nav": "gen", not in the config')
129 if isinstance(site, dict) and site.get("nav") == "gen":
130 site = cfg["site"] = {**site, "nav": gen.site_nav(), "footer": True} # "gen": the fuzzyworld nav and footer, built in gen.py
131 if not (isinstance(site, dict) and set(site) <= {"name", "url", "nav", "current", "footer"} and {"name", "url", "nav"} <= set(site)
132 and isinstance(site["name"], str) and isinstance(site["url"], str) and isinstance(site["nav"], list)
133 and isinstance(site.get("current", ""), str)
134 and all(isinstance(n, list) and len(n) == 2 and all(isinstance(x, str) for x in n) for n in site["nav"])):
135 raise BlogError('site must be {"name", "url", "nav": [[label, url], ...], "current": label (optional)}')
136 if not (cfg["url"].startswith("/") and cfg["url"].endswith("/") and "{slug}" in cfg["url"]):
137 raise BlogError("url must start and end with / and contain {slug}")
138 for old, new in cfg["redirects"].items():
139 if (not isinstance(new, str) or htaccess_unsafe(old) or htaccess_unsafe(new) or old.strip("/") == ""):
140 raise BlogError(f"redirects: bad entry {old!r} -> {new!r} (no spaces or control characters, even percent-encoded)")
141 for key, value in cfg["wp_ids"].items():
142 if not isinstance(value, str) or htaccess_unsafe(key) or htaccess_unsafe(value):
143 raise BlogError(f"wp_ids: bad entry {key!r} -> {value!r} (no spaces or control characters, even percent-encoded)")
144 if cfg["home"]:
145 if not isinstance(cfg["home"], str):
146 raise BlogError("home must be a page slug")
147 check_slug(cfg["home"], "home")
148 feed_path = cfg["feed"]["path"]
149 segments = [x for x in str(feed_path).split("/") if x]
150 if not (isinstance(feed_path, str) and feed_path.startswith("/") and segments) or htaccess_unsafe(feed_path):
151 raise BlogError(f"feed.path {feed_path!r} must be a path like /feed/")
152 for x in segments:
153 check_slug(x, "feed.path")
154 if not isinstance(cfg["tags"], dict):
155 raise BlogError("tags must map a slug to a name")
156 for slug in cfg["tags"]:
157 check_slug(slug, "tags")
158 check_comments_config(cfg)
159 check_watch_config(cfg)
160 cfg["_path"] = path
161 cfg["_dir"] = os.path.dirname(path)
162 cfg["_source"] = os.path.normpath(os.path.join(cfg["_dir"], cfg["source"]))
163 return cfg
164
165
166def check_comments_config(cfg):
167 """Reader comments by email (and, off by default, a web form). Off unless comments_site is set."""
168 site, mailbox, form = cfg["comments_site"], cfg["comments_mailbox"], cfg["comments_form"]
169 if site is None:
170 if mailbox is not None or form is not None:
171 raise BlogError("comments_mailbox and comments_form need comments_site")
172 return
173 if not (isinstance(site, str) and re.fullmatch(r"[A-Za-z0-9_-]+", site)):
174 raise BlogError('comments_site must be a short name such as "writing" (letters, digits, - and _)')
175 if not (isinstance(mailbox, str) and re.fullmatch(r"[^@\s<>\"',;:()\[\]\\]+@[^@\s<>\"',;:()\[\]\\]+\.[^@\s<>\"',;:()\[\]\\]+", mailbox)):
176 raise BlogError("comments_site needs comments_mailbox: the email address that receives comments")
177 if form is not None:
178 action = form.get("action") if isinstance(form, dict) else None
179 parts = urllib.parse.urlsplit(action) if isinstance(action, str) else None
180 if not (isinstance(form, dict) and set(form) == {"action"} and parts and parts.scheme == "https" and parts.netloc
181 and not re.search(r"[\s\x00-\x1f\x7f\"'<>]", action)):
182 raise BlogError('comments_form must be null or {"action": "https://..."} (an https URL without spaces or quotes)')
183
184
185_BRANCH = re.compile(r"^[A-Za-z0-9._/][A-Za-z0-9._/-]*$")
186_REMOTE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
187
188
189def check_watch_config(cfg):
190 """`mirror` and `autodeploy` (both off unless set) drive `fw blog watch` (see blog_watch.py)."""
191 auto, mirror = cfg["autodeploy"], cfg["mirror"]
192 if auto is not None:
193 quiet = auto.get("quiet_seconds", 300) if isinstance(auto, dict) else None
194 if not (isinstance(auto, dict) and set(auto) <= {"quiet_seconds"}
195 and isinstance(quiet, int) and not isinstance(quiet, bool) and quiet >= 60):
196 raise BlogError('autodeploy must be {"quiet_seconds": whole number, at least 60 (optional, default 300)}')
197 if mirror is None:
198 return
199 if not (isinstance(mirror, dict) and set(mirror) <= {"repo", "branch", "remote", "exclude"}):
200 raise BlogError('mirror must be {"repo", "branch" (optional), "remote" (optional), "exclude" (optional)}')
201 repo = mirror.get("repo")
202 if not (isinstance(repo, str) and repo.strip()):
203 raise BlogError("mirror.repo must be a folder path (non-empty text)")
204 branch = mirror.get("branch", "dropbox-mirror")
205 if (not isinstance(branch, str) or not _BRANCH.match(branch) or ".." in branch or "//" in branch
206 or branch.endswith(("/", ".", ".lock")) or branch in ("main", "master", "HEAD")):
207 raise BlogError("mirror.branch must be a plain git branch name (letters, digits . _ / -; not starting with -), "
208 "and not main, master or HEAD")
209 remote = mirror.get("remote")
210 if remote is not None and not (isinstance(remote, str) and _REMOTE.match(remote)):
211 raise BlogError("mirror.remote must be a plain remote name (letters, digits . _ -)")
212 exclude = mirror.get("exclude", [])
213 if not (isinstance(exclude, list) and all(isinstance(x, str) and x.strip() and ".." not in x
214 and not x.startswith("/..") and "\n" not in x
215 and x[0] not in "!+-:" for x in exclude)):
216 raise BlogError('mirror.exclude must be a list of rsync patterns (text; no "..", not starting with ! + - :)')
217
218
219def check_slug(slug, what):
220 """The slug, or BlogError: a slug becomes a folder name and part of a URL (and of .htaccess), so it is ASCII
221 letters, digits and . _ % - only, and it must not decode to `.`, `..` or something with a slash."""
222 decoded = urllib.parse.unquote(str(slug))
223 if (not SLUG_RE.match(str(slug)) or decoded in (".", "..") or re.search(r"[/\\\x00-\x1f\x7f]", decoded)):
224 raise BlogError(f"{what}: bad slug {slug!r} (use letters, digits and . _ % - only)")
225 return slug
226
227
228def htaccess_unsafe(value):
229 """True when a value that goes into .htaccess has a space or control character, written out or percent-encoded."""
230 text = str(value)
231 return any(c.isspace() or ord(c) < 32 or ord(c) == 127 for c in text + urllib.parse.unquote(text))
232
233
234_DEPLOY_HOST = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._@-]*$")
235_DEPLOY_DIR = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*(/[A-Za-z0-9._-]+)*$")
236
237
238def deploy_target(cfg):
239 """host:dir for rsync. The dir is relative to the login folder and must look like a site folder (e.g.
240 writing.fuzzyworld.net): rsync --delete runs against it, so `~`, `.`, `/`, `..` and the like are refused."""
241 deploy = cfg["deploy"] or {}
242 host, folder = deploy.get("host"), deploy.get("dir")
243 if not (isinstance(host, str) and isinstance(folder, str) and host and folder):
244 raise BlogError("config needs deploy: {host, dir}")
245 folder = folder.rstrip("/")
246 if (not _DEPLOY_HOST.match(host) or not _DEPLOY_DIR.match(folder)
247 or any(part in (".", "..") for part in folder.split("/")) or ".." in folder):
248 raise BlogError(f"deploy.dir {deploy.get('dir')!r} (or host) is not safe: use a site folder such as "
249 "writing.fuzzyworld.net, relative to the login folder")
250 return f"{host}:{folder}"
251
252
253# ---------------------------------------------------------------- source files
254
255_FM_LINE = re.compile(r"^([A-Za-z_][\w-]*):\s*(.*?)\s*$")
256
257
258def parse_front_matter(text):
259 """(meta dict, rest of the text). A leading `---` block counts only if it closes and every line is `key: value`;
260 otherwise the `---` was a horizontal rule and there is no front matter."""
261 lines = text.split("\n")
262 if not lines or lines[0].strip() != "---":
263 return {}, text
264 for end in range(1, len(lines)):
265 if lines[end].strip() == "---":
266 break
267 else:
268 return {}, text
269 meta = {}
270 for line in lines[1:end]:
271 if not line.strip():
272 continue
273 m = _FM_LINE.match(line)
274 if not m:
275 return {}, text
276 key, value = m.groups()
277 if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
278 value = value[1:-1]
279 meta[key] = {"true": True, "false": False}.get(value, value)
280 return meta, "\n".join(lines[end + 1:])
281
282
283_FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$")
284_H2 = re.compile(r"^ {0,3}##(?!#)\s+(.*?)\s*#*\s*$")
285_ATX = re.compile(r"^\s*#{1,6}(.*)$")
286_UNDERLINE = re.compile(r"^\s*(=+|-+)\s*$")
287_TEMPLATE_LINE = re.compile(r"^[ \t]*Created by template, on .*$", re.M)
288
289
290class _Fences:
291 """Fenced code blocks as CommonMark has them: a fence is 3+ backticks or tildes (a backtick fence has no backtick
292 in its info string, so an inline ```span``` is not one); it ends at a line of the same character, at least as
293 long, with nothing after it. feed(line) is True for a line that belongs to a block, the fence lines included."""
294
295 def __init__(self):
296 self.mark = None # (character, length) while inside a block
297
298 def feed(self, line):
299 m = _FENCE.match(line)
300 if self.mark is None:
301 if m and not (m.group(1)[0] == "`" and "`" in m.group(2)):
302 self.mark = (m.group(1)[0], len(m.group(1)))
303 return True
304 return False
305 if m and m.group(1)[0] == self.mark[0] and len(m.group(1)) >= self.mark[1] and not m.group(2).strip():
306 self.mark = None
307 return True
308
309
310def remove_comments(text, name=None):
311 """Cuts every HTML comment (private notes must not reach page source) except the <!--more--> marker.
312 A comment ends at its first `-->`, but a <!--more--> inside a comment belongs to the comment. A comment that
313 is never closed is cut to the end of the text (with a warning on stderr when `name` says which file)."""
314 out, i = [], 0
315 while True:
316 start = text.find("<!--", i)
317 if start == -1:
318 out.append(text[i:])
319 break
320 out.append(text[i:start])
321 if text.startswith(MORE, start):
322 out.append(MORE)
323 i = start + len(MORE)
324 continue
325 j = start + 4
326 while True:
327 end, more = text.find("-->", j), text.find(MORE, j)
328 if more != -1 and (end == -1 or more < end):
329 j = more + len(MORE)
330 continue
331 break
332 if end == -1:
333 if name:
334 print(f"blog.py: warning: {name}: a `<!--` is never closed; the text from it to the end is left out",
335 file=sys.stderr)
336 break
337 i = end + 3
338 return "".join(out)
339
340
341def parse_sections(text, name=None):
342 """(title, preamble lines, [(heading, lines)]) of comment-free text. The title is the first `# ` line
343 (not kept in any lines); each `## ` section's lines start with its heading line. Fenced code is never
344 mistaken for headings."""
345 title, pre, sections, fences = None, [], [], _Fences()
346 for line in remove_comments(text, name).replace("\r\n", "\n").split("\n"):
347 if not fences.feed(line):
348 if title is None and line.startswith("# "):
349 title = line[2:].strip()
350 continue
351 h = _H2.match(line)
352 if h:
353 sections.append((h.group(1), [line]))
354 continue
355 (sections[-1][1] if sections else pre).append(line)
356 return title, pre, sections
357
358
359def _private_name(text):
360 return text.strip(" \t#:*_`").casefold() in PRIVATE_SECTIONS
361
362
363def check_private_headings(lines, name=None):
364 """Fail closed: the body about to be published must hold nothing that looks like a private heading written some
365 other way than `## Spelling` (`##Spelling`, `Spelling` over a line of dashes, `### Spelling` ...) and no
366 code fence that is never closed (it would hide what follows from the section parser)."""
367 fences, prev = _Fences(), None
368 for n, line in enumerate(lines, 1):
369 if fences.feed(line):
370 prev = None
371 continue
372 m = _ATX.match(line)
373 if (m and _private_name(m.group(1))) or (not m and prev is not None and _UNDERLINE.match(line) and _private_name(prev)):
374 raise BlogError(f"{name or 'text'}: line {n} of the publishable text looks like a private heading "
375 f"({(prev if not m else line).strip()!r}); write it as `## Spelling` / `## Removed writing` "
376 "on a line of its own, or reword it")
377 prev = line
378 if fences.mark:
379 raise BlogError(f"{name or 'text'}: a code fence is never closed; fix it so private sections are not hidden by it")
380
381
382def title_of(text):
383 return parse_sections(text)[0]
384
385
386def _collect(sections, until):
387 """Lines of the sections up to the one headed `until` (exclusive), skipping private ones."""
388 out = []
389 for heading, lines in sections:
390 if until and heading.casefold() == until.casefold():
391 break
392 if heading.casefold() not in PRIVATE_SECTIONS:
393 out += lines
394 return out
395
396
397def extract_body(text, section="Writing", until=None, name=None):
398 """The publishable Markdown of a source file.
399
400 With a `## <section>` heading: that section, up to the next `## ` heading, or (when `until` names a heading)
401 through every section up to it. Without one (or with section=None): everything after the `# ` title line.
402 Private sections, HTML comments (except <!--more-->) and the "Created by template" line are always dropped.
403 BlogError (naming `name`) if what is left has a private-looking heading in another form, or an unclosed fence.
404 """
405 _, pre, sections = parse_sections(text, name)
406 target = next((i for i, (h, _) in enumerate(sections) if section and h.casefold() == section.casefold()), None)
407 if target is not None:
408 lines = sections[target][1][1:] + (_collect(sections[target + 1:], until) if until else [])
409 else:
410 lines = pre + _collect(sections, until)
411 body = _TEMPLATE_LINE.sub("", "\n".join(lines)).strip("\n").strip()
412 check_private_headings(body.split("\n"), name)
413 return body
414
415
416@dataclasses.dataclass
417class Post:
418 rel: str # path under `source`, with /
419 text: str # the file without its front matter
420 published: bool = False
421 date: datetime.datetime = None
422 slug: str = ""
423 title: str = ""
424 tags: list = dataclasses.field(default_factory=list)
425 body_until: str = None
426 wp_id: int = None
427 path: str = "" # URL path, e.g. /2018/03/02/slug/
428
429
430def parse_date(value):
431 """datetime from "YYYY-MM-DD" or "YYYY-MM-DD[ T]HH:MM[:SS]". A time zone is dropped: dates are WordPress local dates."""
432 if isinstance(value, datetime.datetime):
433 return value.replace(tzinfo=None)
434 try:
435 return datetime.datetime.fromisoformat(str(value).strip()).replace(tzinfo=None)
436 except ValueError:
437 raise BlogError(f"bad date {value!r} (use YYYY-MM-DD)")
438
439
440def localize(cfg, when):
441 """A post's naive date as a time-aware datetime in the config's `timezone` (for feed timestamps)."""
442 return when.replace(tzinfo=zoneinfo.ZoneInfo(cfg["timezone"])) if when.tzinfo is None else when
443
444
445def as_list(value):
446 if isinstance(value, str):
447 value = value.strip()
448 if value.startswith("[") and value.endswith("]"):
449 value = value[1:-1]
450 return [t for t in (t.strip().strip("\"'").strip() for t in value.split(",")) if t]
451 return [str(t).strip() for t in (value or []) if str(t).strip()]
452
453
454_NAME = re.compile(r"^(\d{4}-\d{2}-\d{2})-(.+)\.md$", re.I)
455
456
457def post_url(cfg, date, slug):
458 return (cfg["url"].replace("{yyyy}", f"{date.year:04d}").replace("{mm}", f"{date.month:02d}")
459 .replace("{dd}", f"{date.day:02d}").replace("{slug}", slug))
460
461
462def source_files(cfg, patterns=None, skip_dir=None):
463 """Relative paths of the source files matching the include patterns, sorted. Dropbox's "conflicted copy"
464 files and hidden folders are left out."""
465 src, found = cfg["_source"], set()
466 for pattern in patterns or cfg["include"]:
467 for p in glob.glob(os.path.join(glob.escape(src), pattern), recursive=True):
468 rel = os.path.relpath(p, src).replace(os.sep, "/")
469 parts = rel.split("/")
470 if (not os.path.isfile(p) or "conflicted copy" in rel.lower() or any(x.startswith(".") for x in parts)
471 or (skip_dir and rel.startswith(skip_dir + "/"))):
472 continue
473 found.add(rel)
474 return sorted(found)
475
476
477def read_text(path):
478 with open(path, encoding="utf-8-sig") as f:
479 return f.read().replace("\r\n", "\n")
480
481
482def pages_dir(cfg):
483 return cfg["pages_dir"].strip("/") if cfg["pages_dir"] else None
484
485
486def scan_posts(cfg):
487 """Every post file as a Post, oldest first. Precedence: front matter > config posts[<path>] > the file."""
488 posts = []
489 for rel in source_files(cfg, skip_dir=pages_dir(cfg)):
490 meta, text = parse_front_matter(read_text(os.path.join(cfg["_source"], rel)))
491 bad = sorted(set(meta) - set(POST_FIELDS))
492 if bad:
493 raise BlogError(f"{rel}: unknown front matter key(s): " + ", ".join(bad))
494 merged = {**cfg["posts"].get(rel, {}), **meta}
495 name = _NAME.match(os.path.basename(rel))
496 date = merged.get("date") or (name.group(1) if name else None)
497 stem = name.group(2) if name else os.path.splitext(os.path.basename(rel))[0]
498 post = Post(rel=rel, text=text,
499 published=merged.get("published") is True,
500 date=parse_date(date) if date else None,
501 slug=str(merged.get("slug") or stem),
502 title=str(merged.get("title") or title_of(text) or stem),
503 tags=as_list(merged.get("tags")),
504 body_until=merged.get("body_until") or None,
505 wp_id=int(merged["wp_id"]) if str(merged.get("wp_id", "")).isdigit() else None)
506 if post.published: # a draft's file name may hold anything; it is never written anywhere
507 check_slug(post.slug, rel)
508 if post.date:
509 post.path = post_url(cfg, post.date, post.slug)
510 posts.append(post)
511 posts.sort(key=lambda p: (p.date or datetime.datetime.min, p.rel))
512 return posts
513
514
515def slugify(text):
516 return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
517
518
519def tag_slug(cfg, tag):
520 """A post's tag may be written as a configured slug, a configured name or any other text."""
521 if tag in cfg["tags"]:
522 return tag
523 for slug, name in cfg["tags"].items():
524 if name == tag:
525 return slug
526 return slugify(tag) or "tag"
527
528
529def tag_name(cfg, slug, fallback=None):
530 return cfg["tags"].get(slug) or fallback or slug
531
532
533def list_posts(cfg):
534 lines = []
535 for p in scan_posts(cfg):
536 tags = ", ".join(p.tags)
537 lines.append(f'{"published" if p.published else "draft "} {p.rel} -> {p.path or "(no date)"} '
538 f'{p.title!r} {p.date.date() if p.date else "no date"}' + (f" tags: {tags}" if tags else ""))
539 return lines
540
541
542# ---------------------------------------------------------------- rendering
543
544def redirect_lookup(cfg):
545 """{path: target} from the config's redirects and wp_ids, tolerant of a missing or extra trailing slash."""
546 if "_lookup" not in cfg:
547 table = {}
548 for old, new in cfg["redirects"].items():
549 for key in (old, urllib.parse.unquote(old)):
550 table[key] = new
551 table[key.rstrip("/") + "/"] = new
552 table[key.rstrip("/")] = new
553 cfg["_lookup"] = table
554 return cfg["_lookup"]
555
556
557def rewrite_url(url, cfg):
558 """Absolute links to the old host become site-relative; internal links go through `redirects`
559 (and WordPress's /?p=ID short links through `wp_ids`)."""
560 for prefix in cfg["rewrite_hosts"]:
561 p = prefix.rstrip("/")
562 if url == p or url[len(p):len(p) + 1] in ("/", "?", "#") and url.startswith(p):
563 url = url[len(p):]
564 url = url if url.startswith("/") else "/" + url
565 break
566 if not url.startswith("/") or url.startswith("//"):
567 return url
568 m = re.match(r"([^?#]*)(.*)$", url)
569 path, tail = m.group(1), m.group(2)
570 if path == "/":
571 q = urllib.parse.parse_qs(tail.lstrip("?").split("#")[0])
572 for key in ("p", "page_id"):
573 target = cfg["wp_ids"].get((q.get(key) or [None])[0])
574 if target:
575 return target
576 target = redirect_lookup(cfg).get(path)
577 if target:
578 return target if re.search(r"[?#]", target) else target + tail
579 return url
580
581
582_ATTR = re.compile(r'(<(?:img|source)\b[^>]*?\bsrc|<a\b[^>]*?\bhref)=(["\'])(.*?)\2', re.I | re.S)
583
584
585def map_links(html_text, fn):
586 def fix(m):
587 return f"{m.group(1)}={m.group(2)}{esc(fn(html.unescape(m.group(3))), quote=True)}{m.group(2)}"
588 return _ATTR.sub(fix, html_text)
589
590
591_CODE = re.compile(r'<pre><code(?: class="language-([\w+#.-]+)")?>(.*?)</code></pre>', re.S)
592
593
594def highlight_blocks(html_text):
595 """Fenced code blocks with a language get Pygments token spans (the classes gen.py uses)."""
596 def fix(m):
597 lang = m.group(1)
598 if not lang or not gen.pygments_lexers:
599 return m.group(0)
600 try:
601 lexer = gen.pygments_lexers.get_lexer_by_name(lang, stripnl=False, ensurenl=False)
602 parts = []
603 for ttype, value in lexer.get_tokens(html.unescape(m.group(2))):
604 cls = gen.token_class(ttype)
605 parts.append(f'<span class="{cls}">{esc(value)}</span>' if cls else esc(value))
606 except Exception: # unknown language or a lexer bug: leave the block plain
607 return m.group(0)
608 return f'<pre class="hl"><code class="language-{lang}">{"".join(parts)}</code></pre>'
609 return _CODE.sub(fix, html_text)
610
611
612def render_markdown(text, cfg):
613 if not markdown:
614 raise BlogError("Python-Markdown is not installed in " + gen.PYLIB + " (fw blog installs it)")
615 extensions = ["tables", "fenced_code", "sane_lists"] + (["nl2br"] if cfg["line_breaks"] == "br" else [])
616 out = markdown.markdown(text, extensions=extensions, output_format="html")
617 return map_links(highlight_blocks(out), lambda u: rewrite_url(u, cfg))
618
619
620def plain_text(html_text):
621 return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", html_text))).strip()
622
623
624def split_more(text):
625 """(excerpt Markdown or None, full Markdown) around the <!--more--> marker."""
626 if "<!--more-->" not in text:
627 return None, text
628 head, tail = text.split("<!--more-->", 1)
629 return head.strip(), head.rstrip() + "\n\n" + tail.replace("<!--more-->", "").lstrip()
630
631
632def blocks(html_text):
633 """The text blocks of rendered HTML as (kind, escaped-free text) pairs: kind is "p" or "h" (a chapter heading).
634 <br> is kept as a newline; other markup and images are dropped."""
635 out = []
636 for m in re.finditer(r"<(p|h[1-6])(?:\s[^>]*)?>(.*?)</\1>", html_text, re.S):
637 marked = re.sub(r"<br\s*/?>", "\x00", m.group(2))
638 text = html.unescape(re.sub(r"<[^>]+>", "", marked))
639 text = "\n".join(re.sub(r"[ \t]+", " ", line).strip() for line in text.split("\x00")).strip()
640 if text:
641 out.append(("p" if m.group(1) == "p" else "h", text))
642 return out
643
644
645def make_excerpt(excerpt_md, body_html, cfg, words=80):
646 """(HTML for the lists, whether there is more to read). What is before <!--more-->, else whole blocks from the
647 start until about `words` words (a first paragraph longer than that is cut at a word, even after headings). Chapter headings are
648 kept, but never left dangling at the end."""
649 cut = excerpt_md is not None
650 found = blocks(render_markdown(excerpt_md, cfg) if cut else body_html)
651 if cut:
652 kept, more = found, True
653 else:
654 kept, n, more = [], 0, False
655 for kind, text in found:
656 w = len(text.split())
657 if kind == "p" and w > words and not any(k == "p" for k, _ in kept):
658 text, more = " ".join(text.split(" ")[:words]) + "…", True # the first paragraph, headings or not
659 elif kept and n + w > words:
660 break
661 kept.append((kind, text))
662 n += w
663 while kept and kept[-1][0] == "h" and len(kept) < len(found):
664 kept.pop()
665 more = more or len(kept) < len(found)
666 html_out = "".join(("<p>" if k == "p" else '<h5 class="chapter">') + esc(t).replace("\n", "<br>\n") + ("</p>" if k == "p" else "</h5>")
667 for k, t in kept)
668 return html_out, bool(more)
669
670
671def absolutize(html_text, base_url):
672 """For the feed: site-relative links become absolute."""
673 return map_links(html_text, lambda u: base_url + u if u.startswith("/") and not u.startswith("//") else u)
674
675
676def comments_src(path):
677 """Where the server publishes a post's comments: /2024/05/01/slug/ -> /_comments/2024/05/01/slug.json"""
678 return "/_comments/" + path.strip("/") + ".json"
679
680
681def comments_widget(cfg, path):
682 """The container blog.js fills, the "Comment by email" link and (only with comments_form) the web form."""
683 site = cfg["comments_site"]
684 subject = urllib.parse.quote(f"Comment [{site}:{path}]", safe="")
685 mailto = f"mailto:{urllib.parse.quote(cfg['comments_mailbox'], safe='@')}?subject={subject}"
686 html_ = ('<div class="comments-live"></div>'
687 f'<p class="comment-by-email"><a href="{esc(mailto, quote=True)}">Comment by email</a>. '
688 'Comments are moderated, and your address is never shown.</p>')
689 form = cfg["comments_form"]
690 if form:
691 html_ += (f'<form class="comment-form" method="post" action="{esc(form["action"], quote=True)}">'
692 f'<input type="hidden" name="site" value="{esc(site, quote=True)}">'
693 f'<input type="hidden" name="path" value="{esc(path, quote=True)}">'
694 '<input type="hidden" name="elapsed" value="0">'
695 '<div class="hp" aria-hidden="true"><input type="text" name="website" tabindex="-1" autocomplete="off"></div>'
696 '<p><label>Name <input type="text" name="name" maxlength="60" required></label></p>'
697 '<p><label>Comment<br><textarea name="text" rows="5" maxlength="4000" required></textarea></label></p>'
698 '<p><button type="submit">Send comment</button></p></form>')
699 return html_
700
701
702def render_comments(comments, widget=None, src=None):
703 """Threaded comments: name, date and text only. Everything is escaped; newlines become <br>. Replies nest.
704 With `src` (comments_site is set) the section is always written: it carries data-src, and `widget` follows the list."""
705 ids = {c.get("id") for c in comments}
706 children = {}
707 for c in sorted(comments, key=lambda c: str(c.get("date", ""))):
708 parent = c.get("parent") or 0
709 children.setdefault(parent if parent in ids and parent != c.get("id") else 0, []).append(c)
710
711 def tree(parent):
712 items = []
713 for c in children.get(parent, []):
714 text = esc(str(c.get("text", "")).replace("\r\n", "\n")).replace("\n", "<br>\n")
715 nested = tree(c.get("id")) if c.get("id") in children else ""
716 items.append(f'<li class="comment" id="comment-{re.sub(r"[^A-Za-z0-9_-]", "", str(c.get("id", "")))}">'
717 f'<p class="meta"><strong>{esc(str(c.get("author", "")))}</strong> · '
718 f'{esc(str(c.get("date", ""))[:10])}</p><p>{text}</p>{nested}</li>')
719 return f'<ul class="comments">{"".join(items)}</ul>' if items else ""
720
721 if src:
722 return (f'<section class="comments-section" data-src="{esc(src, quote=True)}"><h2>Comments</h2>' + tree(0)
723 + (widget or "") + '<p class="meta">Comments © their authors.</p></section>')
724 if not comments:
725 return ""
726 return ('<section class="comments-section"><h2>Comments</h2>' + tree(0)
727 + '<p class="meta">Comments © their authors.</p></section>')
728
729
730def post_comments(cfg, comments, post):
731 if not cfg["comments_site"]:
732 return render_comments(comments.get(post.path, []))
733 return render_comments(comments.get(post.path, []), comments_widget(cfg, post.path), comments_src(post.path))
734
735
736def load_comments(cfg):
737 if not cfg["comments"]:
738 return {}
739 path = os.path.join(cfg["_dir"], cfg["comments"])
740 if not os.path.exists(path):
741 return {}
742 try:
743 with open(path, encoding="utf-8") as f:
744 data = json.load(f)
745 except ValueError as e:
746 raise BlogError(f"comments file {path} is not valid JSON: {e}")
747 return data if isinstance(data, dict) else {}
748
749
750def copyright_text(cfg):
751 return (cfg["copyright"].replace("{years}", cfg.get("_years", str(datetime.date.today().year)))
752 .replace("{author}", cfg["author"]))
753
754
755def theme_overrides(shared_css):
756 """CSS that lets a reader's choice (<html data-theme="light|dark">) beat the system setting. _shared.css holds the
757 light colors in :root and the dark ones in a prefers-color-scheme media query; both blocks are repeated here under
758 the attribute, taken from _shared.css itself so they cannot drift."""
759 light = re.search(r":root\s*\{([^}]*--color-bg-primary[^}]*)\}", shared_css)
760 dark = re.search(r"@media \(prefers-color-scheme: dark\)\s*\{\s*:root\s*\{([^}]*)\}", shared_css)
761 if not (light and dark):
762 raise BlogError("_shared.css: cannot find the light and dark color blocks")
763 def decls(block):
764 return "\n".join(" " + line.strip() for line in block.splitlines() if line.strip().startswith("--"))
765 return (f'\n:root[data-theme="light"] {{\n color-scheme: light;\n{decls(light.group(1))}\n}}\n'
766 f':root[data-theme="dark"] {{\n color-scheme: dark;\n{decls(dark.group(1))}\n}}\n')
767
768
769def blog_page(cfg, title, body, path):
770 """A full page. `path` is the page's own URL path (for the canonical link and the current nav entry)."""
771 links = "".join(f'<a href="{esc(p, quote=True)}"' + (' aria-current="page"' if p == path else "") + f'>{esc(label)}</a>'
772 for label, p in cfg["nav"])
773 tagline = f'<span class="tagline">{esc(cfg["tagline"])}</span>' if cfg["tagline"] else ""
774 site = cfg["site"]
775 if site:
776 # The same bar as on code.fuzzyworld.net and apps.fuzzyworld.net, then the blog's own bar under it.
777 site_links = "".join(f'<a href="{esc(u, quote=True)}"' + (' aria-current="page"' if label == site.get("current") else "")
778 + f'>{esc(label)}</a>' for label, u in site["nav"])
779 header = (f'<header class="site"><div class="bar"><a class="brand" href="{esc(site["url"], quote=True)}">{esc(site["name"])}</a>'
780 f'<nav class="site-nav" aria-label="Site">{site_links}</nav></div>\n'
781 f'<div class="bar sub"><a class="title" href="/">{esc(cfg["title"])}</a>{tagline}'
782 f'<nav class="site-nav" aria-label="This blog">{links}</nav></div></header>')
783 else:
784 header = (f'<header class="site"><div class="bar"><a class="brand" href="/">{esc(cfg["title"])}</a>{tagline}'
785 f'<nav class="site-nav" aria-label="Site">{links}</nav></div></header>')
786 feed = cfg["feed"]["path"]
787 legal = (f'<p>{esc(copyright_text(cfg))}</p>\n'
788 f'<p><a href="{esc(site["url"] if site else "https://fuzzyworld.net/", quote=True)}">{esc(site["name"] if site else "fuzzyworld.net")}</a>'
789 f' · <a href="{esc(feed, quote=True)}">Feed</a></p>')
790 if site and site.get("footer"): # the site-wide four-column footer, then the blog's own line under it
791 footer = f'<footer class="site"><div class="cols">{gen.site_footer("writing")}</div>\n<div class="cols legal">\n{legal}\n</div></footer>'
792 else:
793 footer = f'<footer class="site"><div class="cols">\n{legal}\n</div></footer>'
794 head_title = cfg["title"] if path == "/" else f'{title} · {cfg["title"]}'
795 return f"""<!doctype html>
796<html lang="en">
797<head>
798<meta charset="utf-8">
799<meta name="viewport" content="width=device-width, initial-scale=1">
800<title>{esc(head_title)}</title>
801<link rel="canonical" href="{esc(cfg["base_url"] + path, quote=True)}">
802<link rel="alternate" type="application/atom+xml" title="{esc(cfg["title"], quote=True)}" href="{esc(feed, quote=True)}">
803<link rel="icon" href="/favicon.ico">
804<link rel="stylesheet" href="/_shared.css">
805<link rel="stylesheet" href="/blog.css">
806<script src="/blog.js"></script>
807</head>
808<body>
809{header}
810<main>
811{body}
812</main>
813{footer}
814</body>
815</html>
816"""
817
818
819def post_meta_line(cfg, post, minutes):
820 tags = "".join(f' <a class="tag" href="/tag/{esc(s, quote=True)}/">{esc(tag_name(cfg, s, t))}</a>'
821 for s, t in ((tag_slug(cfg, t), t) for t in post.tags))
822 return (f'<p class="meta"><time datetime="{post.date.date()}">{post.date.date()}</time> · {minutes} min read'
823 + (" ·" + tags if tags else "") + "</p>")
824
825
826def listing(cfg, items, headings=True):
827 """Posts (dicts with post, minutes, excerpt) as a list, newest first; with `headings`, grouped by year and month."""
828 out, year, month = [], None, None
829 for it in items:
830 p = it["post"]
831 if headings and p.date.year != year:
832 year, month = p.date.year, None
833 out.append(f'<h2 id="{year}">{year}</h2>')
834 if headings and (p.date.year, p.date.month) != month:
835 month = (p.date.year, p.date.month)
836 out.append(f'<h3 id="{year}-{p.date.month:02d}">{calendar.month_name[p.date.month]} {year}</h3>')
837 out.append(f'<article class="entry"><h4><a href="{esc(p.path, quote=True)}">{esc(p.title)}</a></h4>'
838 f'{post_meta_line(cfg, p, it["minutes"])}'
839 + (f'<div class="excerpt">{it["excerpt"][0]}</div>' if it["excerpt"][0] else "")
840 + (f'<p class="more"><a href="{esc(p.path, quote=True)}">Continue reading →</a></p>' if it["excerpt"][1] else "")
841 + "</article>")
842 return "\n".join(out)
843
844
845# ---------------------------------------------------------------- site files
846
847def apache_pattern(path):
848 """A RewriteRule pattern (per-directory context: no leading slash) matching exactly this path, with or without a trailing slash."""
849 text = urllib.parse.unquote(path).strip("/")
850 text = re.sub(r"([.^$*+?()\[\]{}|\\])", r"\\\1", text).replace(" ", r"\x20")
851 return f"^{text}/?$"
852
853
854def redirect_rules(cfg):
855 """Apache rules for the old WordPress URLs, from the config and the fixed WordPress URL shapes."""
856 rules = ["", "# Redirects from the old WordPress site (generated by blog.py from the config)."]
857 for old, new in sorted(cfg["redirects"].items()):
858 rules.append(f"RewriteRule {apache_pattern(old)} {new} [R=301,L]")
859 for wp_id, path in sorted(cfg["wp_ids"].items(), key=lambda kv: str(kv[0])):
860 for param in ("p", "page_id"):
861 rules += [f"RewriteCond %{{QUERY_STRING}} (^|&){param}={re.escape(str(wp_id))}(&|$)",
862 f"RewriteRule ^$ {path}? [R=301,L]"]
863 if cfg["home"]:
864 rules.append(f"RewriteRule {apache_pattern(cfg['home'])} / [R=301,L]")
865 feed = cfg["feed"]["path"]
866 rules += [f"RewriteRule ^comments/feed/?$ {feed} [R=301,L]",
867 f"RewriteRule ^feed/(rss2|rss|rdf|atom)/?$ {feed} [R=301,L]",
868 "RewriteRule ^tag/([^/]+)/feed/?$ /tag/$1/ [R=301,L]"]
869 if cfg["url"] == DEFAULT_URL:
870 rules.append(r"RewriteRule ^(\d{4}/\d\d/\d\d/[^/]+)/(feed|amp|comment-page-\d+|attachment/.+)/?$ /$1/ [R=301,L]")
871 rules += ["RewriteRule ^(category|author)/.*$ /blog/ [R=301,L]",
872 r"RewriteRule ^page/\d+/?$ /blog/ [R=301,L]",
873 r"RewriteRule ^(\d{4})/?$ /blog/#$1 [NE,R=301,L]",
874 r"RewriteRule ^(\d{4})/(\d\d)/?$ /blog/#$1-$2 [NE,R=301,L]",
875 r"RewriteRule ^(wp-admin|wp-login\.php|xmlrpc\.php) - [R=404,L]"]
876 return "\n".join(rules) + "\n"
877
878
879def htaccess_text(cfg):
880 with open(os.path.join(BLOG_DIR, "htaccess"), encoding="utf-8") as f:
881 return f.read().rstrip("\n") + "\n" + redirect_rules(cfg)
882
883
884def copy_media(cfg, out):
885 """The media folder's files, as they are (so /wp-content/uploads/... paths keep working). Only file types
886 a browser shows or plays: nothing the web server could execute. Symbolic links (files or folders) are skipped."""
887 if not cfg["media"]:
888 return 0
889 root, n = os.path.join(cfg["_source"], cfg["media"]), 0
890 for dirpath, dirs, files in os.walk(root):
891 dirs[:] = [d for d in dirs if not d.startswith(".") and not os.path.islink(os.path.join(dirpath, d))]
892 for name in files:
893 if (name.startswith(".") or os.path.splitext(name)[1].lower() not in MEDIA_EXTENSIONS
894 or os.path.islink(os.path.join(dirpath, name))): # a link could point at anything on the laptop
895 continue
896 dest = os.path.join(out, os.path.relpath(os.path.join(dirpath, name), root))
897 os.makedirs(os.path.dirname(dest), exist_ok=True)
898 shutil.copyfile(os.path.join(dirpath, name), dest)
899 n += 1
900 return n
901
902
903def audit_output(out):
904 """(file count, total bytes) of what an upload of `out` would send; BlogError if anything must not be published.
905 Allowed: generated pages, feed, css, icon, robots.txt, .htaccess, and media with an allowlisted extension.
906 Anything else (a .md, .json or .txt file, any other dotfile, a symlink) is refused. The build's own marker
907 file is skipped: the upload excludes it."""
908 if not os.path.isdir(out):
909 raise BlogError(f"{out} is not a folder")
910 bad, count, size = [], 0, 0
911 for dirpath, dirs, files in os.walk(out):
912 for name in dirs:
913 if name.startswith(".") or os.path.islink(os.path.join(dirpath, name)):
914 bad.append(os.path.relpath(os.path.join(dirpath, name), out))
915 for name in files:
916 path = os.path.join(dirpath, name)
917 if name == MARKER and dirpath == out:
918 continue
919 ext = os.path.splitext(name)[1].lower()
920 if (os.path.islink(path) or not (name in GENERATED_NAMES or (not name.startswith(".")
921 and ext in GENERATED_EXTENSIONS | MEDIA_EXTENSIONS))):
922 bad.append(os.path.relpath(path, out))
923 continue
924 count += 1
925 size += os.path.getsize(path)
926 if bad:
927 raise BlogError("refusing to upload; not allowed in the output: " + ", ".join(sorted(bad)[:20])
928 + (f" (and {len(bad) - 20} more)" if len(bad) > 20 else ""))
929 return count, size
930
931
932def safe_join(out, *parts):
933 """out/parts..., or BlogError when it would land outside `out` (a bad slug, a symlink in the way)."""
934 dest = os.path.join(out, *parts)
935 root, real = os.path.realpath(out), os.path.realpath(dest)
936 if not real.startswith(root + os.sep):
937 raise BlogError(f"refusing to write outside the output folder: {os.path.join(*parts)!r}")
938 return dest
939
940
941def prepare_out(out, cfg=None):
942 """Empty the output folder, but only one this tool made (or an empty one): `--out ~` must not wipe a home folder."""
943 if cfg:
944 out_real, src = os.path.realpath(out), os.path.realpath(cfg["_source"])
945 if out_real == src or out_real.startswith(src + os.sep) or src.startswith(out_real + os.sep):
946 raise BlogError("--out must be a separate folder, not the source folder or inside it")
947 if os.path.exists(out):
948 if os.listdir(out) and not os.path.exists(os.path.join(out, MARKER)):
949 raise BlogError(f"{out} is not empty and was not made by fw blog; choose another --out")
950 shutil.rmtree(out)
951 os.makedirs(out)
952 write(os.path.join(out, MARKER), "Made by fw blog build. It is emptied on every build.\n")
953
954
955# ---------------------------------------------------------------- build
956
957def build(cfg, out, htaccess=True):
958 """Writes the whole site into `out`. Returns a summary string."""
959 posts = [p for p in scan_posts(cfg) if p.published]
960 for p in posts:
961 if not p.date:
962 raise BlogError(f"{p.rel}: published, but no date (name the file YYYY-MM-DD-name.md or set date)")
963 posts.sort(key=lambda p: (p.date, p.rel), reverse=True)
964 if posts:
965 first, last = posts[-1].date.year, posts[0].date.year
966 cfg["_years"] = str(first) if first == last else f"{first}–{last}"
967 comments = load_comments(cfg)
968 prepare_out(out, cfg)
969 written = set()
970
971 def emit(path, content):
972 folder = urllib.parse.unquote(path).strip("/") # the web server decodes %xx before it looks for the file
973 if folder + path[-1:] in written:
974 raise BlogError(f"two pages want the URL {path}")
975 written.add(folder + path[-1:])
976 write(safe_join(out, folder, "index.html") if path.endswith("/") else safe_join(out, folder), content)
977
978 items, tags = [], {}
979 for p in posts:
980 excerpt_md, full = split_more(extract_body(p.text, cfg["body_section"], p.body_until, p.rel))
981 body = render_markdown(full, cfg)
982 text = plain_text(body)
983 minutes = max(1, math.ceil(len(text.split()) / cfg["reading_wpm"]))
984 it = {"post": p, "html": body, "minutes": minutes, "excerpt": make_excerpt(excerpt_md, body, cfg)}
985 items.append(it)
986 for t in p.tags:
987 tags.setdefault(tag_slug(cfg, t), []).append(it)
988 page_body = (f'<article class="post"><h1>{esc(p.title)}</h1>{post_meta_line(cfg, p, minutes)}'
989 f'<div class="post-body">{body}</div></article>{post_comments(cfg, comments, p)}')
990 emit(p.path, blog_page(cfg, p.title, page_body, p.path))
991
992 years = sorted({it["post"].date.year for it in items}, reverse=True)
993 index = ('<h1>Blog</h1>\n' + ('<p class="years">' + " ".join(f'<a href="#{y}">{y}</a>' for y in years) + "</p>\n"
994 + listing(cfg, items) if items else "<p>No posts yet.</p>"))
995 emit("/blog/", blog_page(cfg, "Blog", index, "/blog/"))
996 for slug, its in sorted(tags.items()):
997 name = tag_name(cfg, slug, next((t for t in its[0]["post"].tags if tag_slug(cfg, t) == slug), slug))
998 emit(f"/tag/{slug}/", blog_page(cfg, name, f'<h1>{esc(name)}</h1>\n{listing(cfg, its, headings=False)}', f"/tag/{slug}/"))
999
1000 home_done = False
1001 pdir = pages_dir(cfg)
1002 for rel in source_files(cfg, patterns=[pdir + "/*.md"]) if pdir else []:
1003 meta, text = parse_front_matter(read_text(os.path.join(cfg["_source"], rel)))
1004 bad = sorted(set(meta) - set(PAGE_FIELDS))
1005 if bad:
1006 raise BlogError(f"{rel}: unknown front matter key(s): " + ", ".join(bad))
1007 if meta.get("published") is not True: # like posts: a page is published only when it says so
1008 continue
1009 stem = os.path.splitext(os.path.basename(rel))[0]
1010 slug = check_slug(str(meta.get("slug") or stem), rel)
1011 title = str(meta.get("title") or title_of(text) or stem)
1012 page_html = render_markdown(extract_body(text, None, name=rel), cfg)
1013 if slug == cfg["home"]:
1014 emit("/", blog_page(cfg, title, f'<article class="post"><h1>{esc(title)}</h1>{page_html}</article>', "/"))
1015 home_done = True
1016 else:
1017 path = f"/{slug}/"
1018 emit(path, blog_page(cfg, title, f'<article class="post"><h1>{esc(title)}</h1>{page_html}</article>', path))
1019 if cfg["home"] and not home_done:
1020 raise BlogError(f'home page "{cfg["home"]}" not found in pages_dir')
1021 if not home_done:
1022 emit("/", blog_page(cfg, cfg["title"], index, "/"))
1023 emit("/404.html", blog_page(cfg, "Not found", '<h1>Not found</h1><p>There is nothing at this address. '
1024 '<a href="/blog/">All posts</a></p>', "/404.html"))
1025
1026 entries = [{"title": it["post"].title, "url": cfg["base_url"] + it["post"].path, "updated": localize(cfg, it["post"].date),
1027 "published": localize(cfg, it["post"].date), "content": absolutize(it["html"], cfg["base_url"])}
1028 for it in items[:cfg["feed"]["entries"]]]
1029 feed_path = cfg["feed"]["path"].strip("/")
1030 write(safe_join(out, urllib.parse.unquote(feed_path), "index.xml"),
1031 atom.atom_feed(cfg["title"], cfg["base_url"] + "/",
1032 localize(cfg, posts[0].date) if posts else datetime.datetime.now(datetime.timezone.utc),
1033 entries, copyright_text(cfg), author=cfg["author"] or None,
1034 feed_url=f'{cfg["base_url"]}/{feed_path}/', subtitle=cfg["tagline"] or None))
1035
1036 for name in ("_shared.css", "favicon.ico"):
1037 shutil.copyfile(os.path.join(SITE_DIR, name), os.path.join(out, name))
1038 for name in ("blog.js", "robots.txt"):
1039 shutil.copyfile(os.path.join(BLOG_DIR, name), os.path.join(out, name))
1040 with open(os.path.join(SITE_DIR, "_shared.css")) as f:
1041 shared = f.read()
1042 with open(os.path.join(BLOG_DIR, "blog.css")) as f:
1043 write(os.path.join(out, "blog.css"), f.read() + theme_overrides(shared))
1044 media = copy_media(cfg, out)
1045 if htaccess:
1046 write(os.path.join(out, ".htaccess"), htaccess_text(cfg))
1047 return f"{len(posts)} posts, {len(tags)} tags, {len(written)} pages, {media} media files -> {out}"
1048
1049
1050# ---------------------------------------------------------------- deploy
1051
1052# --delete removes whatever else is in the target folder, except these.
1053RSYNC_OPTS = ["-rlptz", "--delete", "--exclude", ".well-known", "--exclude", ".dh-diag", "--exclude", ".fw-blog-out",
1054 "--exclude", "/_comments/"]
1055
1056
1057def watch_name(config_path):
1058 """<folder name of the config, lowercased>-<first 8 hex of sha256 of its real path>: names the state folder
1059 and the launchd label, so two configs in same-named folders never share state."""
1060 real = os.path.realpath(config_path)
1061 return f"{os.path.basename(os.path.dirname(real)).lower()}-{hashlib.sha256(real.encode()).hexdigest()[:8]}"
1062
1063
1064def state_dir(config_path):
1065 """Where `fw blog watch` and `fw blog deploy` keep their state for one blog: ~/.fw-blog-watch/<watch_name>/
1066 (the environment variable FW_BLOG_WATCH_DIR replaces the whole path, for tests)."""
1067 return os.environ.get("FW_BLOG_WATCH_DIR") or os.path.join(os.path.expanduser("~"), ".fw-blog-watch",
1068 watch_name(config_path))
1069
1070
1071def acquire_lock(directory, now=time.time):
1072 """The exclusive lock of one blog's state folder (a tick of the watcher, or a real manual deploy), as an open
1073 file to close when done; None when someone holds it. The holder's start time is written into the file."""
1074 os.makedirs(directory, exist_ok=True)
1075 f = open(os.path.join(directory, "lock"), "a+")
1076 try:
1077 fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
1078 except OSError:
1079 f.close()
1080 return None
1081 f.seek(0)
1082 f.truncate(0)
1083 f.write(repr(float(now())))
1084 f.flush()
1085 return f
1086
1087
1088def lock_since(directory):
1089 """The start time (epoch seconds) the current lock holder wrote, or None."""
1090 try:
1091 with open(os.path.join(directory, "lock")) as f:
1092 return float(f.read().strip())
1093 except (OSError, ValueError):
1094 return None
1095
1096
1097def generator_hash():
1098 """sha256 of everything that shapes the output besides the posts: this generator, the feed code, gen.py (nav and
1099 footer), the shared CSS/icon and the blog's own CSS/JS/robots/htaccess. `fw blog watch` deploys on its own only
1100 while this equals the value stored in deployed.json."""
1101 files = [os.path.join(HERE, n) for n in ("blog.py", "atom.py", "gen.py")]
1102 files += [os.path.join(SITE_DIR, n) for n in ("_shared.css", "favicon.ico")]
1103 files += sorted(glob.glob(os.path.join(BLOG_DIR, "*")))
1104 h = hashlib.sha256()
1105 for path in files:
1106 if os.path.isfile(path):
1107 h.update(os.path.relpath(path, HERE).encode() + b"\0")
1108 with open(path, "rb") as f:
1109 h.update(hashlib.sha256(f.read()).digest())
1110 return h.hexdigest()
1111
1112
1113def manifest(out):
1114 """{relative path: sha256 of the content} of a built folder (the build's marker file is not part of the site)."""
1115 files = {}
1116 for dirpath, _, names in os.walk(out):
1117 for name in names:
1118 path = os.path.join(dirpath, name)
1119 rel = os.path.relpath(path, out).replace(os.sep, "/")
1120 if rel != MARKER:
1121 with open(path, "rb") as f:
1122 files[rel] = hashlib.sha256(f.read()).hexdigest()
1123 return dict(sorted(files.items()))
1124
1125
1126def config_hash(cfg):
1127 """sha256 of the loaded config (keys starting with _ are internal and left out)."""
1128 public = {k: v for k, v in cfg.items() if not k.startswith("_")}
1129 return hashlib.sha256(json.dumps(public, sort_keys=True, default=str).encode()).hexdigest()
1130
1131
1132def write_deployed(directory, files, when=None, generator=None, cfg=None):
1133 """deployed.json: what the last real upload sent, the generator hash it was built with and (with `cfg`) the hash of
1134 the config and the deploy target. `fw blog watch` compares the next build with it."""
1135 os.makedirs(directory, exist_ok=True)
1136 when = when or datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
1137 tmp = os.path.join(directory, "deployed.json.tmp")
1138 with open(tmp, "w", encoding="utf-8") as f:
1139 json.dump({"files": files, "time": when, "generator": generator or GENERATOR_HASH,
1140 **({"config": config_hash(cfg), "target": deploy_target(cfg)} if cfg else {})}, f, indent=1)
1141 os.replace(tmp, os.path.join(directory, "deployed.json"))
1142
1143
1144def _read_bytes(path):
1145 try:
1146 with open(path, "rb") as f:
1147 return f.read()
1148 except OSError:
1149 return None
1150
1151
1152def _forget_fingerprint(directory):
1153 """After a manual upload the watcher's idea of "handled" may be stale: drop it so the next tick re-evaluates."""
1154 path = os.path.join(directory, "state.json")
1155 try:
1156 with open(path, encoding="utf-8") as f:
1157 state = json.load(f)
1158 if not isinstance(state, dict) or not ({"fingerprint", "handled"} & set(state)):
1159 return
1160 state.pop("fingerprint", None)
1161 state.pop("handled", None)
1162 with open(path + ".tmp", "w", encoding="utf-8") as f:
1163 json.dump(state, f, indent=1)
1164 os.replace(path + ".tmp", path)
1165 except (OSError, ValueError):
1166 pass
1167
1168
1169def deploy_site(cfg, out, dry_run=False, confirm=None, run=subprocess.run, prebuilt=False, locked=False, timeout=None):
1170 """Uploads ONLY a freshly built, audited output folder, never the source folder. Builds into `out` (unless
1171 `prebuilt`: the caller just built it), audits it, then (not for a dry run) asks `confirm(target)` if given and
1172 runs rsync (no time limit unless `timeout`). A real upload takes the state folder's lock around the upload only
1173 (unless the caller `locked` it already: the watcher), and writes deployed.json. True when something was
1174 uploaded; BlogError on any failure."""
1175 target = deploy_target(cfg)
1176 directory = state_dir(cfg["_path"])
1177 baseline = _read_bytes(os.path.join(directory, "deployed.json")) # to notice a deploy by the watcher meanwhile
1178 if not prebuilt:
1179 print(build(cfg, out))
1180 count, size = audit_output(out)
1181 print(f"audit ok: {count} files, {size / 1024:.0f} KB")
1182 lock = None
1183 if dry_run:
1184 cmd = ["rsync", "-n", "-v", *RSYNC_OPTS, out + "/", target + "/"]
1185 else:
1186 if confirm and not confirm(target):
1187 raise BlogError("not uploaded")
1188 if not locked:
1189 lock = acquire_lock(directory)
1190 if lock is None:
1191 raise BlogError("a watch tick is running, try again in a minute")
1192 if _read_bytes(os.path.join(directory, "deployed.json")) != baseline:
1193 lock.close()
1194 raise BlogError("the watcher deployed while you were answering; run fw blog deploy again")
1195 cmd = ["rsync", *RSYNC_OPTS, out + "/", target + "/"]
1196 try:
1197 try:
1198 code = run(cmd, timeout=timeout).returncode
1199 except subprocess.TimeoutExpired:
1200 raise BlogError(f"rsync timed out after {timeout} s")
1201 if code:
1202 raise BlogError(f"rsync failed (exit {code})")
1203 if not dry_run:
1204 write_deployed(directory, manifest(out), cfg=cfg)
1205 if not locked:
1206 _forget_fingerprint(directory)
1207 return not dry_run
1208 finally:
1209 if lock:
1210 lock.close()
1211
1212
1213def ask_to_upload(target):
1214 """The prompt of `fw blog deploy` (needs a terminal)."""
1215 if not sys.stdin.isatty():
1216 raise BlogError(f"not a terminal; pass --yes to upload to {target}")
1217 try:
1218 return input(f"Upload the output above to {target}, deleting other files there? [y/N] ").strip() in ("y", "Y")
1219 except EOFError:
1220 return False
1221 except KeyboardInterrupt:
1222 print(file=sys.stderr)
1223 raise SystemExit(1)
1224
1225
1226GENERATOR_HASH = generator_hash() # computed once, when the module loads: the code that is actually running
1227
1228
1229# ---------------------------------------------------------------- command line
1230
1231def main(argv):
1232 ap = argparse.ArgumentParser(prog="blog.py", description=__doc__.split("\n")[0])
1233 sub = ap.add_subparsers(dest="cmd", required=True)
1234 sub.add_parser("list").add_argument("config")
1235 b = sub.add_parser("build")
1236 b.add_argument("config")
1237 b.add_argument("--out", required=True)
1238 b.add_argument("--no-htaccess", action="store_true", help="skip .htaccess (local preview)")
1239 sub.add_parser("audit").add_argument("out")
1240 sub.add_parser("deploy-target").add_argument("config")
1241 d = sub.add_parser("deploy")
1242 d.add_argument("config")
1243 d.add_argument("--out", required=True)
1244 d.add_argument("--dry-run", action="store_true")
1245 d.add_argument("--yes", action="store_true", help="upload without asking")
1246 args = ap.parse_args(argv[1:])
1247 try:
1248 if args.cmd == "audit":
1249 count, size = audit_output(args.out)
1250 print(f"audit ok: {count} files, {size / 1024:.0f} KB")
1251 return
1252 cfg = load_config(args.config)
1253 if args.cmd == "list":
1254 print("\n".join(list_posts(cfg)))
1255 elif args.cmd == "build":
1256 print(build(cfg, os.path.abspath(args.out), htaccess=not args.no_htaccess))
1257 elif args.cmd == "deploy":
1258 deploy_site(cfg, os.path.abspath(args.out), dry_run=args.dry_run,
1259 confirm=None if args.yes else ask_to_upload)
1260 else:
1261 print(deploy_target(cfg))
1262 except BlogError as e:
1263 raise SystemExit(f"blog.py: {e}")
1264
1265
1266if __name__ == "__main__":
1267 main(sys.argv)