fuzzyworld-tools / files / lib / wp_import.py
wp_import.py
| 1 | #!/usr/bin/env python3 |
| 2 | """WordPress importer and checker for the static blog (laptop only, standard library only). |
| 3 | |
| 4 | wp_import.py import <config> --wp URL [--dry-run] [--force] [--refresh] [--probe-links] |
| 5 | [--cache DIR] [--no-media] [--media-from SRC] [--timezone TZ] [--home SLUG] |
| 6 | wp_import.py check <config> [--out DIR] [--cache DIR] |
| 7 | |
| 8 | `fw blog import-wp` and `fw blog check` are the usual way in. The importer reads the WordPress REST API |
| 9 | (posts, pages, comments, tags, media) and caches the raw JSON in <source>/wordpress/_api/ (a later run uses |
| 10 | the cache; --refresh fetches again; --cache DIR reads or fills another folder). It matches each public post |
| 11 | to a Markdown file in <source> by comparing normalized word sequences (difflib ratio >= 0.95; each file is |
| 12 | used once). A post with no match is written to <source>/wordpress/<YYYY-MM-DD>-<slug>.md as Markdown made |
| 13 | from its HTML. Pages go to <source>/<pages_dir>/ (each with `published: true` in its front matter: pages, like posts, are opt-in), comments to |
| 14 | the `comments` file (relative to the config file's folder, like the generator reads it; name, date and text only), |
| 15 | images and files are copied from the old server with a read-only rsync, and the config receives `posts`, |
| 16 | `tags`, `redirects`, `wp_ids` and a few other entries without overwriting existing ones (unless --force). |
| 17 | |
| 18 | --dry-run writes only <source>/wordpress/_api/ and <source>/wordpress/_report.txt. Everything else is written |
| 19 | inside <source> and the config file only. Password-protected posts and their comments are skipped and listed |
| 20 | in the report. --probe-links sends HEAD requests (to the old site) for internal links that the new site will |
| 21 | not generate, and records the 301 targets as `redirects`. |
| 22 | |
| 23 | `check` builds the site into a temporary folder (or reads --out), compares each built post with the cached |
| 24 | WordPress text, compares <br> counts, and verifies that every old post, page and tag URL and every redirect |
| 25 | target exists in the output. It exits with status 1 when it finds a problem. |
| 26 | """ |
| 27 | |
| 28 | import argparse |
| 29 | import collections |
| 30 | import datetime |
| 31 | import difflib |
| 32 | import glob |
| 33 | import html |
| 34 | import html.parser |
| 35 | import json |
| 36 | import os |
| 37 | import re |
| 38 | import shutil |
| 39 | import subprocess |
| 40 | import sys |
| 41 | import tempfile |
| 42 | import unicodedata |
| 43 | import urllib.error |
| 44 | import urllib.parse |
| 45 | import urllib.request |
| 46 | import zoneinfo |
| 47 | |
| 48 | HERE = os.path.dirname(os.path.abspath(__file__)) |
| 49 | sys.path.insert(0, os.path.join(HERE, "..", "server")) |
| 50 | import blog # noqa: E402 |
| 51 | |
| 52 | MATCH_THRESHOLD = 0.95 # a source file with at least this word-sequence similarity is the post's source |
| 53 | CHECK_THRESHOLD = 0.98 # `check` lists built posts below this similarity to the WordPress text |
| 54 | USER_AGENT = "fw-blog-import/1 (static blog migration)" |
| 55 | API_NAMES = ("posts", "pages", "comments", "tags", "media") |
| 56 | # Names the development cache used before the importer existed (one file per page of results). |
| 57 | LEGACY_CACHE = {"posts": ("p[0-9]*.json",), "comments": ("c.json",)} |
| 58 | |
| 59 | |
| 60 | class WpImportError(blog.BlogError): |
| 61 | """A problem reported without a traceback.""" |
| 62 | |
| 63 | |
| 64 | # ---------------------------------------------------------------- HTML -> tree |
| 65 | |
| 66 | VOID = {"br", "hr", "img", "meta", "link", "input", "area", "base", "col", "embed", "source", "track", "wbr"} |
| 67 | BLOCK = {"p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote", "ul", "ol", "li", "pre", "hr", "figure", |
| 68 | "figcaption", "table", "thead", "tbody", "tr", "td", "th", "section", "article", "header", "footer", |
| 69 | "aside", "center", "address", "dl", "dt", "dd", "form", "main", "nav", "details", "summary"} |
| 70 | KEEP_TAGS = {"del", "ins", "sup", "sub", "u", "s", "small", "mark", "kbd"} # inline tags kept as HTML in Markdown |
| 71 | DROP_CLASSES = {"sharedaddy", "jp-relatedposts", "jetpack-likes-widget-wrapper"} |
| 72 | |
| 73 | |
| 74 | class Node: |
| 75 | __slots__ = ("tag", "attrs", "children", "text") |
| 76 | |
| 77 | def __init__(self, tag, attrs=None, text=None): |
| 78 | self.tag, self.attrs, self.children, self.text = tag, dict(attrs or {}), [], text |
| 79 | |
| 80 | |
| 81 | class _Builder(html.parser.HTMLParser): |
| 82 | def __init__(self): |
| 83 | super().__init__(convert_charrefs=True) |
| 84 | self.root = Node("#root") |
| 85 | self.stack = [self.root] |
| 86 | |
| 87 | def handle_starttag(self, tag, attrs): |
| 88 | node = Node(tag, {k: (v if v is not None else "") for k, v in attrs}) |
| 89 | self.stack[-1].children.append(node) |
| 90 | if tag not in VOID: |
| 91 | self.stack.append(node) |
| 92 | |
| 93 | def handle_startendtag(self, tag, attrs): |
| 94 | self.handle_starttag(tag, attrs) |
| 95 | if tag not in VOID: |
| 96 | self.handle_endtag(tag) |
| 97 | |
| 98 | def handle_endtag(self, tag): |
| 99 | for i in range(len(self.stack) - 1, 0, -1): |
| 100 | if self.stack[i].tag == tag: |
| 101 | del self.stack[i:] |
| 102 | return |
| 103 | |
| 104 | def handle_data(self, data): |
| 105 | self.stack[-1].children.append(Node(None, text=data)) |
| 106 | |
| 107 | def handle_comment(self, data): |
| 108 | self.stack[-1].children.append(Node("#comment", text=data)) |
| 109 | |
| 110 | |
| 111 | def parse_html(text): |
| 112 | b = _Builder() |
| 113 | b.feed(text or "") |
| 114 | b.close() |
| 115 | return b.root |
| 116 | |
| 117 | |
| 118 | def classes(node): |
| 119 | return (node.attrs.get("class") or "").split() |
| 120 | |
| 121 | |
| 122 | def is_dropped(node): |
| 123 | """What WordPress adds around a post and the page does not need: scripts, the reading-time plugin's span, |
| 124 | share buttons, and every HTML comment except <!--more--> (comments may hold text that was cut on purpose).""" |
| 125 | tag = node.tag |
| 126 | if tag == "#comment": |
| 127 | return node.text.strip() != "more" |
| 128 | if tag in ("script", "style", "noscript", "head", "title"): |
| 129 | return True |
| 130 | cl = classes(node) |
| 131 | if tag == "span" and ("rt-reading-time" in cl or "span-reading-time" in cl): |
| 132 | return True |
| 133 | if node.attrs.get("id") == "bsf_rt_marker": |
| 134 | return True |
| 135 | if tag == "span" and str(node.attrs.get("id", "")).startswith("more-"): |
| 136 | return True |
| 137 | return bool(DROP_CLASSES & set(cl)) |
| 138 | |
| 139 | |
| 140 | def youtube_url(src): |
| 141 | m = re.match(r"(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/embed/([\w-]+)", src or "") |
| 142 | return f"https://www.youtube.com/watch?v={m.group(1)}" if m else None |
| 143 | |
| 144 | |
| 145 | def embed_of(node): |
| 146 | """(link text, URL) for an embed the site cannot show (its CSP blocks frames and scripts): a YouTube frame |
| 147 | and a Flickr embed become a plain link. None for anything else.""" |
| 148 | if node.tag == "iframe": |
| 149 | url = youtube_url(node.attrs.get("src")) |
| 150 | return (f"Video: {url}", url) if url else None |
| 151 | if node.tag == "a" and "data-flickr-embed" in node.attrs and node.attrs.get("href"): |
| 152 | alt = next((c.attrs.get("alt") for c in node.children if c.tag == "img" and c.attrs.get("alt")), None) |
| 153 | return (node.attrs.get("title") or alt or "Photo on Flickr", node.attrs["href"]) |
| 154 | return None |
| 155 | |
| 156 | |
| 157 | # ---------------------------------------------------------------- tree -> Markdown |
| 158 | |
| 159 | def collapse(text): |
| 160 | return re.sub(r"[ \t\r\n\f ]+", " ", text) |
| 161 | |
| 162 | |
| 163 | def esc_text(text): |
| 164 | text = text.replace("\\", "\\\\") |
| 165 | text = re.sub(r"([*`\[\]])", r"\\\1", text) |
| 166 | text = re.sub(r"(?<![A-Za-z0-9])_|_(?![A-Za-z0-9])", r"\\_", text) |
| 167 | text = re.sub(r"&(?=#?\w+;)", "&", text) |
| 168 | return text.replace("<", "<") |
| 169 | |
| 170 | |
| 171 | def esc_line_starts(text): |
| 172 | out = [] |
| 173 | for line in text.split("\n"): |
| 174 | m = re.match(r"(\d+)([.)])(\s|$)", line) |
| 175 | if m: |
| 176 | line = f"{m.group(1)}\\{m.group(2)}" + line[m.end(2):] |
| 177 | elif re.match(r"(#|>|[-+](\s|$)|-{2,}\s*$)", line): |
| 178 | line = "\\" + line |
| 179 | out.append(line) |
| 180 | return "\n".join(out) |
| 181 | |
| 182 | |
| 183 | def finish_inline(text, single_line=False): |
| 184 | """Tidy one paragraph's worth of inline Markdown. A "\\n" in it came from a <br>: the first of several in a |
| 185 | row stays a newline, the others become literal <br> so the count of <br> is kept.""" |
| 186 | text = text.strip(" \n") |
| 187 | text = re.sub(r"(?: *\n)+ *", lambda m: "\n" + "<br>" * (m.group(0).count("\n") - 1), text) |
| 188 | text = re.sub(r" {2,}", " ", text) |
| 189 | if single_line: |
| 190 | text = re.sub(r"\s*\n\s*", " ", text) |
| 191 | return esc_line_starts(text) |
| 192 | |
| 193 | |
| 194 | def wrap_marks(marker, content): |
| 195 | """*content*, one marker pair per line (so an emphasis never spans a <br>), spaces kept outside.""" |
| 196 | out = [] |
| 197 | for line in content.split("\n"): |
| 198 | core = line.strip() |
| 199 | out.append(line[:len(line) - len(line.lstrip())] + marker + core + marker + line[len(line.rstrip()):] if core else line) |
| 200 | return "\n".join(out) if content.strip() else "" |
| 201 | |
| 202 | |
| 203 | def link_target(url): |
| 204 | return url.strip().replace(" ", "%20").replace("(", "%28").replace(")", "%29") |
| 205 | |
| 206 | |
| 207 | class Converter: |
| 208 | def __init__(self, notes=None): |
| 209 | self.notes = notes if notes is not None else [] |
| 210 | |
| 211 | def inline(self, node): |
| 212 | if node.tag is None: |
| 213 | return esc_text(collapse(node.text)) |
| 214 | if is_dropped(node): |
| 215 | return "" |
| 216 | tag = node.tag |
| 217 | if tag == "#comment": |
| 218 | return "" |
| 219 | if tag == "br": |
| 220 | return "\n" |
| 221 | if tag == "img": |
| 222 | src = node.attrs.get("src") |
| 223 | alt = re.sub(r"[\[\]]", "", collapse(node.attrs.get("alt", "")).strip()) |
| 224 | return f"})" if src else "" |
| 225 | embed = embed_of(node) |
| 226 | if embed: |
| 227 | return f"[{esc_text(embed[0])}]({link_target(embed[1])})" |
| 228 | if tag == "iframe": |
| 229 | self.notes.append(f"iframe dropped: {node.attrs.get('src', '?')}") |
| 230 | return "" |
| 231 | inner = "".join(self.inline(c) for c in node.children) |
| 232 | if tag == "a": |
| 233 | href = node.attrs.get("href", "").strip() |
| 234 | if not href or href.lower().startswith("javascript:"): |
| 235 | return inner |
| 236 | return f"[{inner}]({link_target(href)})" if inner.strip() else "" |
| 237 | if tag in ("em", "i"): |
| 238 | return wrap_marks("*", inner) |
| 239 | if tag in ("strong", "b"): |
| 240 | return wrap_marks("**", inner) |
| 241 | if tag == "code": |
| 242 | text = "".join(self.raw_text(c) for c in node.children) |
| 243 | text = collapse(text).strip() |
| 244 | if not text: |
| 245 | return "" |
| 246 | fence = "`" * (max([len(r) for r in re.findall(r"`+", text)] + [0]) + 1) |
| 247 | pad = " " if text.startswith("`") or text.endswith("`") else "" |
| 248 | return f"{fence}{pad}{text}{pad}{fence}" |
| 249 | if tag in KEEP_TAGS: |
| 250 | return f"<{tag}>{inner}</{tag}>" if inner.strip() else "" |
| 251 | if tag in BLOCK: |
| 252 | return f" {inner} " |
| 253 | return inner |
| 254 | |
| 255 | def raw_text(self, node): |
| 256 | if node.tag is None: |
| 257 | return node.text |
| 258 | if is_dropped(node): |
| 259 | return "" |
| 260 | if node.tag == "br": |
| 261 | return "\n" |
| 262 | return "".join(self.raw_text(c) for c in node.children) |
| 263 | |
| 264 | def blocks(self, children): |
| 265 | """The Markdown blocks (strings) for a list of nodes.""" |
| 266 | out, run = [], [] |
| 267 | |
| 268 | def flush(): |
| 269 | text = finish_inline("".join(run)) |
| 270 | run.clear() |
| 271 | if text: |
| 272 | out.append(text) |
| 273 | |
| 274 | for node in children: |
| 275 | if node.tag is not None and is_dropped(node): |
| 276 | continue |
| 277 | if node.tag == "#comment": # only <!--more--> survives is_dropped |
| 278 | flush() |
| 279 | out.append("<!--more-->") |
| 280 | elif node.tag is None or node.tag not in BLOCK and node.tag != "iframe": |
| 281 | run.append(self.inline(node)) |
| 282 | elif node.tag == "iframe": |
| 283 | flush() |
| 284 | text = self.inline(node) |
| 285 | if text: |
| 286 | out.append(text) |
| 287 | else: |
| 288 | flush() |
| 289 | out.extend(self.block(node)) |
| 290 | flush() |
| 291 | return out |
| 292 | |
| 293 | def block(self, node): |
| 294 | tag = node.tag |
| 295 | if tag == "hr": |
| 296 | return ["---"] |
| 297 | if re.fullmatch(r"h[1-6]", tag): |
| 298 | # a `## ` line would end the post body, so no heading is higher than ### |
| 299 | text = finish_inline("".join(self.inline(c) for c in node.children), single_line=True) |
| 300 | return [f"{'#' * max(3, int(tag[1]))} {text}"] if text else [] |
| 301 | if tag == "pre": |
| 302 | return self.pre(node) |
| 303 | if tag in ("ul", "ol"): |
| 304 | text = self.list(node, tag == "ol") |
| 305 | return [text] if text else [] |
| 306 | if tag == "blockquote": |
| 307 | inner = "\n\n".join(self.blocks(node.children)) |
| 308 | return ["\n".join(f"> {line}" if line else ">" for line in inner.split("\n"))] if inner else [] |
| 309 | return self.blocks(node.children) |
| 310 | |
| 311 | def pre(self, node): |
| 312 | text = self.raw_text(node).strip("\n") |
| 313 | if not text.strip(): |
| 314 | return [] |
| 315 | code = next((c for c in node.children if c.tag == "code"), None) |
| 316 | lang = next((c[9:] for c in classes(code) if c.startswith("language-")), "") if code else "" |
| 317 | fence = "`" * max(3, max([len(r) for r in re.findall(r"`+", text)] + [0]) + 1) |
| 318 | return [f"{fence}{lang}\n{text}\n{fence}"] |
| 319 | |
| 320 | def list(self, node, ordered): |
| 321 | items, loose = [], False |
| 322 | for n, li in enumerate((c for c in node.children if c.tag == "li" and not is_dropped(c)), 1): |
| 323 | blocks = self.blocks(li.children) |
| 324 | if not blocks: |
| 325 | continue |
| 326 | text = blocks[0] |
| 327 | for b in blocks[1:]: |
| 328 | is_list = re.match(r"(- |\d+\. )", b) is not None |
| 329 | loose = loose or not is_list |
| 330 | text += ("\n" if is_list else "\n\n") + b |
| 331 | text = text.replace("\n", "\n ") |
| 332 | items.append((f"{n}. " if ordered else "- ") + re.sub(r"\n {4}(?=\n)", "\n", text)) |
| 333 | return ("\n\n" if loose else "\n").join(items) |
| 334 | |
| 335 | |
| 336 | def html_to_markdown(text, notes=None): |
| 337 | """Markdown for WordPress HTML: <br> becomes a line break, </p> a blank line; see Converter for the rest.""" |
| 338 | conv = Converter(notes) |
| 339 | return "\n\n".join(conv.blocks(parse_html(text).children)).strip() |
| 340 | |
| 341 | |
| 342 | def html_text(text): |
| 343 | """The words a reader sees, as one string (embed links included, scripts and comments not).""" |
| 344 | out = [] |
| 345 | |
| 346 | def walk(node): |
| 347 | if node.tag is None: |
| 348 | out.append(node.text) |
| 349 | return |
| 350 | if is_dropped(node) or node.tag in ("img", "iframe") and not embed_of(node): |
| 351 | return |
| 352 | embed = embed_of(node) |
| 353 | if embed: |
| 354 | out.append(f" {embed[0]} ") |
| 355 | return |
| 356 | sep = " " if node.tag in BLOCK or node.tag == "br" else "" |
| 357 | out.append(sep) |
| 358 | for c in node.children: |
| 359 | walk(c) |
| 360 | out.append(sep) |
| 361 | |
| 362 | walk(parse_html(text)) |
| 363 | return "".join(out) |
| 364 | |
| 365 | |
| 366 | def html_br_count(text): |
| 367 | def count(node): |
| 368 | if node.tag is not None and is_dropped(node): |
| 369 | return 0 |
| 370 | return (1 if node.tag == "br" else 0) + sum(count(c) for c in node.children) |
| 371 | return count(parse_html(text)) |
| 372 | |
| 373 | |
| 374 | def html_plain(text): |
| 375 | """Plain text with paragraphs and line breaks kept (for comments: no markup survives).""" |
| 376 | out = [] |
| 377 | |
| 378 | def walk(node): |
| 379 | if node.tag is None: |
| 380 | out.append(collapse(node.text)) |
| 381 | elif node.tag == "br": |
| 382 | out.append("\n") |
| 383 | elif not is_dropped(node): |
| 384 | sep = "\n\n" if node.tag in BLOCK else "" |
| 385 | out.append(sep) |
| 386 | for c in node.children: |
| 387 | walk(c) |
| 388 | out.append(sep) |
| 389 | |
| 390 | walk(parse_html(text)) |
| 391 | s = re.sub(r" *\n *", "\n", "".join(out)) |
| 392 | return re.sub(r"\n{3,}", "\n\n", s).strip() |
| 393 | |
| 394 | |
| 395 | # ---------------------------------------------------------------- comparing texts |
| 396 | |
| 397 | def norm_words(text): |
| 398 | """Words only: lower case, no punctuation, no apostrophes (WordPress turns ' into ’ and -- into –).""" |
| 399 | text = unicodedata.normalize("NFKC", text).lower() |
| 400 | text = re.sub(r"['’‘ʼ`]", "", text) |
| 401 | return re.findall(r"[^\W_]+", text) |
| 402 | |
| 403 | |
| 404 | def md_words(markdown_text): |
| 405 | """The words of a Markdown source (links keep their text, images and URLs are dropped).""" |
| 406 | text = re.sub(r"!\[[^\]]*\]\([^)]*\)", " ", markdown_text) |
| 407 | text = re.sub(r"\]\([^)]*\)", "] ", text) |
| 408 | text = re.sub(r"<https?://[^>]*>", " ", text) |
| 409 | text = re.sub(r"<[^>]+>", " ", text) |
| 410 | return norm_words(html.unescape(text)) |
| 411 | |
| 412 | |
| 413 | def similarity(a, b): |
| 414 | if not a and not b: |
| 415 | return 1.0 |
| 416 | return difflib.SequenceMatcher(None, a, b, autojunk=False).ratio() |
| 417 | |
| 418 | |
| 419 | def title_text(rendered): |
| 420 | return collapse(html.unescape(re.sub(r"<[^>]+>", "", rendered or ""))).strip() |
| 421 | |
| 422 | |
| 423 | # ---------------------------------------------------------------- fetching |
| 424 | |
| 425 | class _NoRedirect(urllib.request.HTTPRedirectHandler): |
| 426 | def redirect_request(self, *args, **kwargs): |
| 427 | return None |
| 428 | |
| 429 | |
| 430 | class Fetcher: |
| 431 | """The REST API and its cache. `cache` is a folder of <name>.json files; a file there is used instead of |
| 432 | fetching unless `refresh`. `get_json(url) -> (data, total_pages)` and `head(url) -> (status, location)` can be |
| 433 | replaced (tests do).""" |
| 434 | |
| 435 | def __init__(self, wp, cache, refresh=False, get_json=None, head=None, log=lambda s: None): |
| 436 | self.wp, self.cache, self.refresh, self.log = wp.rstrip("/"), cache, refresh, log |
| 437 | self.get_json = get_json or self._get_json |
| 438 | self.head = head or self._head |
| 439 | |
| 440 | def _get_json(self, url): |
| 441 | req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "application/json"}) |
| 442 | with urllib.request.urlopen(req, timeout=60) as r: |
| 443 | return json.load(r), int(r.headers.get("X-WP-TotalPages") or 0) |
| 444 | |
| 445 | def _head(self, url): |
| 446 | opener = urllib.request.build_opener(_NoRedirect) |
| 447 | req = urllib.request.Request(url, method="HEAD", headers={"User-Agent": USER_AGENT}) |
| 448 | try: |
| 449 | with opener.open(req, timeout=30) as r: |
| 450 | return r.status, None |
| 451 | except urllib.error.HTTPError as e: |
| 452 | return e.code, e.headers.get("Location") |
| 453 | except (urllib.error.URLError, OSError) as e: |
| 454 | return 0, str(e) |
| 455 | |
| 456 | def path(self, name): |
| 457 | return os.path.join(self.cache, name + ".json") |
| 458 | |
| 459 | def cached(self, name): |
| 460 | """The cached list for `name`, or None.""" |
| 461 | if os.path.exists(self.path(name)): |
| 462 | with open(self.path(name), encoding="utf-8") as f: |
| 463 | return json.load(f) |
| 464 | files = sorted(f for pat in LEGACY_CACHE.get(name, ()) for f in glob.glob(os.path.join(self.cache, pat))) |
| 465 | if files: |
| 466 | data = [] |
| 467 | for f in files: |
| 468 | with open(f, encoding="utf-8") as fh: |
| 469 | data += json.load(fh) |
| 470 | return data |
| 471 | return None |
| 472 | |
| 473 | def collection(self, name): |
| 474 | if not self.refresh: |
| 475 | data = self.cached(name) |
| 476 | if data is not None: |
| 477 | return data |
| 478 | data, page = [], 1 |
| 479 | while True: |
| 480 | order = "id" if name == "tags" else "date" # the tags endpoint has no date to order by (HTTP 400) |
| 481 | url = f"{self.wp}/wp-json/wp/v2/{name}?per_page=100&page={page}&orderby={order}&order=asc" |
| 482 | try: |
| 483 | chunk, total = self.get_json(url) |
| 484 | except urllib.error.HTTPError as e: |
| 485 | if e.code == 400 and page > 1: # past the last page |
| 486 | break |
| 487 | raise WpImportError(f"{url}: HTTP {e.code}") |
| 488 | except (urllib.error.URLError, OSError, ValueError) as e: |
| 489 | raise WpImportError(f"{url}: {e}") |
| 490 | data += chunk |
| 491 | if len(chunk) < 100 or (total and page >= total): |
| 492 | break |
| 493 | page += 1 |
| 494 | os.makedirs(self.cache, exist_ok=True) |
| 495 | atomic_write(self.path(name), json.dumps(data, ensure_ascii=False, indent=1) + "\n") |
| 496 | self.log(f"fetched {len(data)} {name}") |
| 497 | return data |
| 498 | |
| 499 | |
| 500 | def atomic_write(path, text): |
| 501 | os.makedirs(os.path.dirname(path), exist_ok=True) |
| 502 | tmp = path + ".tmp" |
| 503 | with open(tmp, "w", encoding="utf-8") as f: |
| 504 | f.write(text) |
| 505 | os.replace(tmp, path) |
| 506 | |
| 507 | |
| 508 | # ---------------------------------------------------------------- reading the API data |
| 509 | |
| 510 | def rendered(field): |
| 511 | return field.get("rendered", "") if isinstance(field, dict) else (field or "") |
| 512 | |
| 513 | |
| 514 | def is_protected(item): |
| 515 | return bool((item.get("content") or {}).get("protected")) or "post-password-required" in (item.get("class_list") or []) |
| 516 | |
| 517 | |
| 518 | def safe_slug(slug, wp_id): |
| 519 | """A file name for a WordPress slug. WordPress sends non-ASCII slugs percent-encoded (caf%c3%a9): the file gets |
| 520 | the readable name (café). The slug itself stays as WordPress has it, because that is the URL.""" |
| 521 | slug = unicodedata.normalize("NFC", urllib.parse.unquote(str(slug or ""))) |
| 522 | slug = re.sub(r"[^\w.-]", "", slug).lstrip(".") |
| 523 | return slug or f"post-{wp_id}" |
| 524 | |
| 525 | |
| 526 | def link_path(link, wp): |
| 527 | """/path/ of a WordPress link (the host is dropped; a query is kept).""" |
| 528 | u = urllib.parse.urlsplit(link or "") |
| 529 | return (u.path or "/") + (f"?{u.query}" if u.query else "") |
| 530 | |
| 531 | |
| 532 | def read_posts(fetcher, notes): |
| 533 | """(public posts, skipped protected posts, other) from the API cache; each public post is a dict.""" |
| 534 | tags = {t["id"]: t for t in fetcher.collection("tags")} |
| 535 | posts, skipped, ignored = [], [], [] |
| 536 | for item in fetcher.collection("posts"): |
| 537 | title = title_text(rendered(item.get("title"))) |
| 538 | if item.get("status", "publish") != "publish": |
| 539 | ignored.append(f'{title} ({item.get("status")})') |
| 540 | elif is_protected(item): |
| 541 | skipped.append({"id": item["id"], "title": title}) |
| 542 | else: |
| 543 | body = rendered(item.get("content")) |
| 544 | posts.append({ |
| 545 | "id": item["id"], "title": title, "slug": str(item["slug"]), |
| 546 | "date": datetime.datetime.fromisoformat(item["date"]), |
| 547 | "date_gmt": item.get("date_gmt"), "html": body, |
| 548 | "path": link_path(item.get("link"), fetcher.wp), |
| 549 | "tags": [tags[t]["slug"] for t in item.get("tags", []) if t in tags], |
| 550 | "tag_names": {tags[t]["slug"]: html.unescape(tags[t]["name"]) for t in item.get("tags", []) if t in tags}, |
| 551 | "text": html_text(body)}) |
| 552 | posts[-1]["words"] = norm_words(posts[-1]["text"]) |
| 553 | return posts, skipped, ignored |
| 554 | |
| 555 | |
| 556 | def read_pages(fetcher): |
| 557 | pages, skipped = [], [] |
| 558 | for item in fetcher.collection("pages"): |
| 559 | title = title_text(rendered(item.get("title"))) |
| 560 | if item.get("status", "publish") != "publish": |
| 561 | continue |
| 562 | if is_protected(item): |
| 563 | skipped.append(title) |
| 564 | continue |
| 565 | pages.append({"id": item["id"], "title": title, "slug": str(item["slug"]), "html": rendered(item.get("content")), |
| 566 | "path": link_path(item.get("link"), fetcher.wp)}) |
| 567 | return pages, skipped |
| 568 | |
| 569 | |
| 570 | # ---------------------------------------------------------------- matching posts to source files |
| 571 | |
| 572 | def source_candidates(cfg): |
| 573 | """Every post file except what the importer wrote itself, with its words.""" |
| 574 | out = [] |
| 575 | for p in blog.scan_posts(cfg): |
| 576 | if p.rel.startswith("wordpress/"): |
| 577 | continue |
| 578 | body = blog.extract_body(p.text, cfg["body_section"], p.body_until, p.rel) |
| 579 | out.append({"rel": p.rel, "post": p, "words": md_words(body), "title_words": norm_words(p.title), |
| 580 | "date": p.date or datetime.datetime.min}) |
| 581 | return out |
| 582 | |
| 583 | |
| 584 | def match_posts(posts, files, threshold=MATCH_THRESHOLD): |
| 585 | """Assign source files to posts. Returns (assigned {post index: (score, file index)}, best {post index: |
| 586 | [(score, file index)] by score}, conflicts [text]). |
| 587 | |
| 588 | Candidates are ranked by score, then equal titles, then files dated on or before the publish date and nearer |
| 589 | to it. Each file is used once; a file or post with several candidates at or above the threshold is reported.""" |
| 590 | best, above = {}, [] |
| 591 | for i, post in enumerate(posts): |
| 592 | sm = difflib.SequenceMatcher(None, autojunk=False) |
| 593 | sm.set_seq2(post["words"]) |
| 594 | ranked = [] |
| 595 | for j, f in enumerate(files): |
| 596 | if not f["words"] or not post["words"]: |
| 597 | continue |
| 598 | sm.set_seq1(f["words"]) |
| 599 | if sm.real_quick_ratio() >= 0.5: |
| 600 | q = sm.quick_ratio() |
| 601 | if q >= 0.5: |
| 602 | ranked.append((q, j)) |
| 603 | ranked.sort(reverse=True) |
| 604 | scored = [] |
| 605 | for n, (q, j) in enumerate(ranked): |
| 606 | if q < threshold and n >= 3: |
| 607 | break |
| 608 | sm.set_seq1(files[j]["words"]) |
| 609 | scored.append((sm.ratio(), j)) |
| 610 | scored.sort(reverse=True) |
| 611 | best[i] = scored |
| 612 | above += [(s, i, j) for s, j in scored if s >= threshold] |
| 613 | |
| 614 | def key(t): |
| 615 | score, i, j = t |
| 616 | f, p = files[j], posts[i] |
| 617 | late = f["date"].date() > p["date"].date() |
| 618 | return (-round(score, 4), f["title_words"] != norm_words(p["title"]), late, |
| 619 | abs((f["date"].date() - p["date"].date()).days), f["rel"], p["path"]) |
| 620 | |
| 621 | assigned, used = {}, set() |
| 622 | for score, i, j in sorted(above, key=key): |
| 623 | if i not in assigned and j not in used: |
| 624 | assigned[i] = (score, j) |
| 625 | used.add(j) |
| 626 | conflicts = [] |
| 627 | claims = collections.defaultdict(list) |
| 628 | for score, i, j in above: |
| 629 | claims[j].append((score, i)) |
| 630 | for j in sorted(claims, key=lambda j: files[j]["rel"]): |
| 631 | if len(claims[j]) > 1: |
| 632 | who = ", ".join(f'"{posts[i]["title"]}" ({s:.3f})' for s, i in sorted(claims[j], reverse=True)) |
| 633 | winner = next((posts[i]["title"] for i, (s, jj) in assigned.items() if jj == j), None) |
| 634 | conflicts.append(f'{files[j]["rel"]} is claimed by {who}; given to "{winner}"') |
| 635 | for i in sorted(best): |
| 636 | cands = [(s, j) for s, j in best[i] if s >= threshold] |
| 637 | if len(cands) > 1: |
| 638 | conflicts.append(f'"{posts[i]["title"]}" matches several files: ' |
| 639 | + ", ".join(f'{files[j]["rel"]} ({s:.3f})' for s, j in cands)) |
| 640 | return assigned, best, conflicts |
| 641 | |
| 642 | |
| 643 | # ---------------------------------------------------------------- planned changes |
| 644 | |
| 645 | class Plan: |
| 646 | """Everything the import would write: files, comments, config entries.""" |
| 647 | |
| 648 | def __init__(self): |
| 649 | self.files = {} # path relative to source -> text |
| 650 | self.post_entries = {} # relative path -> config `posts` entry |
| 651 | self.comments = {} # post path -> [comment] |
| 652 | self.tags = {} # slug -> name |
| 653 | self.wp_ids = {} # str id -> path |
| 654 | self.redirects = {} # old path -> new path |
| 655 | self.lines = collections.defaultdict(list) # report sections |
| 656 | |
| 657 | |
| 658 | def date_text(when): |
| 659 | return when.strftime("%Y-%m-%dT%H:%M:%S") |
| 660 | |
| 661 | |
| 662 | def post_markdown(post, notes): |
| 663 | body = html_to_markdown(post["html"], notes) |
| 664 | return f"# {post['title']}\n\n## Writing\n\n{body}\n" |
| 665 | |
| 666 | |
| 667 | def page_markdown(page, notes): |
| 668 | """A page is published only with `published: true`; its slug is set when the file name cannot say it.""" |
| 669 | front = ["published: true"] |
| 670 | if safe_slug(page["slug"], page["id"]) != page["slug"]: |
| 671 | front.append(f"slug: {page['slug']}") |
| 672 | return f"---\n{chr(10).join(front)}\n---\n# {page['title']}\n\n{html_to_markdown(page['html'], notes)}\n" |
| 673 | |
| 674 | |
| 675 | def build_plan(cfg, posts, pages, comments, assigned, files, notes, home): |
| 676 | plan = Plan() |
| 677 | src = cfg["_source"] |
| 678 | existing_titles = {f["rel"]: f["post"].title for f in files} |
| 679 | for i, post in enumerate(posts): |
| 680 | entry = {"published": True, "date": date_text(post["date"]), "slug": post["slug"], "wp_id": post["id"]} |
| 681 | if post["tags"]: |
| 682 | entry["tags"] = list(post["tags"]) |
| 683 | if i in assigned: |
| 684 | _, j = assigned[i] |
| 685 | rel = files[j]["rel"] |
| 686 | if norm_words(existing_titles[rel]) != norm_words(post["title"]): |
| 687 | entry["title"] = post["title"] |
| 688 | else: |
| 689 | slug = safe_slug(post["slug"], post["id"]) |
| 690 | entry["slug"] = post["slug"] |
| 691 | rel = f"wordpress/{post['date'].strftime('%Y-%m-%d')}-{slug}.md" |
| 692 | plan.files[rel] = post_markdown(post, notes) |
| 693 | plan.post_entries[rel] = entry |
| 694 | post["rel"] = rel |
| 695 | plan.wp_ids[str(post["id"])] = post["path"] |
| 696 | for slug, name in post["tag_names"].items(): |
| 697 | plan.tags[slug] = name |
| 698 | post_paths = {str(p["id"]): p["path"] for p in posts} |
| 699 | for page in pages: |
| 700 | if page["slug"] == "blog": |
| 701 | plan.lines["notes"].append('page "blog" not imported: /blog/ is generated') |
| 702 | continue |
| 703 | if page["slug"] == home: |
| 704 | page["path"] = "/" |
| 705 | plan.files[f"{blog.pages_dir(cfg) or 'pages'}/{safe_slug(page['slug'], page['id'])}.md"] = page_markdown(page, notes) |
| 706 | plan.wp_ids[str(page["id"])] = page["path"] |
| 707 | for c in comments: |
| 708 | if c.get("status", "approved") != "approved" or c.get("type", "comment") != "comment": |
| 709 | continue |
| 710 | path = post_paths.get(str(c.get("post"))) |
| 711 | if path is None: |
| 712 | continue # a protected or unknown post (counted by the caller) |
| 713 | plan.comments.setdefault(path, []).append({ |
| 714 | "id": int(c["id"]), "parent": int(c.get("parent") or 0), |
| 715 | "author": html.unescape(c.get("author_name") or ""), |
| 716 | "date": c.get("date", ""), "text": html_plain(rendered(c.get("content")))}) |
| 717 | for lst in plan.comments.values(): |
| 718 | lst.sort(key=lambda c: (c["date"], c["id"])) |
| 719 | return plan |
| 720 | |
| 721 | |
| 722 | # ---------------------------------------------------------------- links and redirects |
| 723 | |
| 724 | LINK_RE = re.compile(r"""(?:href|src)\s*=\s*["']([^"']+)["']|\]\(\s*<?([^)\s>]+)|(https?://[^\s<>"')\]]+)""") |
| 725 | |
| 726 | |
| 727 | def internal_links(text, hosts): |
| 728 | """Internal links (as /path or /path?query, without #fragment) found in HTML or Markdown.""" |
| 729 | found = set() |
| 730 | for m in LINK_RE.finditer(text or ""): |
| 731 | url = html.unescape(next(g for g in m.groups() if g)).strip().rstrip(".,;:") |
| 732 | for h in hosts: |
| 733 | if url.startswith(h) and url[len(h):len(h) + 1] in ("", "/", "?", "#"): |
| 734 | url = url[len(h):] or "/" |
| 735 | break |
| 736 | if not url.startswith("/") or url.startswith("//"): |
| 737 | continue |
| 738 | found.add(re.sub(r"#.*$", "", url) or "/") |
| 739 | return found |
| 740 | |
| 741 | |
| 742 | def norm_path(path): |
| 743 | p = urllib.parse.unquote(path.split("?")[0].split("#")[0]) |
| 744 | return p if p.endswith("/") or "." in p.rsplit("/", 1)[-1] else p + "/" |
| 745 | |
| 746 | |
| 747 | def generated_paths(cfg, posts, pages, plan, home): |
| 748 | paths = {"/", "/blog/", "/feed/", "/404.html"} | {p["path"] for p in posts} |
| 749 | paths |= {pg["path"] for pg in pages if pg["slug"] != "blog"} |
| 750 | paths |= {f"/tag/{blog.tag_slug(cfg, t)}/" for t in plan.tags} |
| 751 | if home: |
| 752 | paths.add(f"/{home}/") # also redirected to / |
| 753 | return {norm_path(p) for p in paths} |
| 754 | |
| 755 | |
| 756 | def classify_links(cfg, links, generated, media_urls, wp_ids, redirect_keys, media_dir): |
| 757 | """Split links into (fine, missing media, to probe, odd) lists.""" |
| 758 | fine, missing, probe, odd = [], [], [], [] |
| 759 | for link in sorted(links): |
| 760 | path, _, query = link.partition("?") |
| 761 | if query: |
| 762 | q = urllib.parse.parse_qs(query) |
| 763 | ident = next((q[k][0] for k in ("p", "page_id") if k in q), None) |
| 764 | if path == "/" and ident in wp_ids: |
| 765 | fine.append(link) |
| 766 | else: |
| 767 | odd.append(link) |
| 768 | continue |
| 769 | norm = norm_path(path) |
| 770 | if norm in generated or norm in redirect_keys or path in redirect_keys: |
| 771 | fine.append(link) |
| 772 | elif norm.startswith("/wp-content/uploads/"): |
| 773 | local = os.path.join(media_dir, urllib.parse.unquote(path.lstrip("/"))) |
| 774 | (fine if path in media_urls or os.path.exists(local) else missing).append(link) |
| 775 | elif norm.startswith("/wp-content/") or norm.startswith("/wp-"): |
| 776 | odd.append(link) |
| 777 | else: |
| 778 | probe.append(link) |
| 779 | return fine, missing, probe, odd |
| 780 | |
| 781 | |
| 782 | def probe_links(fetcher, wp, probe, generated, cache_file, refresh): |
| 783 | """HEAD every link; returns ({old: new} for 301s, [report lines]).""" |
| 784 | cache = {} |
| 785 | if os.path.exists(cache_file) and not refresh: |
| 786 | with open(cache_file, encoding="utf-8") as f: |
| 787 | cache = json.load(f) |
| 788 | redirects, lines = {}, [] |
| 789 | for link in probe: |
| 790 | if blog.htaccess_unsafe(link): |
| 791 | lines.append(f"{link!r}: skipped (a space or control character cannot go into .htaccess)") |
| 792 | continue |
| 793 | if link not in cache: |
| 794 | cache[link] = list(fetcher.head(wp.rstrip("/") + link)) |
| 795 | status, location = cache[link] |
| 796 | if status in (301, 308) and location: |
| 797 | target = urllib.parse.urlsplit(urllib.parse.urljoin(wp.rstrip("/") + link, location)) |
| 798 | if target.netloc == urllib.parse.urlsplit(wp).netloc: |
| 799 | new = (target.path or "/") + (f"?{target.query}" if target.query else "") |
| 800 | if blog.htaccess_unsafe(new): |
| 801 | lines.append(f"{link} -> {new!r}: skipped (a space or control character cannot go into .htaccess)") |
| 802 | continue |
| 803 | if norm_path(new) == norm_path(link): |
| 804 | lines.append(f"{link}: only adds a trailing slash or a query; handled by the generator") |
| 805 | continue |
| 806 | redirects[urllib.parse.quote(link, safe="/%:@!$&'*+,;=-._~")] = new |
| 807 | if norm_path(new) not in generated: |
| 808 | lines.append(f"{link} -> {new}: the target is not a generated page") |
| 809 | continue |
| 810 | lines.append(f"{link} -> {location}: leaves the site") |
| 811 | elif status == 200: |
| 812 | lines.append(f"{link}: exists on the old site but not on the new one") |
| 813 | else: |
| 814 | lines.append(f"{link}: HTTP {status}" + (f" ({location})" if location and status in (302, 307) else "")) |
| 815 | atomic_write(cache_file, json.dumps(cache, ensure_ascii=False, indent=1, sort_keys=True) + "\n") |
| 816 | return redirects, lines |
| 817 | |
| 818 | |
| 819 | # ---------------------------------------------------------------- applying |
| 820 | |
| 821 | def inside(root, path): |
| 822 | root, path = os.path.realpath(root), os.path.realpath(path) |
| 823 | return path == root or path.startswith(root + os.sep) |
| 824 | |
| 825 | |
| 826 | def write_file(src, rel, text, dry, force): |
| 827 | """'create', 'same', 'keep' (exists and differs: left alone) or 'overwrite'. Writes unless `dry`.""" |
| 828 | path = os.path.join(src, rel) |
| 829 | if not inside(src, os.path.dirname(path)): |
| 830 | raise WpImportError(f"refusing to write outside the source folder: {rel}") |
| 831 | if os.path.exists(path): |
| 832 | with open(path, encoding="utf-8") as f: |
| 833 | if f.read() == text: |
| 834 | return "same" |
| 835 | if not force: |
| 836 | return "keep" |
| 837 | action = "overwrite" |
| 838 | else: |
| 839 | action = "create" |
| 840 | if not dry: |
| 841 | atomic_write(path, text) |
| 842 | return action |
| 843 | |
| 844 | |
| 845 | def merge_dict(target, new, force, changes, label): |
| 846 | """Adds the missing keys of `new` to `target`; an existing value is replaced only with `force`.""" |
| 847 | for key, value in new.items(): |
| 848 | if key not in target: |
| 849 | target[key] = value |
| 850 | changes.append(f"{label}: add {key}") |
| 851 | elif target[key] != value: |
| 852 | if force: |
| 853 | target[key] = value |
| 854 | changes.append(f"{label}: replace {key}") |
| 855 | else: |
| 856 | changes.append(f"{label}: kept existing {key} (differs; use --force to replace)") |
| 857 | return target |
| 858 | |
| 859 | |
| 860 | def merge_config(raw, cfg, plan, opts, wp): |
| 861 | """(new raw config dict, [change lines]). Never overwrites without opts['force'].""" |
| 862 | force, changes = opts["force"], [] |
| 863 | new = json.loads(json.dumps(raw)) |
| 864 | include = list(new.get("include") or cfg["include"]) |
| 865 | if not any(p.startswith("**") or p == "wordpress/*.md" for p in include): |
| 866 | include.append("wordpress/*.md") |
| 867 | if include != new.get("include"): |
| 868 | new["include"] = include |
| 869 | changes.append("include: add wordpress/*.md") |
| 870 | has_pages = any(not rel.startswith("wordpress/") for rel in plan.files) |
| 871 | for key, value, wanted in (("pages_dir", "pages", has_pages), |
| 872 | ("media", "media", not opts["no_media"]), ("comments", "comments.json", bool(plan.comments)), |
| 873 | ("timezone", opts["timezone"], True), ("home", opts.get("home"), bool(opts.get("home")))): |
| 874 | if wanted and value and not new.get(key): |
| 875 | new[key] = value |
| 876 | changes.append(f"{key}: set to {value}") |
| 877 | hosts = list(new.get("rewrite_hosts") or []) |
| 878 | base = wp.rstrip("/") |
| 879 | for variant in (base, base.replace("https://", "http://", 1)): |
| 880 | if variant not in hosts: |
| 881 | hosts.append(variant) |
| 882 | changes.append(f"rewrite_hosts: add {variant}") |
| 883 | if hosts: |
| 884 | new["rewrite_hosts"] = hosts |
| 885 | for key, value in (("tags", plan.tags), ("wp_ids", plan.wp_ids), ("redirects", plan.redirects)): |
| 886 | if value: |
| 887 | new[key] = merge_dict(dict(new.get(key) or {}), dict(sorted(value.items())), force, changes, key) |
| 888 | posts = dict(new.get("posts") or {}) |
| 889 | for rel, entry in sorted(plan.post_entries.items()): |
| 890 | existing = dict(posts.get(rel) or {}) |
| 891 | posts[rel] = merge_dict(existing, entry, force, changes, f"posts[{rel}]") |
| 892 | new["posts"] = posts |
| 893 | return new, changes |
| 894 | |
| 895 | |
| 896 | def merge_comments(base, plan, comments_name, dry, force): |
| 897 | """`base` is the config file's folder: `comments` is relative to it, as the generator reads it.""" |
| 898 | path = os.path.join(base, comments_name) |
| 899 | existing = {} |
| 900 | if os.path.exists(path): |
| 901 | with open(path, encoding="utf-8") as f: |
| 902 | existing = json.load(f) |
| 903 | merged = {k: list(v) for k, v in existing.items()} |
| 904 | for key, lst in plan.comments.items(): |
| 905 | if force or key not in merged: |
| 906 | merged[key] = lst |
| 907 | else: |
| 908 | have = {c.get("id") for c in merged[key]} |
| 909 | merged[key] += [c for c in lst if c["id"] not in have] |
| 910 | text = json.dumps(merged, ensure_ascii=False, indent=1, sort_keys=True) + "\n" |
| 911 | same = False |
| 912 | if os.path.exists(path): |
| 913 | with open(path, encoding="utf-8") as f: |
| 914 | same = f.read() == text |
| 915 | if not same and not dry: |
| 916 | atomic_write(path, text) |
| 917 | return "same" if same else ("update" if existing else "create") |
| 918 | |
| 919 | |
| 920 | def media_patterns(): |
| 921 | pats = ["--include=*/"] |
| 922 | for ext in sorted(blog.MEDIA_EXTENSIONS): |
| 923 | pats.append("--include=*" + "".join(f"[{c.lower()}{c.upper()}]" if c.isalpha() else c for c in ext)) |
| 924 | return pats + ["--exclude=*"] |
| 925 | |
| 926 | |
| 927 | def sync_media(src, media_folder, source_spec, run=subprocess.run): |
| 928 | """Read-only rsync of the old uploads folder into <source>/<media>/wp-content/uploads/ (media types only).""" |
| 929 | dest = os.path.join(src, media_folder, "wp-content", "uploads") |
| 930 | if not inside(src, dest): |
| 931 | raise WpImportError("the media folder must be inside the source folder") |
| 932 | if not shutil.which("rsync"): |
| 933 | raise WpImportError("rsync is not installed") |
| 934 | os.makedirs(dest, exist_ok=True) |
| 935 | cmd = ["rsync", "-rt", "--prune-empty-dirs", *media_patterns(), source_spec.rstrip("/") + "/", dest + "/"] |
| 936 | result = run(cmd, capture_output=True, text=True) |
| 937 | if result.returncode: |
| 938 | raise WpImportError(f"rsync failed ({result.returncode}): {(result.stderr or '').strip()[:300]}") |
| 939 | return dest |
| 940 | |
| 941 | |
| 942 | # ---------------------------------------------------------------- the import |
| 943 | |
| 944 | def run_import(config_path, wp, dry_run=False, force=False, refresh=False, probe=False, cache=None, no_media=False, |
| 945 | media_from=None, timezone="America/Los_Angeles", home=None, fetcher=None, rsync=subprocess.run, |
| 946 | say=print): |
| 947 | """Runs the whole import; returns the report text. The report file wordpress/_report.txt holds only what |
| 948 | does not change between two runs (so a second run leaves it as it was); what this run did is added to the |
| 949 | returned text.""" |
| 950 | cfg = blog.load_config(config_path) |
| 951 | src = cfg["_source"] |
| 952 | wp = wp.rstrip("/") |
| 953 | wpdir = os.path.join(src, "wordpress") |
| 954 | api = os.path.abspath(cache) if cache else os.path.join(wpdir, "_api") |
| 955 | fetcher = fetcher or Fetcher(wp, api, refresh=refresh, log=say) |
| 956 | notes = [] |
| 957 | try: |
| 958 | zoneinfo.ZoneInfo(timezone) |
| 959 | except (zoneinfo.ZoneInfoNotFoundError, ValueError, OSError): |
| 960 | raise WpImportError(f"--timezone {timezone!r} is not an IANA time zone name") |
| 961 | |
| 962 | posts, skipped, ignored = read_posts(fetcher, notes) |
| 963 | pages, skipped_pages = read_pages(fetcher) |
| 964 | comments = fetcher.collection("comments") |
| 965 | try: |
| 966 | media = fetcher.collection("media") |
| 967 | except WpImportError as e: |
| 968 | say(f"media list not available: {e}") |
| 969 | media = [] |
| 970 | media_urls = set() |
| 971 | for m in media: |
| 972 | for url in [m.get("source_url")] + [s.get("source_url") for s in ((m.get("media_details") or {}).get("sizes") or {}).values()]: |
| 973 | if url: |
| 974 | media_urls.add(link_path(url, wp)) |
| 975 | |
| 976 | files = source_candidates(cfg) |
| 977 | assigned, best, conflicts = match_posts(posts, files) |
| 978 | if home is None and not cfg["home"] and any(p["slug"] == "highlights" for p in pages): |
| 979 | home = "highlights" |
| 980 | home = cfg["home"] or home |
| 981 | plan = build_plan(cfg, posts, pages, comments, assigned, files, notes, home) |
| 982 | |
| 983 | protected_ids = {str(s["id"]) for s in skipped} |
| 984 | skipped_comments = sum(1 for c in comments if str(c.get("post")) in protected_ids) |
| 985 | kept_comments = sum(len(v) for v in plan.comments.values()) |
| 986 | other_comments = len(comments) - kept_comments - skipped_comments |
| 987 | |
| 988 | # Links the new site will not generate. |
| 989 | hosts = [wp, wp.replace("https://", "http://", 1)] + [h.rstrip("/") for h in cfg["rewrite_hosts"]] |
| 990 | texts = [p["html"] for p in posts] + [pg["html"] for pg in pages] |
| 991 | texts += [blog.extract_body(files[j]["post"].text, cfg["body_section"], files[j]["post"].body_until, files[j]["rel"]) for _, j in assigned.values()] |
| 992 | links = set() |
| 993 | for t in texts: |
| 994 | links |= internal_links(t, hosts) |
| 995 | generated = generated_paths(cfg, posts, pages, plan, home) |
| 996 | media_dir = os.path.join(src, cfg["media"] or "media") |
| 997 | fine, missing_media, to_probe, odd = classify_links(cfg, links, generated, media_urls, plan.wp_ids, |
| 998 | set(blog.redirect_lookup(cfg)), media_dir) |
| 999 | probe_lines = [] |
| 1000 | if probe and to_probe: |
| 1001 | plan.redirects, probe_lines = probe_links(fetcher, wp, to_probe, generated, os.path.join(api, "probes.json"), refresh) |
| 1002 | |
| 1003 | # Apply (or only plan, in a dry run). |
| 1004 | results = {} |
| 1005 | for rel, text in sorted(plan.files.items()): |
| 1006 | results[rel] = write_file(src, rel, text, dry_run, force) |
| 1007 | config_changes = [] |
| 1008 | with open(os.path.abspath(config_path), encoding="utf-8") as f: |
| 1009 | raw = json.load(f) |
| 1010 | opts = {"force": force, "no_media": no_media, "timezone": timezone, "home": home} |
| 1011 | new_raw, config_changes = merge_config(raw, cfg, plan, opts, wp) |
| 1012 | comments_name = new_raw.get("comments") or "comments.json" |
| 1013 | comments_result = merge_comments(cfg["_dir"], plan, comments_name, dry_run, force) if plan.comments else "none" |
| 1014 | media_result = "skipped (--no-media)" if no_media else "not run (dry run)" |
| 1015 | if not dry_run: |
| 1016 | if new_raw != raw: |
| 1017 | atomic_write(os.path.abspath(config_path), json.dumps(new_raw, ensure_ascii=False, indent=2) + "\n") |
| 1018 | blog.load_config(config_path) # the merged config must still be valid |
| 1019 | if not no_media: |
| 1020 | source_spec = media_from or f"dh:{urllib.parse.urlsplit(wp).hostname}/wp-content/uploads/" |
| 1021 | try: |
| 1022 | media_result = "copied from " + source_spec + " into " + os.path.relpath( |
| 1023 | sync_media(src, new_raw.get("media") or "media", source_spec, rsync), src) |
| 1024 | except WpImportError as e: |
| 1025 | media_result = f"FAILED: {e}" |
| 1026 | |
| 1027 | report, actions = make_report(cfg, wp, dry_run, posts, skipped, ignored, pages, skipped_pages, comments, kept_comments, |
| 1028 | skipped_comments, other_comments, media, files, assigned, best, conflicts, plan, results, |
| 1029 | config_changes, comments_result, media_result, fine, missing_media, to_probe, odd, probe, |
| 1030 | probe_lines, notes, timezone) |
| 1031 | atomic_write(os.path.join(wpdir, "_report.txt"), report) |
| 1032 | return report + actions |
| 1033 | |
| 1034 | |
| 1035 | def section(lines, title, items): |
| 1036 | lines.append(f"\n== {title} ==") |
| 1037 | lines += [f" {i}" for i in items] if items else [" (none)"] |
| 1038 | |
| 1039 | |
| 1040 | def make_report(cfg, wp, dry_run, posts, skipped, ignored, pages, skipped_pages, comments, kept_comments, skipped_comments, |
| 1041 | other_comments, media, files, assigned, best, conflicts, plan, results, config_changes, comments_result, |
| 1042 | media_result, fine, missing_media, to_probe, odd, probe, probe_lines, notes, timezone): |
| 1043 | """(report file text, what-this-run-did text).""" |
| 1044 | lines = ["WordPress import report", f"site: {wp}", |
| 1045 | "", f"public posts: {len(posts)} password-protected (skipped): {len(skipped)} other status (ignored): {len(ignored)}", |
| 1046 | f"pages: {len(pages)} comments: {len(comments)} (kept {kept_comments}, on skipped posts {skipped_comments}, other {other_comments})", |
| 1047 | f"media library entries: {len(media)} source files considered: {len(files)}"] |
| 1048 | section(lines, "Skipped: password-protected posts (their comments are skipped too)", [f'"{s["title"]}"' for s in skipped] |
| 1049 | + [f'page "{t}"' for t in skipped_pages] + [f'{t} (ignored)' for t in ignored]) |
| 1050 | matched = sorted(((posts[i]["path"], score, files[j]["rel"], posts[i]["title"]) for i, (score, j) in assigned.items())) |
| 1051 | section(lines, f"Matched to a source file (score >= {MATCH_THRESHOLD}): {len(matched)}", |
| 1052 | [f"{score:.4f} {rel} <- \"{title}\" {path}" for path, score, rel, title in matched]) |
| 1053 | unmatched = [] |
| 1054 | for i, post in enumerate(posts): |
| 1055 | if i not in assigned: |
| 1056 | cand = best.get(i) or [] |
| 1057 | hint = f"best candidate {files[cand[0][1]]['rel']} at {cand[0][0]:.3f}" if cand else "no candidate" |
| 1058 | unmatched.append(f'"{post["title"]}" ({post["date"].date()}) -> {post["rel"]} [{hint}]') |
| 1059 | section(lines, f"Not matched (written from the WordPress HTML): {len(unmatched)}", unmatched) |
| 1060 | section(lines, "Conflicts (double claims)", conflicts) |
| 1061 | section(lines, "Files kept because they differ from what the importer would write (--force overwrites)", |
| 1062 | [rel for rel, action in sorted(results.items()) if action == "keep"] + [c for c in config_changes if "kept existing" in c]) |
| 1063 | section(lines, "Titles that differ from the source file (the config gets `title`)", |
| 1064 | [f'{rel}: "{e["title"]}"' for rel, e in sorted(plan.post_entries.items()) if "title" in e and not rel.startswith("wordpress/")]) |
| 1065 | mismatch = [p for p in posts if p.get("date_gmt") and not tz_agrees(p, timezone)] |
| 1066 | section(lines, f"Dates that disagree with timezone {timezone} (check `timezone` in the config)", |
| 1067 | [f'"{p["title"]}" {p["date"]} vs gmt {p["date_gmt"]}' for p in mismatch[:20]]) |
| 1068 | links_info = [f"{len(fine)} internal link(s) fine (generated pages, redirects, media)"] |
| 1069 | links_info += [f"missing media: {l}" for l in missing_media] |
| 1070 | links_info += [f"unusual link (not handled): {l}" for l in odd] |
| 1071 | if probe: |
| 1072 | links_info += [f"{len(to_probe)} link(s) probed"] + probe_lines |
| 1073 | links_info += [f"redirect {a} -> {b}" for a, b in sorted(plan.redirects.items())] |
| 1074 | else: |
| 1075 | links_info += [f"unchecked (use --probe-links): {l}" for l in to_probe] |
| 1076 | section(lines, "Internal links", links_info) |
| 1077 | section(lines, "Conversion notes (embeds dropped etc.)", sorted(set(notes))) |
| 1078 | done = [f"\n== This run: {'DRY RUN (nothing written except _api/ and the report)' if dry_run else 'import'} =="] |
| 1079 | done.append(f" files: {dict(collections.Counter(results.values())) or 'nothing to write'}") |
| 1080 | done += [f" {action:9} {rel}" for rel, action in sorted(results.items()) if action != "same"] |
| 1081 | done += [f" comments file: {comments_result}", f" media: {media_result}"] |
| 1082 | done += [f" config: {c}" for c in config_changes] or [" config: no changes"] |
| 1083 | return "\n".join(lines) + "\n", "\n".join(done) + "\n" |
| 1084 | |
| 1085 | |
| 1086 | def tz_agrees(post, timezone): |
| 1087 | try: |
| 1088 | gmt = datetime.datetime.fromisoformat(post["date_gmt"]).replace(tzinfo=datetime.timezone.utc) |
| 1089 | local = post["date"].replace(tzinfo=zoneinfo.ZoneInfo(timezone)) |
| 1090 | return abs((gmt - local).total_seconds()) < 1 |
| 1091 | except (ValueError, KeyError): |
| 1092 | return True |
| 1093 | |
| 1094 | |
| 1095 | # ---------------------------------------------------------------- check |
| 1096 | |
| 1097 | BODY_RE = re.compile(r'<div class="post-body">(.*?)</div></article>', re.S) |
| 1098 | |
| 1099 | |
| 1100 | def output_has(out, path): |
| 1101 | """True when the built site has a page or file at this URL path (a feed folder counts).""" |
| 1102 | p = urllib.parse.unquote(path.split("?")[0].split("#")[0]).lstrip("/") |
| 1103 | full = os.path.join(out, p) |
| 1104 | if not inside(out, full): |
| 1105 | return False |
| 1106 | if os.path.isfile(full): |
| 1107 | return True |
| 1108 | return any(os.path.isfile(os.path.join(full, name)) for name in ("index.html", "index.xml")) |
| 1109 | |
| 1110 | |
| 1111 | def run_check(config_path, out=None, cache=None, say=print): |
| 1112 | """Compares the built site with the cached WordPress data. Returns (report text, number of problems).""" |
| 1113 | cfg = blog.load_config(config_path) |
| 1114 | src = cfg["_source"] |
| 1115 | api = os.path.abspath(cache) if cache else os.path.join(src, "wordpress", "_api") |
| 1116 | fetcher = Fetcher("http://invalid.invalid", api) |
| 1117 | for name in ("posts", "pages", "tags"): |
| 1118 | if fetcher.cached(name) is None: |
| 1119 | raise WpImportError(f"no cached WordPress data in {api} (run fw blog import-wp --dry-run first)") |
| 1120 | tmp = None |
| 1121 | if out is None: |
| 1122 | tmp = tempfile.mkdtemp(prefix="fw-blog-check-") |
| 1123 | out = tmp |
| 1124 | blog.build(cfg, out) |
| 1125 | try: |
| 1126 | return check_site(cfg, out, fetcher) |
| 1127 | finally: |
| 1128 | if tmp: |
| 1129 | shutil.rmtree(tmp, ignore_errors=True) |
| 1130 | |
| 1131 | |
| 1132 | def check_site(cfg, out, fetcher): |
| 1133 | notes = [] |
| 1134 | posts, skipped, _ = read_posts(fetcher, notes) |
| 1135 | pages, _ = read_pages(fetcher) |
| 1136 | problems, lines = 0, ["Blog check", f"output: {out}", f"posts compared: {len(posts)} (line_breaks = {cfg['line_breaks']})"] |
| 1137 | low, brs, missing = [], [], [] |
| 1138 | for p in posts: |
| 1139 | page = os.path.join(out, urllib.parse.unquote(p["path"]).strip("/"), "index.html") |
| 1140 | if not os.path.isfile(page): |
| 1141 | missing.append(f'post {p["path"]} "{p["title"]}"') |
| 1142 | continue |
| 1143 | with open(page, encoding="utf-8") as f: |
| 1144 | m = BODY_RE.search(f.read()) |
| 1145 | built = m.group(1) if m else "" |
| 1146 | score = similarity(p["words"], norm_words(blog.plain_text(built))) |
| 1147 | if score < CHECK_THRESHOLD: |
| 1148 | low.append((score, p["path"], p["title"])) |
| 1149 | wp_br, built_br = html_br_count(p["html"]), len(re.findall(r"<br\b", built)) |
| 1150 | if wp_br != built_br: |
| 1151 | brs.append(f'{p["path"]} WordPress {wp_br}, built {built_br} "{p["title"]}"') |
| 1152 | for p in pages: |
| 1153 | want = "/" if p["path"] in ("/", f'/{cfg["home"]}/') else p["path"] |
| 1154 | if p["slug"] != "blog" and not output_has(out, want): |
| 1155 | missing.append(f'page {p["path"]} "{p["title"]}"') |
| 1156 | tag_links = {} |
| 1157 | for t in fetcher.cached("tags") or []: |
| 1158 | tag_links[t["slug"]] = link_path(t.get("link"), fetcher.wp) |
| 1159 | used = {t for p in posts for t in p["tags"]} |
| 1160 | for slug in sorted(used): |
| 1161 | if not output_has(out, tag_links.get(slug, f"/tag/{slug}/")): |
| 1162 | missing.append(f"tag {tag_links.get(slug, f'/tag/{slug}/')}") |
| 1163 | bad_targets = [] |
| 1164 | targets = list(cfg["redirects"].items()) + [(f"?p={k}", v) for k, v in cfg["wp_ids"].items()] |
| 1165 | for old, new in sorted(targets): |
| 1166 | if not (re.match(r"[a-z]+://", new) or output_has(out, new)): |
| 1167 | bad_targets.append(f"{old} -> {new}") |
| 1168 | section(lines, f"Posts below {CHECK_THRESHOLD} similarity to the WordPress text: {len(low)}", |
| 1169 | [f'{s:.3f} {path} "{title}"' for s, path, title in sorted(low)]) |
| 1170 | section(lines, f"<br> counts that differ: {len(brs)}", brs) |
| 1171 | section(lines, f"Old URLs missing from the output: {len(missing)}", missing) |
| 1172 | section(lines, f"Redirect targets missing from the output: {len(bad_targets)}", bad_targets) |
| 1173 | problems = len(low) + len(brs) + len(missing) + len(bad_targets) |
| 1174 | lines.append(f"\n{'OK' if not problems else str(problems) + ' problem(s)'}") |
| 1175 | return "\n".join(lines) + "\n", problems |
| 1176 | |
| 1177 | |
| 1178 | # ---------------------------------------------------------------- command line |
| 1179 | |
| 1180 | def main(argv): |
| 1181 | ap = argparse.ArgumentParser(prog="wp_import.py", description=__doc__.split("\n")[0]) |
| 1182 | sub = ap.add_subparsers(dest="cmd", required=True) |
| 1183 | i = sub.add_parser("import") |
| 1184 | i.add_argument("config") |
| 1185 | i.add_argument("--wp", required=True, help="the WordPress site, e.g. https://blog.example.org") |
| 1186 | i.add_argument("--dry-run", action="store_true") |
| 1187 | i.add_argument("--force", action="store_true", help="overwrite existing files and config entries") |
| 1188 | i.add_argument("--refresh", action="store_true", help="fetch the API data again instead of using the cache") |
| 1189 | i.add_argument("--probe-links", action="store_true", help="HEAD internal links the new site lacks; record 301 targets") |
| 1190 | i.add_argument("--cache", help="folder for the API JSON (default <source>/wordpress/_api)") |
| 1191 | i.add_argument("--no-media", action="store_true") |
| 1192 | i.add_argument("--media-from", help="rsync source of the old uploads (default dh:<host>/wp-content/uploads/)") |
| 1193 | i.add_argument("--timezone", default="America/Los_Angeles", help="written to the config when it has none") |
| 1194 | i.add_argument("--home", help="slug of the WordPress page that is the front page (default: highlights, if present)") |
| 1195 | c = sub.add_parser("check") |
| 1196 | c.add_argument("config") |
| 1197 | c.add_argument("--out", help="an already built site (default: build into a temporary folder)") |
| 1198 | c.add_argument("--cache") |
| 1199 | args = ap.parse_args(argv[1:]) |
| 1200 | try: |
| 1201 | if args.cmd == "import": |
| 1202 | report = run_import(args.config, args.wp, dry_run=args.dry_run, force=args.force, refresh=args.refresh, |
| 1203 | probe=args.probe_links, cache=args.cache, no_media=args.no_media, |
| 1204 | media_from=args.media_from, timezone=args.timezone, home=args.home, |
| 1205 | say=lambda s: print(s, file=sys.stderr)) |
| 1206 | print(report) |
| 1207 | return 1 if "FAILED" in report else 0 |
| 1208 | report, problems = run_check(args.config, args.out, args.cache) |
| 1209 | print(report) |
| 1210 | return 1 if problems else 0 |
| 1211 | except blog.BlogError as e: |
| 1212 | print(f"wp_import.py: {e}", file=sys.stderr) |
| 1213 | return 1 |
| 1214 | |
| 1215 | |
| 1216 | if __name__ == "__main__": |
| 1217 | sys.exit(main(sys.argv)) |