fuzzyworld

fuzzyworld-tools / files / tests / test_blog.py

test_blog.py

42.9 KB · history · blame

1"""Tests for server/blog.py and server/atom.py. Run: python3 -m unittest discover tests
2
3Every source file here is made up, in a temporary folder.
4"""
5import datetime
6import json
7import os
8import sys
9import tempfile
10import unittest
11import xml.etree.ElementTree as ET
12
13sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "server"))
14import atom # noqa: E402
15import blog # noqa: E402
16import gen # noqa: E402
17
18needs_markdown = unittest.skipUnless(blog.markdown, "Python-Markdown is not installed (see FW_PYLIB)")
19
20POST = """# First Post
21
22Created by template, on 2018-03-02
23
24## Writing
25
26Line one
27line two
28
29Second paragraph with [a link](https://old.example.org/2018/03/02/first-post/).
30
31## Spelling
32
33SECRET-SPELLING-WORDS
34
35## Removed writing
36
37SECRET-REMOVED-TEXT
38"""
39
40
41class Tree:
42 """A made-up blog folder: write files, then load_config()."""
43
44 def __init__(self, test, config=None):
45 self.dir = tempfile.mkdtemp()
46 test.addCleanup(self.cleanup)
47 self.src = os.path.join(self.dir, "src")
48 os.makedirs(self.src)
49 self.config = {"title": "Test Blog", "base_url": "https://blog.example.org", "source": "src",
50 "author": "A. Writer", "copyright": "© {years} A. Writer. All rights reserved.",
51 "rewrite_hosts": ["https://old.example.org"], "nav": [["Blog", "/blog/"]]}
52 self.config.update(config or {})
53
54 def cleanup(self):
55 import shutil
56 shutil.rmtree(self.dir, ignore_errors=True)
57
58 def file(self, rel, text, base=None):
59 path = os.path.join(base or self.src, rel)
60 os.makedirs(os.path.dirname(path), exist_ok=True)
61 with open(path, "w", encoding="utf-8") as f:
62 f.write(text)
63
64 def cfg(self):
65 path = os.path.join(self.dir, "fw-blog.json")
66 with open(path, "w") as f:
67 json.dump(self.config, f)
68 return blog.load_config(path)
69
70 def build(self, htaccess=True):
71 cfg = self.cfg()
72 out = os.path.join(self.dir, "out")
73 blog.build(cfg, out, htaccess=htaccess)
74 return cfg, out
75
76 def read(self, out, rel):
77 with open(os.path.join(out, rel), encoding="utf-8") as f:
78 return f.read()
79
80
81def all_output(out):
82 texts = []
83 for dirpath, _, files in os.walk(out):
84 for name in files:
85 with open(os.path.join(dirpath, name), "rb") as f:
86 texts.append(f.read().decode("utf-8", errors="replace"))
87 return "\n".join(texts)
88
89
90class FrontMatterAndConfig(unittest.TestCase):
91 def test_front_matter_parsing(self):
92 meta, rest = blog.parse_front_matter("---\npublished: true\ntitle: \"Hi: there\"\ntags: a, b\n---\n# T\nbody")
93 self.assertEqual(meta, {"published": True, "title": "Hi: there", "tags": "a, b"})
94 self.assertEqual(rest, "# T\nbody")
95
96 def test_horizontal_rule_is_not_front_matter(self):
97 text = "---\nnot a key value line\n---\ntext"
98 self.assertEqual(blog.parse_front_matter(text), ({}, text))
99 self.assertEqual(blog.parse_front_matter("---\nnever closed"), ({}, "---\nnever closed"))
100
101 def test_unknown_and_missing_config_keys(self):
102 t = Tree(self, {"bogus": 1})
103 with self.assertRaises(blog.BlogError):
104 t.cfg()
105 t = Tree(self)
106 del t.config["title"]
107 with self.assertRaises(blog.BlogError):
108 t.cfg()
109
110 def test_timezone_key_is_validated(self):
111 self.assertEqual(Tree(self).cfg()["timezone"], "UTC")
112 self.assertEqual(Tree(self, {"timezone": "America/Los_Angeles"}).cfg()["timezone"], "America/Los_Angeles")
113 for bad in ("Mars/Olympus", "", 5):
114 with self.assertRaises(blog.BlogError):
115 Tree(self, {"timezone": bad}).cfg()
116
117 def test_precedence_front_matter_over_config_over_file(self):
118 t = Tree(self, {"posts": {"2018-03-02-first-post.md": {"title": "Config Title", "slug": "config-slug", "published": True,
119 "tags": ["x"], "date": "2019-01-01"}}})
120 t.file("2018-03-02-first-post.md", "---\ntitle: Front Title\n---\n" + POST)
121 post = blog.scan_posts(t.cfg())[0]
122 self.assertEqual(post.title, "Front Title") # front matter beats config
123 self.assertEqual(post.slug, "config-slug") # config beats file name
124 self.assertEqual(post.date.date(), datetime.date(2019, 1, 1))
125 self.assertTrue(post.published)
126 self.assertEqual(post.path, "/2019/01/01/config-slug/")
127
128 def test_file_defaults(self):
129 t = Tree(self)
130 t.file("2018-03-02-first-post.md", POST)
131 post = blog.scan_posts(t.cfg())[0]
132 self.assertEqual((post.title, post.slug, post.path), ("First Post", "first-post", "/2018/03/02/first-post/"))
133 self.assertFalse(post.published) # nothing says published
134
135 def test_published_only_when_exactly_true(self):
136 t = Tree(self, {"posts": {"2018-01-01-b.md": {"published": "true"}, "2018-01-02-c.md": {"published": 1}}})
137 t.file("2018-01-01-a.md", "---\npublished: yes\n---\n# A\ntext")
138 t.file("2018-01-01-b.md", "# B\ntext")
139 t.file("2018-01-02-c.md", "# C\ntext")
140 t.file("2018-01-03-d.md", "---\npublished: true\n---\n# D\ntext")
141 t.file("2018-01-04-e.md", "---\npublished: false\n---\n# E\ntext")
142 got = {p.rel: p.published for p in blog.scan_posts(t.cfg())}
143 self.assertEqual(got, {"2018-01-01-a.md": False, "2018-01-01-b.md": False, "2018-01-02-c.md": False,
144 "2018-01-03-d.md": True, "2018-01-04-e.md": False})
145
146 def test_conflicted_copies_skipped(self):
147 t = Tree(self)
148 t.file("2018-03-02-a.md", "# A")
149 t.file("2018-03-02-a (Pavel's conflicted copy 2018-03-03).md", "# A2")
150 self.assertEqual([p.rel for p in blog.scan_posts(t.cfg())], ["2018-03-02-a.md"])
151
152
153class ExtractBody(unittest.TestCase):
154 def test_writing_section(self):
155 body = blog.extract_body(POST)
156 self.assertTrue(body.startswith("Line one"))
157 for secret in ("SECRET-SPELLING-WORDS", "SECRET-REMOVED-TEXT", "## Spelling", "Created by template", "First Post"):
158 self.assertNotIn(secret, body)
159
160 def test_without_writing_section(self):
161 body = blog.extract_body("# Title\n\nCreated by template, on 2018-01-01\n\nJust text.\n\n## Spelling\n\nSECRET\n")
162 self.assertEqual(body, "Just text.")
163
164 def test_writing_not_last_section(self):
165 text = "# T\n\n## Writing\n\nBody.\n\n## Info\n\nINFO-TEXT\n\n## Spelling\n\nSECRET\n"
166 self.assertEqual(blog.extract_body(text), "Body.")
167
168 def test_body_until_override(self):
169 text = "# T\n\n## Writing\n\nBody.\n\n## Actual writing\n\nMore body.\n\n## Spelling\n\nSECRET\n\n## Info\n\nINFO-TEXT\n"
170 body = blog.extract_body(text, "Writing", "Info")
171 self.assertIn("Body.", body)
172 self.assertIn("More body.", body)
173 self.assertNotIn("INFO-TEXT", body)
174 self.assertNotIn("SECRET", body)
175
176 def test_private_sections_dropped_without_writing_section_and_with_section_none(self):
177 text = "# T\n\nIntro.\n\n## Removed writing\n\nSECRET\n\n## Other\n\nKept.\n"
178 for body in (blog.extract_body(text), blog.extract_body(text, None)):
179 self.assertNotIn("SECRET", body)
180 self.assertIn("Kept.", body)
181
182 def test_html_comments_removed(self):
183 text = ("# T\n\n## Writing\n\nVisible <!-- inline secret --> text.\n\n<!--\nmulti\nline SECRET-COMMENT\n## Spelling\n-->\n\n"
184 "Before more.\n<!--more-->\nAfter more.\n\n<!-- never closed SECRET-OPEN\n")
185 body = blog.extract_body(text)
186 self.assertNotIn("SECRET", body)
187 self.assertNotIn("inline secret", body)
188 self.assertIn("Visible text.", body)
189 self.assertIn("<!--more-->", body) # the excerpt marker survives for the renderer
190
191 def test_heading_inside_code_fence_is_not_a_section(self):
192 text = "# T\n\n## Writing\n\n```\n## Spelling\ncode\n```\n\nEnd.\n"
193 self.assertIn("code", blog.extract_body(text))
194
195 def test_title_of(self):
196 self.assertEqual(blog.title_of("intro\n# The Title\n\n## Writing\n"), "The Title")
197 self.assertIsNone(blog.title_of("no title"))
198
199
200@needs_markdown
201class Build(unittest.TestCase):
202 def blog_tree(self, **config):
203 t = Tree(self, config)
204 t.file("2018-03-02-first-post.md", "---\npublished: true\ntags: Fiction, Notes\n---\n" + POST)
205 t.file("2016-05-06-old-post.md", "---\npublished: true\n---\n# Old\n\nSome old text.\n<!--more-->\nrest\n")
206 t.file("2019-01-01-draft.md", "# Draft\n\nDRAFT-TEXT\n")
207 return t
208
209 def test_unpublished_not_built_and_private_text_never_output(self):
210 t = self.blog_tree()
211 cfg, out = t.build()
212 everything = all_output(out)
213 for secret in ("DRAFT-TEXT", "SECRET-SPELLING-WORDS", "SECRET-REMOVED-TEXT", "Created by template"):
214 self.assertNotIn(secret, everything)
215 self.assertFalse(os.path.exists(os.path.join(out, "2019")))
216 self.assertTrue(os.path.exists(os.path.join(out, "2018/03/02/first-post/index.html")))
217
218 def test_nl2br(self):
219 t = self.blog_tree(line_breaks="br")
220 _, out = t.build()
221 self.assertIn("Line one<br", t.read(out, "2018/03/02/first-post/index.html"))
222 t = self.blog_tree(line_breaks="markdown")
223 _, out = t.build()
224 self.assertNotIn("Line one<br", t.read(out, "2018/03/02/first-post/index.html"))
225
226 def test_absolute_links_rewritten_and_redirected(self):
227 t = Tree(self, {"rewrite_hosts": ["https://old.example.org"], "redirects": {"/2018/03/02/moved/": "/2018/03/02/new/"},
228 "wp_ids": {"12": "/about/"}})
229 t.file("2018-03-02-p.md", "---\npublished: true\n---\n# P\n\n[a](https://old.example.org/x/y/#frag) "
230 "[b](https://old.example.org/2018/03/02/moved/) [c](https://elsewhere.org/z/) "
231 "[d](https://old.example.org/?p=12) ![i](https://old.example.org/wp-content/uploads/a.png)\n")
232 _, out = t.build()
233 page = t.read(out, "2018/03/02/p/index.html")
234 self.assertIn('href="/x/y/#frag"', page)
235 self.assertIn('href="/2018/03/02/new/"', page)
236 self.assertIn('href="https://elsewhere.org/z/"', page)
237 self.assertIn('href="/about/"', page)
238 self.assertIn('src="/wp-content/uploads/a.png"', page)
239 self.assertNotIn("old.example.org", page)
240
241 def test_atom_feed_parses_and_has_absolute_links(self):
242 t = self.blog_tree()
243 _, out = t.build()
244 root = ET.fromstring(t.read(out, "feed/index.xml").encode("utf-8"))
245 ns = {"a": atom.NS}
246 entries = root.findall("a:entry", ns)
247 self.assertEqual(len(entries), 2)
248 self.assertEqual(entries[0].find("a:title", ns).text, "First Post") # newest first
249 self.assertIn("2018", root.find("a:rights", ns).text)
250 content = entries[0].find("a:content", ns).text
251 self.assertIn('href="https://blog.example.org/2018/03/02/first-post/"', content) # old host -> site -> absolute
252 self.assertEqual(blog.absolutize('<a href="/x/">a</a><img src="//cdn/y.png"><a href="https://z/">z</a>', "https://b.org"),
253 '<a href="https://b.org/x/">a</a><img src="//cdn/y.png"><a href="https://z/">z</a>')
254
255 def test_feed_times_use_the_configured_timezone(self):
256 def times(**config):
257 t = Tree(self, config)
258 t.file("2018-03-02-p.md", "---\npublished: true\ndate: 2018-03-02 09:15:00\n---\n# P\n\ntext\n")
259 _, out = t.build()
260 root = ET.fromstring(t.read(out, "feed/index.xml").encode("utf-8"))
261 entry = root.find("a:entry", {"a": atom.NS})
262 return (root.find("a:updated", {"a": atom.NS}).text, entry.find("a:published", {"a": atom.NS}).text)
263 self.assertEqual(times(), ("2018-03-02T09:15:00Z",) * 2) # default UTC
264 self.assertEqual(times(timezone="America/Los_Angeles"), ("2018-03-02T17:15:00Z",) * 2) # PST in March
265 self.assertEqual(times(timezone="Europe/Berlin"), ("2018-03-02T08:15:00Z",) * 2)
266 self.assertEqual(blog.localize({"timezone": "America/Los_Angeles"}, datetime.datetime(2018, 7, 1, 12)).utcoffset(),
267 datetime.timedelta(hours=-7)) # PDT in July
268
269 def test_feed_escapes_markup_and_illegal_characters(self):
270 xml = atom.atom_feed("T & <t>", "https://x.org/", datetime.datetime(2020, 1, 2),
271 [{"title": "A <b>&", "url": "https://x.org/a/", "updated": datetime.date(2020, 1, 2),
272 "content": "<p>x\x0c</p>"}], rights="© me")
273 root = ET.fromstring(xml.encode("utf-8"))
274 self.assertEqual(root.find("{%s}title" % atom.NS).text, "T & <t>")
275 self.assertEqual(root.find("{%s}entry/{%s}content" % (atom.NS, atom.NS)).text, "<p>x</p>")
276
277 def test_comments_escaped_threaded_and_minimal(self):
278 t = self.blog_tree(comments="comments.json")
279 t.file("comments.json", json.dumps({"/2018/03/02/first-post/": [
280 {"id": 1, "parent": 0, "author": "Eve <script>", "date": "2018-03-03T10:00:00", "text": "Hi <b>there</b>\nnew line",
281 "email": "eve@example.org", "author_url": "https://evil.example"},
282 {"id": 2, "parent": 1, "author": "Bob", "date": "2018-03-04T10:00:00", "text": "reply"}]}), base=t.dir)
283 _, out = t.build()
284 page = t.read(out, "2018/03/02/first-post/index.html")
285 self.assertIn("Eve &lt;script&gt;", page)
286 self.assertIn("Hi &lt;b&gt;there&lt;/b&gt;<br>", page)
287 self.assertNotIn("<script>", page)
288 self.assertNotIn("eve@example.org", page)
289 self.assertNotIn("evil.example", page)
290 self.assertIn("Comments © their authors.", page)
291 self.assertLess(page.index("Eve"), page.index("Bob"))
292 self.assertIn('<ul class="comments"><li class="comment" id="comment-2"', page) # the reply is nested
293
294 def test_copyright_years(self):
295 t = self.blog_tree()
296 _, out = t.build()
297 self.assertIn("© 2016–2018 A. Writer. All rights reserved.", t.read(out, "blog/index.html"))
298 self.assertIn("© 2016–2018", t.read(out, "feed/index.xml"))
299 t = Tree(self)
300 t.file("2018-03-02-only.md", "---\npublished: true\n---\n# Only\ntext")
301 _, out = t.build()
302 self.assertIn("© 2018 A. Writer", t.read(out, "index.html"))
303
304 def test_excerpt_keeps_paragraphs_and_links_to_the_rest(self):
305 long_para = " ".join(f"w{i}" for i in range(78))
306 t = self.blog_tree()
307 t.file("2018-04-01-long.md", f"---\npublished: true\n---\n# Long One\n\n## Writing\n\n{long_para}\n\nsecond paragraph here\n\nthird one\n")
308 t.file("2018-04-02-short.md", "---\npublished: true\n---\n# Short One\n\n## Writing\n\nonly a line\n")
309 _, out = t.build()
310 index = t.read(out, "blog/index.html")
311 long_entry = index.split("Long One")[1].split("</article>")[0]
312 self.assertIn("<p>w0 w1", long_entry) # paragraphs stay paragraphs
313 self.assertNotIn("second paragraph here", long_entry) # whole paragraphs, up to about 80 words
314 self.assertIn("Continue reading", long_entry)
315 self.assertIn('href="/2018/04/01/long/"', long_entry)
316 short_entry = index.split("Short One")[1].split("</article>")[0]
317 self.assertIn("<p>only a line</p>", short_entry)
318 self.assertNotIn("Continue reading", short_entry) # nothing more to read
319
320 def test_excerpt_keeps_chapter_headings(self):
321 t = self.blog_tree()
322 t.file("2018-04-03-chapters.md", "---\npublished: true\n---\n# Chapters\n\n## Writing\n\n### Lights & time\n\nfirst words\n\n"
323 + " ".join(f"w{i}" for i in range(90)) + "\n\n### Second chapter\n\nlater\n")
324 _, out = t.build()
325 entry = t.read(out, "blog/index.html").split("Chapters")[1].split("</article>")[0]
326 self.assertIn('<h5 class="chapter">Lights &amp; time</h5>', entry) # the chapter title shows in the list
327 self.assertLess(entry.index("Lights"), entry.index("first words"))
328 self.assertNotIn("Second chapter", entry) # a heading is not left dangling at the end
329
330 def test_theme_switch_and_progress_bar(self):
331 t = self.blog_tree()
332 _, out = t.build()
333 self.assertIn('<script src="/blog.js"></script>', t.read(out, "2018/03/02/first-post/index.html"))
334 self.assertTrue(os.path.exists(os.path.join(out, "blog.js")))
335 css = t.read(out, "blog.css")
336 shared = t.read(out, "_shared.css")
337 self.assertIn(".post-body br", css) # as on the old site: line breaks in the source do not show
338 self.assertIn(':root[data-theme="dark"]', css)
339 self.assertIn(':root[data-theme="light"]', css)
340 dark = css.split(':root[data-theme="dark"]')[1]
341 light = css.split(':root[data-theme="light"]')[1].split(":root[data-theme=")[0]
342 self.assertIn("--color-bg-primary: #111113;", dark) # the values come from _shared.css
343 self.assertIn("--color-bg-primary: #ffffff;", light)
344 self.assertIn("--color-bg-primary: #111113;", shared)
345 blog.audit_output(out) # .js is an allowed generated file: no BlogError
346
347 def test_site_bar_above_the_blog_bar(self):
348 site = {"name": "mysite", "url": "https://example.org/", "current": "Writing",
349 "nav": [["Code", "https://code.example.org/"], ["Writing", "https://blog.example.org/"]]}
350 t = self.blog_tree(site=site)
351 _, out = t.build()
352 page = t.read(out, "2018/03/02/first-post/index.html")
353 self.assertIn('<a class="brand" href="https://example.org/">mysite</a>', page)
354 self.assertIn('<a href="https://code.example.org/">Code</a>', page)
355 self.assertIn('<a href="https://blog.example.org/" aria-current="page">Writing</a>', page)
356 self.assertIn('aria-label="This blog"', page)
357 self.assertIn('<a href="/blog/">Blog</a>', page)
358 for bad in ("x", {"name": "a"}, {"name": "a", "url": "u", "nav": [["only one"]]}, {"name": "a", "url": "u", "nav": [], "x": 1}):
359 with self.assertRaises(blog.BlogError):
360 Tree(self, {"site": bad}).cfg()
361
362 def test_site_nav_gen_uses_the_fuzzyworld_nav(self):
363 t = self.blog_tree(site={"name": "fuzzyworld", "url": "https://fuzzyworld.net/", "nav": "gen", "current": "Writing"})
364 self.assertEqual(t.cfg()["site"]["nav"], gen.site_nav())
365 _, out = t.build()
366 page = t.read(out, "2018/03/02/first-post/index.html")
367 for label, url in gen.site_nav():
368 self.assertIn(f'<a href="{url}"', page)
369 self.assertIn(f">{label}</a>", page)
370
371 def test_site_footer_is_shared_by_blog_and_landing_page(self):
372 t = self.blog_tree(site={"name": "fuzzyworld", "url": "https://fuzzyworld.net/", "nav": "gen", "current": "Writing"})
373 _, out = t.build()
374 page = t.read(out, "2018/03/02/first-post/index.html")
375 self.assertIn(gen.site_footer("writing"), page)
376 self.assertIn("All rights reserved", page) # the blog's own line stays under the shared columns
377 with tempfile.TemporaryDirectory() as d:
378 src, dest = os.path.join(d, "in.html"), os.path.join(d, "out.html")
379 with open(src, "w") as f:
380 f.write("<nav><!--SITE-NAV--></nav><footer><!--SITE-FOOTER--></footer>")
381 gen.build_landing(src, dest)
382 with open(dest) as f:
383 self.assertIn(gen.site_footer("landing"), f.read())
384 self.assertIn('href="/ci/"', gen.site_footer("code")) # own site stays relative
385 self.assertIn('href="https://apps.fuzzyworld.net/"', gen.site_footer("code"))
386 self.assertIn('href="/"', gen.site_footer("apps"))
387 self.assertIn('href="https://code.fuzzyworld.net/"', gen.site_footer("apps"))
388 t = self.blog_tree(site={"name": "x", "url": "https://x.org/", "nav": [["Code", "https://c.x.org/"]]})
389 _, out = t.build()
390 self.assertNotIn("Built with", t.read(out, "2018/03/02/first-post/index.html")) # explicit nav: no shared footer
391 with self.assertRaises(blog.BlogError):
392 Tree(self, {"site": {"name": "x", "url": "u", "nav": [], "footer": True}}).cfg()
393 self.assertIn('href="https://code.fuzzyworld.net/ci/"', gen.site_footer("apps"))
394
395 def test_landing_page_nav_comes_from_nav(self):
396 import tempfile
397 with tempfile.TemporaryDirectory() as d:
398 src, dest = os.path.join(d, "in.html"), os.path.join(d, "out.html")
399 with open(src, "w") as f:
400 f.write('<nav><!--SITE-NAV--></nav><!--SITE-FOOTER-->')
401 gen.build_landing(src, dest)
402 with open(dest) as f:
403 text = f.read()
404 self.assertNotIn("SITE-NAV", text)
405 for label, url in gen.site_nav():
406 self.assertIn(f'<a href="{url}">{label}</a>', text)
407
408 def test_landing_text_without_landing_md_is_an_error(self):
409 with tempfile.TemporaryDirectory() as d:
410 src = os.path.join(d, "in.html")
411 with open(src, "w") as f:
412 f.write("<nav><!--SITE-NAV--></nav><!--SITE-FOOTER--><!--LANDING-TEXT-->")
413 with self.assertRaises(SystemExit):
414 gen.build_landing(src, os.path.join(d, "out.html"))
415
416 @needs_markdown
417 def test_landing_text_is_rendered_from_landing_md(self):
418 with tempfile.TemporaryDirectory() as d:
419 src, dest = os.path.join(d, "in.html"), os.path.join(d, "out.html")
420 with open(src, "w") as f:
421 f.write("<nav><!--SITE-NAV--></nav><!--SITE-FOOTER--><main><!--LANDING-TEXT--></main>")
422 with open(os.path.join(d, "landing.md"), "w") as f:
423 f.write("Hello <https://example.com/>\n\n- one\n- two\n")
424 gen.build_landing(src, dest)
425 with open(dest) as f:
426 text = f.read()
427 self.assertNotIn("LANDING-TEXT", text)
428 self.assertIn('<a href="https://example.com/">', text)
429 self.assertIn("<li>two</li>", text)
430
431 def test_landing_page_without_the_placeholder_is_an_error(self):
432 with tempfile.TemporaryDirectory() as d:
433 src = os.path.join(d, "in.html")
434 with open(src, "w") as f:
435 f.write("<nav></nav>")
436 with self.assertRaises(SystemExit):
437 gen.build_landing(src, os.path.join(d, "out.html"))
438
439 def test_index_pages_tags_and_home(self):
440 t = self.blog_tree(pages_dir="pages", home="start", tags={"fiction": "Fiction"}, feed={"path": "/feed/", "entries": 1})
441 t.file("pages/start.md", "---\npublished: true\ntitle: Start Here\n---\nWelcome.\n")
442 t.file("pages/about.md", "---\npublished: true\n---\n# About\n\nAbout text.\n")
443 t.file("pages/forgotten.md", "# Forgotten\n\nPAGE-NOT-PUBLISHED\n") # pages are opt-in, like posts
444 t.file("pages/hidden.md", "---\npublished: false\n---\n# Hidden\n\nPAGE-FALSE\n")
445 _, out = t.build()
446 self.assertNotIn("PAGE-NOT-PUBLISHED", all_output(out))
447 self.assertNotIn("PAGE-FALSE", all_output(out))
448 self.assertFalse(os.path.exists(os.path.join(out, "forgotten")))
449 blog_index = t.read(out, "blog/index.html")
450 self.assertIn('id="2018"', blog_index)
451 self.assertIn('id="2018-03"', blog_index)
452 self.assertIn('id="2016-05"', blog_index)
453 self.assertLess(blog_index.index('id="2018"'), blog_index.index('id="2016"'))
454 self.assertIn("Some old text.", blog_index)
455 self.assertNotIn("rest", blog_index.split("Some old text.")[1].split("</article>")[0]) # excerpt stops at <!--more-->
456 self.assertIn('href="/tag/fiction/"', t.read(out, "2018/03/02/first-post/index.html"))
457 self.assertIn("Fiction", t.read(out, "tag/fiction/index.html"))
458 self.assertIn("Welcome.", t.read(out, "index.html"))
459 self.assertIn("About text.", t.read(out, "about/index.html"))
460 self.assertTrue(os.path.exists(os.path.join(out, "404.html")))
461 root = ET.fromstring(t.read(out, "feed/index.xml").encode("utf-8"))
462 self.assertEqual(len(root.findall("{%s}entry" % atom.NS)), 1) # feed.entries
463
464 def test_highlighting_fenced_code(self):
465 t = Tree(self)
466 t.file("2018-03-02-code.md", "---\npublished: true\n---\n# C\n\n```python\ndef f(): return 1 < 2\n```\n")
467 _, out = t.build()
468 page = t.read(out, "2018/03/02/code/index.html")
469 self.assertIn("&lt;", page)
470 if blog.gen.pygments_lexers:
471 self.assertIn('<span class="k">def</span>', page)
472
473 def test_htaccess_and_redirects(self):
474 t = self.blog_tree(redirects={"/old-page/": "/about/"}, wp_ids={"7": "/2018/03/02/first-post/"}, home="start",
475 pages_dir="pages")
476 t.file("pages/start.md", "---\npublished: true\n---\n# Start\ntext")
477 _, out = t.build()
478 ht = t.read(out, ".htaccess")
479 self.assertIn("script-src 'self'", ht) # blog.js only: no inline scripts, no other origins
480 self.assertIn("frame-src 'none'", ht)
481 self.assertIn("DirectoryIndex index.html index.xml", ht)
482 self.assertIn("ForceType application/atom+xml", ht)
483 self.assertIn("ErrorDocument 404 /404.html", ht)
484 self.assertIn("RewriteRule ^old-page/?$ /about/ [R=301,L]", ht)
485 self.assertIn("RewriteCond %{QUERY_STRING} (^|&)p=7(&|$)", ht)
486 self.assertIn("RewriteCond %{QUERY_STRING} (^|&)page_id=7(&|$)", ht)
487 self.assertIn("RewriteRule ^start/?$ / [R=301,L]", ht)
488 self.assertIn("/blog/#$1 [NE,R=301,L]", ht)
489 self.assertIn("/blog/#$1-$2 [NE,R=301,L]", ht)
490 self.assertIn("RewriteRule ^(wp-admin|wp-login\\.php|xmlrpc\\.php) - [R=404,L]", ht)
491 _, out2 = Tree.build(self.blog_tree(), htaccess=False)
492 self.assertFalse(os.path.exists(os.path.join(out2, ".htaccess")))
493
494 def test_pattern_escaping(self):
495 self.assertEqual(blog.apache_pattern("/a.b/c+d/"), r"^a\.b/c\+d/?$")
496
497
498@needs_markdown
499class UploadSafety(unittest.TestCase):
500 """~/Blog holds much that must never be published: only generated files and allowlisted media may be uploaded."""
501
502 def make(self):
503 t = Tree(self, {"media": "media", "pages_dir": "pages", "comments": "comments.json"})
504 t.file("2018-03-02-first-post.md", "---\npublished: true\n---\n" + POST)
505 t.file("2018-03-03-draft.md", "# Draft\n\nDRAFT-TEXT\n")
506 t.file("notes.md", "# Notes\n\nNOTES-TEXT\n")
507 t.file("pages/about.md", "---\npublished: true\n---\n# About\n\nAbout.\n")
508 t.file("media/wp-content/uploads/pic.png", "png")
509 t.file("media/wp-content/uploads/book.epub", "epub")
510 t.file("media/wp-content/uploads/readme.md", "MEDIA-MD-SECRET")
511 t.file("media/wp-content/uploads/data.json", "MEDIA-JSON-SECRET")
512 t.file("media/wp-content/uploads/notes.txt", "MEDIA-TXT-SECRET")
513 t.file("media/wp-content/uploads/.env", "MEDIA-ENV-SECRET")
514 t.file("media/wp-content/uploads/run.php", "MEDIA-PHP-SECRET")
515 t.file("media/_api/posts.json", "MEDIA-API-SECRET")
516 t.file("wordpress/_api/posts.json", "API-SECRET")
517 t.file("comments.json", '{"/x/": []}', base=t.dir)
518 return t
519
520 def test_build_contains_only_allowed_files(self):
521 t = self.make()
522 _, out = t.build()
523 self.assertTrue(os.path.exists(os.path.join(out, "wp-content/uploads/pic.png")))
524 self.assertTrue(os.path.exists(os.path.join(out, "wp-content/uploads/book.epub")))
525 files = [os.path.relpath(os.path.join(d, f), out) for d, _, fs in os.walk(out) for f in fs]
526 for f in files:
527 name = os.path.basename(f)
528 if name != "robots.txt":
529 self.assertNotIn(os.path.splitext(name)[1], (".md", ".json", ".txt", ".php", ".env"), f)
530 everything = all_output(out)
531 for secret in ("DRAFT-TEXT", "NOTES-TEXT", "SECRET", "fw-blog.json", "comments.json"):
532 self.assertNotIn(secret, everything)
533 count, size = blog.audit_output(out) # the audit accepts what the build wrote
534 self.assertGreater(count, 5)
535 self.assertGreater(size, 0)
536
537 def test_audit_refuses_stray_files(self):
538 t = self.make()
539 _, out = t.build()
540 for name in ("leak.md", "config.json", "notes.txt", ".env", ".git", "x.php", "noext"):
541 path = os.path.join(out, name)
542 with open(path, "w") as f:
543 f.write("x")
544 with self.assertRaises(blog.BlogError, msg=name):
545 blog.audit_output(out)
546 os.remove(path)
547 sub = os.path.join(out, "wp-content", "extra.md")
548 with open(sub, "w") as f:
549 f.write("x")
550 with self.assertRaises(blog.BlogError):
551 blog.audit_output(out)
552 os.remove(sub)
553 os.symlink(os.path.join(t.src, "notes.md"), os.path.join(out, "link.png"))
554 with self.assertRaises(blog.BlogError):
555 blog.audit_output(out)
556
557 def test_out_inside_source_refused_and_foreign_folder_not_emptied(self):
558 t = self.make()
559 cfg = t.cfg()
560 with self.assertRaises(blog.BlogError):
561 blog.build(cfg, os.path.join(t.src, "out"))
562 foreign = os.path.join(t.dir, "precious")
563 os.makedirs(foreign)
564 with open(os.path.join(foreign, "keep.txt"), "w") as f:
565 f.write("x")
566 with self.assertRaises(blog.BlogError):
567 blog.build(cfg, foreign)
568 self.assertTrue(os.path.exists(os.path.join(foreign, "keep.txt")))
569
570
571SECRET_TAIL = "\n\n## Spelling\n\nSECRET-WORDS\n"
572
573
574class PrivateSectionsFailClosed(unittest.TestCase):
575 """A private heading written in an unexpected way must never publish its section: it is removed or the build stops."""
576
577 def body(self, text, **kw):
578 return blog.extract_body("# T\n\n## Writing\n\n" + text, **kw)
579
580 def test_heading_variants_that_are_not_recognized_stop_the_build(self):
581 for text in ("Visible.\n\n##Spelling\n\nSECRET-WORDS\n", # no space after ##
582 "Visible.\n\nSpelling\n--------\n\nSECRET-WORDS\n", # underlined (setext)
583 "Visible.\n\nRemoved writing\n===\n\nSECRET-WORDS\n",
584 "Visible.\n\n### Spelling\n\nSECRET-WORDS\n", # wrong level
585 "Visible.\n\n#Spelling\nSECRET-WORDS\n",
586 "Visible.\n\n```\nunclosed fence\n\n## Spelling\n\nSECRET-WORDS\n"):
587 with self.assertRaisesRegex(blog.BlogError, r"my-post\.md"):
588 self.body(text, name="my-post.md")
589
590 def test_other_section_names_are_caught_when_there_is_no_writing_section(self):
591 for heading in ("## Spelling:", "## **Spelling**", "## spelling", "## Removed Writing ##"):
592 text = f"# T\n\nVisible.\n\n{heading}\n\nSECRET-WORDS\n"
593 try:
594 body = blog.extract_body(text, None, name="x.md")
595 except blog.BlogError:
596 continue
597 self.assertNotIn("SECRET-WORDS", body, heading)
598
599 def test_error_names_the_line(self):
600 with self.assertRaisesRegex(blog.BlogError, r"my-post\.md: line 3 .*Spelling"):
601 self.body("Visible.\n\n##Spelling\nSECRET-WORDS\n", name="my-post.md")
602
603 def test_variants_that_are_removed_cleanly(self):
604 for text in ("Visible.\n\n ## Spelling\n\nSECRET-WORDS\n", # up to 3 spaces is still a heading
605 "Visible.\n\n```y``` an inline span\n\n## Spelling\n\nSECRET-WORDS\n", # not a fence
606 "Visible.\n\n~~~\n```\n~~~\n\n## Removed writing\n\nSECRET-WORDS\n"):
607 body = self.body(text)
608 self.assertNotIn("SECRET-WORDS", body)
609 self.assertIn("Visible.", body)
610
611 def test_fences_follow_commonmark(self):
612 text = "````\n```\n## Spelling\ncode\n````\n\nAfter.\n\n## Spelling\n\nSECRET-WORDS\n"
613 body = self.body(text) # the inner ``` does not close the outer fence
614 self.assertIn("code", body)
615 self.assertIn("## Spelling\ncode", body) # still code
616 self.assertIn("After.", body)
617 self.assertNotIn("SECRET-WORDS", body)
618 body = self.body("```\n~~~\n## Spelling\ncode\n```\n\nEnd.\n") # another character does not close it either
619 self.assertIn("code", body)
620 body = self.body("~~~~\n~~~\n## Spelling\ncode\n~~~~\n\nEnd." + SECRET_TAIL) # a shorter fence does not close it
621 self.assertIn("code", body)
622 self.assertNotIn("SECRET-WORDS", body)
623
624 def test_inline_triple_backticks_do_not_open_a_fence(self):
625 fences = blog._Fences()
626 self.assertFalse(fences.feed("see ```y``` here"))
627 self.assertFalse(fences.feed("```y``` here"))
628 self.assertTrue(fences.feed("```python"))
629 self.assertTrue(fences.feed("```js")) # an info string cannot close a fence
630 self.assertIsNotNone(fences.mark)
631 self.assertTrue(fences.feed("```"))
632 self.assertIsNone(fences.mark)
633
634 def test_comment_containing_the_more_marker(self):
635 body = self.body("Before.\n\n<!-- note <!--more--> SECRET-WORDS -->\n\nAfter.\n")
636 self.assertNotIn("SECRET-WORDS", body)
637 self.assertNotIn("-->", body)
638 self.assertNotIn("more", body)
639 self.assertIn("After.", body)
640 body = self.body("One.\n<!-- c -->\ntwo <!--more--> three <!-- d --> four\n") # a real marker survives
641 self.assertEqual(body.count("<!--more-->"), 1)
642 self.assertNotIn("<!-- c", body)
643 self.assertIn("three four", body)
644
645 def test_unclosed_comment_warns_naming_the_file(self):
646 import contextlib
647 import io
648 err = io.StringIO()
649 with contextlib.redirect_stderr(err):
650 body = self.body("Visible.\n\n<!-- never closed SECRET-WORDS\n", name="draft-one.md")
651 self.assertIn("draft-one.md", err.getvalue())
652 self.assertIn("warning", err.getvalue())
653 self.assertNotIn("SECRET-WORDS", body)
654 err = io.StringIO()
655 with contextlib.redirect_stderr(err):
656 self.body("Visible. <!-- closed --> fine\n", name="draft-one.md")
657 self.assertEqual(err.getvalue(), "")
658
659 def test_a_build_stops_on_a_leaky_heading(self):
660 t = Tree(self)
661 t.file("2018-03-02-leaky.md", "---\npublished: true\n---\n# Leaky\n\n## Writing\n\nText.\n\n##Spelling\n\nSECRET-WORDS\n")
662 with self.assertRaisesRegex(blog.BlogError, "2018-03-02-leaky.md"):
663 t.build()
664 self.assertNotIn("SECRET-WORDS", all_output(os.path.join(t.dir, "out")) if os.path.isdir(os.path.join(t.dir, "out")) else "")
665
666
667class SlugsAndPaths(unittest.TestCase):
668 def test_check_slug(self):
669 for good in ("first-post", "caf%c3%a9", "v1.2", "A_b", "7"):
670 self.assertEqual(blog.check_slug(good, "x"), good)
671 for bad in ("", ".", "..", "../x", "a/b", ".hidden", "a b", "%2e%2e", "a%2fb", "é", "-x", "a\\b", "a%00b", "a\nb"):
672 with self.assertRaises(blog.BlogError, msg=repr(bad)):
673 blog.check_slug(bad, "x")
674
675 def test_bad_slugs_in_posts_pages_tags_feed_and_home(self):
676 t = Tree(self)
677 t.file("2018-03-02-x.md", "---\npublished: true\nslug: ../../evil\n---\n# X\n\nBody.\n")
678 with self.assertRaises(blog.BlogError):
679 t.build()
680 t = Tree(self, {"pages_dir": "pages"})
681 t.file("pages/p.md", "---\npublished: true\nslug: ../evil\n---\n# P\n")
682 with self.assertRaises(blog.BlogError):
683 t.build()
684 for config in ({"tags": {"../x": "Bad"}}, {"feed": {"path": "/../x/"}}, {"feed": {"path": "/a b/"}}, {"home": "../x"},
685 {"feed": {"path": "no-slash"}}):
686 with self.assertRaises(blog.BlogError, msg=str(config)):
687 Tree(self, config).cfg()
688
689 def test_post_slug_may_be_percent_encoded_and_is_served_decoded(self):
690 t = Tree(self)
691 t.file("2018-03-02-x.md", "---\npublished: true\nslug: caf%c3%a9\n---\n# Café\n\nBody.\n")
692 _, out = t.build()
693 self.assertTrue(os.path.isfile(os.path.join(out, "2018/03/02/café/index.html"))) # the server decodes %c3%a9
694 self.assertIn('href="/2018/03/02/caf%c3%a9/"', t.read(out, "blog/index.html"))
695
696 def test_output_never_leaves_the_out_folder(self):
697 t = Tree(self, {"url": "/../../escaped/{slug}/"})
698 t.file("2018-03-02-x.md", "---\npublished: true\n---\n# X\n\nBody.\n")
699 with self.assertRaises(blog.BlogError):
700 t.build()
701 self.assertFalse(os.path.exists(os.path.join(t.dir, "escaped")))
702 out = os.path.join(t.dir, "o")
703 os.makedirs(os.path.join(t.dir, "elsewhere"))
704 os.makedirs(out)
705 os.symlink(os.path.join(t.dir, "elsewhere"), os.path.join(out, "linked"))
706 with self.assertRaises(blog.BlogError):
707 blog.safe_join(out, "linked", "index.html")
708 self.assertEqual(blog.safe_join(out, "a", "b.html"), os.path.join(out, "a", "b.html"))
709
710
711class DeployTarget(unittest.TestCase):
712 def target(self, deploy):
713 t = Tree(self, {"deploy": deploy})
714 return blog.deploy_target(t.cfg())
715
716 def test_good_targets(self):
717 self.assertEqual(self.target({"host": "dh", "dir": "writing.example.org"}), "dh:writing.example.org")
718 self.assertEqual(self.target({"host": "me@dh", "dir": "sites/writing.example.org/"}), "me@dh:sites/writing.example.org")
719
720 def test_dangerous_dirs_are_refused(self):
721 for folder in ("", "~", ".", "/", "..", "../x", "a/../b", "a/./b", "-rf", "-x/y", "/abs/path", "~/x", "a b", "x;y",
722 "a//b", "$HOME"):
723 with self.assertRaises(blog.BlogError, msg=repr(folder)):
724 self.target({"host": "dh", "dir": folder})
725 for host in ("", "-oProxyCommand=x", "a b"):
726 with self.assertRaises(blog.BlogError, msg=repr(host)):
727 self.target({"host": host, "dir": "site.example.org"})
728 with self.assertRaises(blog.BlogError):
729 self.target(None)
730
731 def test_command_exits_non_zero_and_prints_nothing(self):
732 t = Tree(self, {"deploy": {"host": "dh", "dir": "~"}})
733 path = os.path.join(t.dir, "fw-blog.json")
734 t.cfg()
735 with self.assertRaises(SystemExit) as caught:
736 blog.main(["blog.py", "deploy-target", path])
737 self.assertNotEqual(caught.exception.code, 0)
738 self.assertIn("deploy.dir", str(caught.exception.code))
739
740
741class HtaccessValues(unittest.TestCase):
742 def test_values_with_spaces_or_control_characters_are_refused(self):
743 self.assertTrue(blog.htaccess_unsafe("/a b/"))
744 self.assertTrue(blog.htaccess_unsafe("/a%20b/"))
745 self.assertTrue(blog.htaccess_unsafe("/a%0Ad/"))
746 self.assertTrue(blog.htaccess_unsafe("/a\tb"))
747 self.assertTrue(blog.htaccess_unsafe("/a%0d%0aHeader: x"))
748 self.assertFalse(blog.htaccess_unsafe("/caf%c3%a9/?x=1"))
749 for config in ({"redirects": {"/a%0Ab/": "/x/"}}, {"redirects": {"/a/": "/x%20y/"}}, {"redirects": {"/a/": "/x\nRewriteRule"}},
750 {"wp_ids": {"7": "/x%0a/"}}, {"wp_ids": {"7%0a": "/x/"}}, {"wp_ids": {"7": "/a b/"}},
751 {"feed": {"path": "/f%0a/"}}):
752 with self.assertRaises(blog.BlogError, msg=str(config)):
753 Tree(self, config).cfg()
754 Tree(self, {"redirects": {"/caf%c3%a9/": "/x/"}, "wp_ids": {"7": "/y/"}}).cfg() # fine
755
756
757@needs_markdown
758class SmallerFixes(unittest.TestCase):
759 def test_excerpt_of_a_chapter_heading_and_a_long_paragraph(self):
760 t = Tree(self)
761 t.file("2018-04-03-chapter.md", "---\npublished: true\n---\n# Opening\n\n## Writing\n\n### Chapter one\n\n"
762 + " ".join(f"w{i}" for i in range(120)) + "\n\nsecond paragraph\n")
763 _, out = t.build()
764 entry = t.read(out, "blog/index.html").split("Opening")[1].split("</article>")[0]
765 self.assertIn('<h5 class="chapter">Chapter one</h5>', entry)
766 self.assertIn("<p>w0 w1", entry) # not an empty excerpt
767 self.assertIn("w79…", entry)
768 self.assertNotIn("w80", entry)
769 self.assertNotIn("second paragraph", entry)
770 self.assertIn("Continue reading", entry)
771
772 def test_tags_in_brackets(self):
773 self.assertEqual(blog.as_list('[a, "b c", \'d\']'), ["a", "b c", "d"])
774 self.assertEqual(blog.as_list("[]"), [])
775 self.assertEqual(blog.as_list("a, b"), ["a", "b"])
776 self.assertEqual(blog.as_list(["x", " y "]), ["x", "y"])
777 t = Tree(self)
778 t.file("2018-03-02-x.md", "---\npublished: true\ntags: [Fiction, \"Short Stories\"]\n---\n# X\n\nBody.\n")
779 self.assertEqual(blog.scan_posts(t.cfg())[0].tags, ["Fiction", "Short Stories"])
780
781 def test_front_matter_after_a_byte_order_mark(self):
782 t = Tree(self)
783 t.file("2018-03-02-x.md", "\ufeff---\npublished: true\ntitle: With BOM\n---\n# X\n\nBody.\n")
784 post = blog.scan_posts(t.cfg())[0]
785 self.assertTrue(post.published)
786 self.assertEqual(post.title, "With BOM")
787
788 def test_media_symlinks_are_not_followed(self):
789 t = Tree(self, {"media": "media"})
790 t.file("media/real.png", "png")
791 outside = os.path.join(t.dir, "outside")
792 t.file("private.png", "PRIVATE-IMAGE", base=outside)
793 t.file("secret/other.png", "PRIVATE-DIR-IMAGE", base=outside)
794 os.symlink(os.path.join(outside, "private.png"), os.path.join(t.src, "media", "link.png"))
795 os.symlink(os.path.join(outside, "secret"), os.path.join(t.src, "media", "linked-dir"))
796 t.file("2018-03-02-x.md", "---\npublished: true\n---\n# X\n\nBody.\n")
797 _, out = t.build()
798 self.assertTrue(os.path.exists(os.path.join(out, "real.png")))
799 self.assertFalse(os.path.lexists(os.path.join(out, "link.png")))
800 self.assertFalse(os.path.lexists(os.path.join(out, "linked-dir")))
801 self.assertNotIn("PRIVATE", all_output(out))
802
803 def test_fw_blog_build_passes_options_through_and_rejects_unknown_ones(self):
804 import subprocess
805 fw = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bin", "fw")
806 env = dict(os.environ, FW_PYLIB=blog.gen.PYLIB, TMPDIR=tempfile.mkdtemp())
807 self.addCleanup(__import__("shutil").rmtree, env["TMPDIR"], True)
808 if subprocess.run(["python3", "-c", "import markdown, pygments"], env=dict(env, PYTHONPATH=blog.gen.PYLIB)).returncode:
809 self.skipTest("python3 on the PATH cannot import Markdown and Pygments from FW_PYLIB")
810 t = Tree(self)
811 t.file("2018-03-02-x.md", "---\npublished: true\n---\n# X\n\nBody.\n")
812 t.cfg()
813 config = os.path.join(t.dir, "fw-blog.json")
814 out = os.path.join(t.dir, "built")
815 run = lambda *args: subprocess.run(["bash", fw, "blog", "build", config, *args], env=env, capture_output=True, text=True)
816 r = run("--out", out, "--no-htaccess")
817 self.assertEqual(r.returncode, 0, r.stderr)
818 self.assertTrue(os.path.exists(os.path.join(out, "index.html")))
819 self.assertFalse(os.path.exists(os.path.join(out, ".htaccess")))
820 r = run("--out", out)
821 self.assertEqual(r.returncode, 0, r.stderr)
822 self.assertTrue(os.path.exists(os.path.join(out, ".htaccess")))
823 r = run("--out", out, "--bogus")
824 self.assertEqual(r.returncode, 2)
825 self.assertIn("unknown option --bogus", r.stderr)
826
827
828if __name__ == "__main__":
829 unittest.main()