fuzzyworld-tools / commit
One nav source (gen.NAV) for landing page and blog; runner timeouts, quieter log, tests
README.md | 5 +- runner/fw-runner.py | 26 +++++-- server/blog.py | 2 + server/content/projects.md | 4 +- server/gen.py | 18 +++++ server/install.sh | 3 +- server/runner-api | 6 +- server/site/index.html | 2 +- tests/test_blog.py | 31 ++++++++ tests/test_runner.py | 177 +++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 258 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e8c94e8..21fcac9 100644 --- a/README.md +++ b/README.md @@ -300,7 +300,8 @@ fw blog deploy ~/Blog/fw-blog.json --dry-run # then, for real: --yes (or answer top of `server/blog.py`. `source` and `comments` are relative to the config file's folder; `pages_dir` and `media` are relative to `source`. `site` puts the site-wide bar above the blog's own bar (as on code and apps): `{"name": "fuzzyworld", "url": "https://fuzzyworld.net/", "nav": [[label, url], ...], "current": "Writing"}`; - the nav list must be kept in step with `NAV` in `server/gen.py` and `server/site/index.html` by hand. + `"nav": "gen"` takes the fuzzyworld nav from `NAV` in `server/gen.py`, the one source (the landing page's nav is + built from it too, by `gen.py landing` in `install.sh`). - **Output:** posts at `/YYYY/MM/DD/<slug>/`, `/blog/` (every post by year and month, with excerpts and a "Continue reading" link), `/tag/<slug>/`, pages, an Atom feed at `/feed/` (`server/atom.py`), `.htaccess` (redirects from `redirects` and `wp_ids`, no script execution, a security policy that allows only the site's own `blog.js`), @@ -317,7 +318,7 @@ fw blog deploy ~/Blog/fw-blog.json --dry-run # then, for real: --yes (or answer media and the config entries. `--dry-run` writes only a cache and `wordpress/_report.txt`; a second run changes nothing. `fw blog check` lists posts whose built text differs from WordPress. - Python-Markdown and Pygments are installed into `~/fw/pylib` on the laptop the first time `fw blog` runs. -- Tests: `python3 -m unittest discover tests`. +- Tests: `python3 -m unittest discover tests` (the blog, the WordPress importer, and the runner with a throwaway-`HOME` run of `server/runner-api`). ## VS Code in the browser diff --git a/runner/fw-runner.py b/runner/fw-runner.py index 7d2d08c..4b9a120 100755 --- a/runner/fw-runner.py +++ b/runner/fw-runner.py @@ -80,6 +80,8 @@ SHOOT_IMAGE = f"fw-shoot:{PLAYWRIGHT}" # `fw deploy --dry-run` from the published fuzzyworld-tools, whether or not the app depends on it. FW_DEPLOY_CHECK = "npx --yes --package=git+https://code.fuzzyworld.net/fuzzyworld-tools.git#semver:^1.0.0 fw deploy --here --dry-run" BEAT_EVERY = 30 # seconds between heartbeats; DreamHost gives a job back after ~2 minutes without one +BEAT_TIMEOUT = 20 # seconds one heartbeat may take: the lease is ~2 minutes, so a hung ssh must not eat it +DOCKER_INFO_TIMEOUT = 20 # seconds `docker info` may take (a wedged Docker daemon hangs it) LOG_EVERY = 15 # seconds between uploads of a running job's log (live CI pages) JOB_TIMEOUT = 30 * 60 # seconds, unless the job sets `timeout` MAX_LOG = 20 * 1024 * 1024 # runner-api refuses bigger files @@ -113,8 +115,8 @@ def now_iso(): # ---------------------------------------------------------------- DreamHost API -def api(*args, data=None): - r = subprocess.run(SSH + [HOST, " ".join(args)], input=data, capture_output=True, timeout=300) +def api(*args, data=None, timeout=300): + r = subprocess.run(SSH + [HOST, " ".join(args)], input=data, capture_output=True, timeout=timeout) if r.returncode != 0: raise RuntimeError(f"runner-api {args[0]}: {r.stderr.decode(errors='replace').strip()}") return r.stdout @@ -722,7 +724,7 @@ class Heartbeat(threading.Thread): def run(self): while not self.stopped.wait(BEAT_EVERY): try: - api("beat", self.job_id, RUNNER) + api("beat", self.job_id, RUNNER, timeout=BEAT_TIMEOUT) except Exception as e: # a network blip: the lease is ~2 minutes, so the next beat may still be in time log(f"heartbeat failed: {e}") @@ -730,13 +732,18 @@ class Heartbeat(threading.Thread): self.stopped.set() +def parse_top_idle(out): + """The idle percentage from the last "CPU usage:" line of macOS `top -l 2` output, or None.""" + m = re.findall(r"CPU usage:.*?([\d.]+)% idle", out) + return float(m[-1]) if m else None + + def cpu_idle_percent(): """Percent of CPU idle over the last second or so, or None if this system doesn't say.""" try: if sys.platform == "darwin": out = subprocess.run(["top", "-l", "2", "-n", "0", "-s", "1"], capture_output=True, text=True, timeout=20).stdout - m = re.findall(r"CPU usage:.*?([\d.]+)% idle", out) - return float(m[-1]) if m else None + return parse_top_idle(out) def times(): f = [int(x) for x in open("/proc/stat").readline().split()[1:]] return f[3] + f[4], sum(f) # idle + iowait, total @@ -765,8 +772,13 @@ def busy_reason(): """Why a polite runner should not claim a job right now, or "" if it may.""" if not on_ac_power(): return "on battery" - if subprocess.run(["docker", "info"], capture_output=True).returncode != 0: - return "Docker is not running" + try: + if subprocess.run(["docker", "info"], capture_output=True, timeout=DOCKER_INFO_TIMEOUT).returncode != 0: + return "Docker is not running" + except subprocess.TimeoutExpired: + return "Docker is not answering" + except FileNotFoundError: + return "Docker is not installed" idle = cpu_idle_percent() if idle is not None and idle < MIN_IDLE: return f"CPU busy ({100 - idle:.0f}% used)" diff --git a/server/blog.py b/server/blog.py index 182b05d..b38cbb2 100644 --- a/server/blog.py +++ b/server/blog.py @@ -116,6 +116,8 @@ def load_config(path): raise BlogError("nav must be a list of [label, path] pairs") site = cfg["site"] if site is not None: + if isinstance(site, dict) and site.get("nav") == "gen": + site = cfg["site"] = {**site, "nav": gen.site_nav()} # "gen": the fuzzyworld nav, built from NAV in gen.py if not (isinstance(site, dict) and set(site) <= {"name", "url", "nav", "current"} and {"name", "url", "nav"} <= set(site) and isinstance(site["name"], str) and isinstance(site["url"], str) and isinstance(site["nav"], list) and isinstance(site.get("current", ""), str) diff --git a/server/content/projects.md b/server/content/projects.md index 96b3369..babb031 100644 --- a/server/content/projects.md +++ b/server/content/projects.md @@ -12,11 +12,11 @@ An endless skate across an endless lake of thin black ice, for a phone. The ice ## [auto-print](https://code.fuzzyworld.net/auto-print/) -A command-line tool that turns markdown manuscripts into books: an epub, a docx, a pdf, and a Kindle kpf for each one. It's made for writers who keep their drafts in markdown next to their notes — it publishes only the story, plus any extra pages you name (a dedication, a character guide), and never changes the source files. One config file lists the books, and one command builds all of them, using [pandoc](https://pandoc.org) under the hood. I built it to turn my [upcoming writing](https://blog.fuzzyworld.net/upcoming/) into books. +A command-line tool that turns markdown manuscripts into books: an epub, a docx, a pdf, and a Kindle kpf for each one. It's made for writers who keep their drafts in markdown next to their notes — it publishes only the story, plus any extra pages you name (a dedication, a character guide), and never changes the source files. One config file lists the books, and one command builds all of them, using [pandoc](https://pandoc.org) under the hood. I built it to turn my [upcoming writing](https://writing.fuzzyworld.net/upcoming/) into books. ## 🛰 [Tower](https://apps.fuzzyworld.net/tower/) -The companion website for [Conversations with Aliens](https://blog.fuzzyworld.net/upcoming/), a novel I'm working on. Tower is the story's fictional space station: a spinning cylinder orbiting Earth, with an inner surface the size of Wisconsin, lit by sunlight piped down its axis. Browse the unrolled surface map, view Tower from outside, or stand on its inner surface with the ground curving up on both sides and the far side of the world hanging 100 km overhead. The world generates itself live in the browser, complete with weather, a day/night cycle, railways, and a scenic tour. +The companion website for [Conversations with Aliens](https://writing.fuzzyworld.net/upcoming/), a novel I'm working on. Tower is the story's fictional space station: a spinning cylinder orbiting Earth, with an inner surface the size of Wisconsin, lit by sunlight piped down its axis. Browse the unrolled surface map, view Tower from outside, or stand on its inner surface with the ground curving up on both sides and the far side of the world hanging 100 km overhead. The world generates itself live in the browser, complete with weather, a day/night cycle, railways, and a scenic tour. ## <img src="https://apps.fuzzyworld.net/galaxy-z-flip7-butterchurn/icon-192.png" class="small-logo" alt=""> [BlueShadow](https://apps.fuzzyworld.net/galaxy-z-flip7-butterchurn/) diff --git a/server/gen.py b/server/gen.py index 26d86dc..c980f54 100755 --- a/server/gen.py +++ b/server/gen.py @@ -4,6 +4,7 @@ gen.py repo <name> build or update the pages of one public repo gen.py index rebuild the site index (public repos only) gen.py apps rebuild the apps.fuzzyworld.net index from deployed apps + gen.py landing <src> <dest> write the landing page: src with <!--SITE-NAV--> filled in from NAV gen.py content render server/content/<name>.md to code.fuzzyworld.net/<name>/ (site pages written in Markdown) gen.py ci <name> rebuild one public repo's CI pages, then the CI dashboard gen.py ci-dashboard rebuild code.fuzzyworld.net/ci/ (queue, running, recent; public repos only) @@ -152,6 +153,21 @@ WRITING_URL = "https://writing.fuzzyworld.net" # a static site built by `fw bl SITE_URLS = {"code": BASE_URL, "apps": APPS_URL, "writing": WRITING_URL} +def site_nav(): + """NAV as [label, absolute URL] pairs: the one source for the nav on the landing page and the blog's `site` bar.""" + return [[label, SITE_URLS[st] + path] for _, label, st, path in NAV] + + +def build_landing(src, dest): + """Writes the landing page: `src` with <!--SITE-NAV--> replaced by the nav links built from NAV.""" + links = "".join(f'<a href="{html.escape(url, quote=True)}">{html.escape(label)}</a>' for label, url in site_nav()) + with open(src, encoding="utf-8") as f: + text = f.read() + if "<!--SITE-NAV-->" not in text: + raise SystemExit(f"{src}: no <!--SITE-NAV--> placeholder") + write(dest, text.replace("<!--SITE-NAV-->", links)) + + def site_link(site, path, here): """Links within a site stay relative; links to the other site are absolute.""" return path if site == here else SITE_URLS[site] + path @@ -1463,6 +1479,8 @@ def main(argv): build_index() elif len(argv) == 2 and argv[1] == "apps": build_apps_index() + elif len(argv) == 4 and argv[1] == "landing": + build_landing(argv[2], argv[3]) elif len(argv) == 2 and argv[1] == "content": build_content() elif len(argv) == 3 and argv[1] == "ci": diff --git a/server/install.sh b/server/install.sh index 1c1fcd6..a17e9bf 100755 --- a/server/install.sh +++ b/server/install.sh @@ -33,7 +33,8 @@ cp ~/fw/tools/server/site/_shared.css ~/fw/tools/server/site/_icons.css ~/fw/too chmod 644 "$apps"/_shared.css "$apps"/_icons.css "$apps"/style.css "$apps"/favicon.ico # Main fuzzyworld.net site main_site=~/fuzzyworld.net -cp ~/fw/tools/server/site/index.html ~/fw/tools/server/site/main.css "$main_site/" +python3 ~/fw/tools/server/gen.py landing ~/fw/tools/server/site/index.html "$main_site/index.html" # nav built from NAV in gen.py +cp ~/fw/tools/server/site/main.css "$main_site/" mkdir -p "$main_site/projects" cp ~/fw/tools/server/site/projects-redirect.html "$main_site/projects/index.html" # the old projects page, now /projects/ on code. chmod 644 "$main_site/projects/index.html" diff --git a/server/runner-api b/server/runner-api index d92fbf0..77fc3e5 100755 --- a/server/runner-api +++ b/server/runner-api @@ -18,7 +18,7 @@ # put-shot <app> <file> store one screenshot (stdin) in apps.fuzzyworld.net/_shots/<app>/ # shots-done <app> <repo> refresh the apps index and the repo's page # Anything else is refused. Every call is logged to ~/fw/logs/runner-api.log, except a claim that -# finds nothing (runners ask every few minutes). A claim also records the runner in ~/fw/runners/<runner> +# finds nothing (runners ask every few minutes) and heartbeats. A claim also records the runner in ~/fw/runners/<runner> # (its file's time is when it last checked in; the CI dashboard says "waiting for a runner" when none has). set -u cmd=${SSH_ORIGINAL_COMMAND:-} @@ -31,7 +31,7 @@ MAX_FILE=$((20 * 1024 * 1024)) log="$HOME/fw/logs/runner-api.log" mkdir -p "$(dirname "$log")" "$QUEUE/pending" "$QUEUE/running" "$QUEUE/declined" "$RUNNERS" note() { echo "$(date -u '+%F %T') ${SSH_CLIENT%% *} $*" >> "$log"; } -[[ "$cmd" == claim* ]] || note "$cmd" +[[ "$cmd" == claim* || "$cmd" == beat* ]] || note "$cmd" # not the polling and heartbeats (they would fill the log) die() { echo "runner-api: $*" >&2; exit 1; } # Page updates run in the background, so the runner never waits for them. @@ -85,7 +85,7 @@ case "${w[0]:-}" in job=${w[1]:-} runner=${w[2]:-} valid_job "$job" && valid_name "$runner" || die "usage: beat <job-id> <runner>" [ -e "$QUEUE/running/$job.json" ] && [ "$(cat "$QUEUE/running/$job.beat" 2>/dev/null)" = "$runner" ] \ - || die "job $job is not running on $runner (it was put back in the queue)" + || { note "$cmd refused (not running on $runner)"; die "job $job is not running on $runner (it was put back in the queue)"; } touch "$QUEUE/running/$job.beat" ;; release) job=${w[1]:-} runner=${w[2]:-} diff --git a/server/site/index.html b/server/site/index.html index 09241ba..b266c40 100644 --- a/server/site/index.html +++ b/server/site/index.html @@ -8,7 +8,7 @@ <link rel="stylesheet" href="/main.css"> </head> <body> -<header class="site"><div class="bar"><a class="brand" href="/">fuzzyworld</a><nav class="site-nav" aria-label="Site"><a href="https://code.fuzzyworld.net/">Code</a><a href="https://apps.fuzzyworld.net/">Apps</a><a href="https://writing.fuzzyworld.net/">Writing</a><a href="https://code.fuzzyworld.net/projects/">Projects</a><a href="https://code.fuzzyworld.net/my-ai-use/">Agentic</a><a href="https://code.fuzzyworld.net/ci/">CI</a><a href="https://code.fuzzyworld.net/contribute/">Contribute</a></nav></div></header> +<header class="site"><div class="bar"><a class="brand" href="/">fuzzyworld</a><nav class="site-nav" aria-label="Site"><!--SITE-NAV--></nav></div></header> <main> <h1>Welcome to Pavel Safronov's website</h1> <p>I'm a software engineer who dabbles in fiction writing.</p> diff --git a/tests/test_blog.py b/tests/test_blog.py index 979c513..42620dd 100644 --- a/tests/test_blog.py +++ b/tests/test_blog.py @@ -13,6 +13,7 @@ import xml.etree.ElementTree as ET sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "server")) import atom # noqa: E402 import blog # noqa: E402 +import gen # noqa: E402 needs_markdown = unittest.skipUnless(blog.markdown, "Python-Markdown is not installed (see FW_PYLIB)") @@ -358,6 +359,36 @@ class Build(unittest.TestCase): with self.assertRaises(blog.BlogError): Tree(self, {"site": bad}).cfg() + def test_site_nav_gen_uses_the_fuzzyworld_nav(self): + t = self.blog_tree(site={"name": "fuzzyworld", "url": "https://fuzzyworld.net/", "nav": "gen", "current": "Writing"}) + self.assertEqual(t.cfg()["site"]["nav"], gen.site_nav()) + _, out = t.build() + page = t.read(out, "2018/03/02/first-post/index.html") + for label, url in gen.site_nav(): + self.assertIn(f'<a href="{url}"', page) + self.assertIn(f">{label}</a>", page) + + def test_landing_page_nav_comes_from_nav(self): + import tempfile + with tempfile.TemporaryDirectory() as d: + src, dest = os.path.join(d, "in.html"), os.path.join(d, "out.html") + with open(src, "w") as f: + f.write('<nav><!--SITE-NAV--></nav>') + gen.build_landing(src, dest) + with open(dest) as f: + text = f.read() + self.assertNotIn("SITE-NAV", text) + for label, url in gen.site_nav(): + self.assertIn(f'<a href="{url}">{label}</a>', text) + + def test_landing_page_without_the_placeholder_is_an_error(self): + with tempfile.TemporaryDirectory() as d: + src = os.path.join(d, "in.html") + with open(src, "w") as f: + f.write("<nav></nav>") + with self.assertRaises(SystemExit): + gen.build_landing(src, os.path.join(d, "out.html")) + def test_index_pages_tags_and_home(self): t = self.blog_tree(pages_dir="pages", home="start", tags={"fiction": "Fiction"}, feed={"path": "/feed/", "entries": 1}) t.file("pages/start.md", "---\npublished: true\ntitle: Start Here\n---\nWelcome.\n") diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..21c5712 --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,177 @@ +"""Tests for the CI runner (runner/fw-runner.py) and its server side (server/runner-api). + +Run with: python3 -m unittest discover tests +""" + +import importlib.util +import json +import os +import subprocess +import tempfile +import time +import unittest +from unittest import mock + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +spec = importlib.util.spec_from_file_location("fw_runner", os.path.join(ROOT, "runner", "fw-runner.py")) +fw_runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fw_runner) + +API = os.path.join(ROOT, "server", "runner-api") +JOB = "1700000000-demo-0123456789ab" +SHA = "a" * 40 + + +class MissingTags(unittest.TestCase): + def need(self, tags, ref="refs/heads/main", protected=True, has_secrets=False, have=("docker", "chrome")): + jobs = [{"tags": tags}] + with mock.patch.object(fw_runner, "TAGS", set(have)), mock.patch.object(fw_runner, "shot_plan", return_value=None): + return fw_runner.missing_tags({"ref": ref}, jobs, "/nowhere", protected, has_secrets) + + def test_nothing_missing(self): + self.assertEqual(self.need(["docker"]), []) + + def test_lists_missing_tags_sorted(self): + self.assertEqual(self.need(["zeta", "alpha", "docker"]), ["alpha", "zeta"]) + + def test_screenshots_need_chrome_only_on_a_protected_branch(self): + jobs = [{"tags": []}] + with mock.patch.object(fw_runner, "TAGS", {"docker"}), mock.patch.object(fw_runner, "shot_plan", return_value=({}, "app", [])): + self.assertEqual(fw_runner.missing_tags({"ref": "refs/heads/main"}, jobs, "/x", True, False), ["chrome"]) + self.assertEqual(fw_runner.missing_tags({"ref": "refs/tags/v1"}, jobs, "/x", True, False), []) + self.assertEqual(fw_runner.missing_tags({"ref": "refs/heads/main"}, jobs, "/x", False, False), []) + + def test_secrets_is_not_a_runner_tag(self): + self.assertEqual(self.need(["secrets"], has_secrets=True), []) + self.assertEqual(self.need(["secrets"], has_secrets=False), ["secrets"]) + self.assertEqual(self.need(["secrets"], protected=False, has_secrets=False), []) # off the default branch: never has them + + +class TopParsing(unittest.TestCase): + def test_last_sample_wins(self): + out = ("CPU usage: 10.0% user, 5.0% sys, 85.0% idle\n" + + "CPU usage: 30.5% user, 9.5% sys, 60.0% idle\n") + self.assertEqual(fw_runner.parse_top_idle(out), 60.0) + + def test_no_line(self): + self.assertIsNone(fw_runner.parse_top_idle("nothing useful")) + + +class BusyReason(unittest.TestCase): + def busy(self, run): + with mock.patch.object(fw_runner, "on_ac_power", return_value=True), \ + mock.patch.object(fw_runner, "cpu_idle_percent", return_value=99.0), \ + mock.patch.object(fw_runner.subprocess, "run", run): + return fw_runner.busy_reason() + + def test_docker_ok(self): + self.assertEqual(self.busy(lambda *a, **k: subprocess.CompletedProcess(a, 0)), "") + + def test_docker_down(self): + self.assertEqual(self.busy(lambda *a, **k: subprocess.CompletedProcess(a, 1)), "Docker is not running") + + def test_docker_hangs(self): + def hang(*a, **k): + self.assertEqual(k.get("timeout"), fw_runner.DOCKER_INFO_TIMEOUT) + raise subprocess.TimeoutExpired(a, k["timeout"]) + self.assertEqual(self.busy(hang), "Docker is not answering") + + def test_docker_missing(self): + def missing(*a, **k): + raise FileNotFoundError("docker") + self.assertEqual(self.busy(missing), "Docker is not installed") + + +class Heartbeat(unittest.TestCase): + def test_beat_has_a_short_timeout(self): + seen = {} + def fake_run(cmd, **k): + seen["timeout"] = k["timeout"] + return subprocess.CompletedProcess(cmd, 0, b"", b"") + with mock.patch.object(fw_runner.subprocess, "run", fake_run): + fw_runner.api("beat", JOB, "r1", timeout=fw_runner.BEAT_TIMEOUT) + self.assertEqual(seen["timeout"], fw_runner.BEAT_TIMEOUT) + self.assertLess(fw_runner.BEAT_TIMEOUT, 60) + + + def test_heartbeat_thread_passes_the_timeout(self): + calls = [] + hb = fw_runner.Heartbeat(JOB) + def fake_api(*args, **kw): + calls.append((args, kw)) + hb.stop() + with mock.patch.object(fw_runner, "BEAT_EVERY", 0.01), mock.patch.object(fw_runner, "api", fake_api): + hb.start() + hb.join(5) + self.assertEqual(calls[0][1], {"timeout": fw_runner.BEAT_TIMEOUT}) + self.assertEqual(calls[0][0][0], "beat") + + +class RunnerApi(unittest.TestCase): + """server/runner-api run for real against a throwaway HOME.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.home = self.tmp.name + self.addCleanup(self.tmp.cleanup) + pending = os.path.join(self.home, "fw", "queue", "pending") + os.makedirs(pending) + with open(os.path.join(pending, JOB + ".json"), "w") as f: + json.dump({"id": JOB}, f) + + def api(self, cmd): + env = {"PATH": os.environ["PATH"], "HOME": self.home, "SSH_ORIGINAL_COMMAND": cmd, "SSH_CLIENT": "127.0.0.1 1 2"} + return subprocess.run(["bash", API], env=env, capture_output=True, text=True) + + def path(self, *p): + return os.path.join(self.home, "fw", "queue", *p) + + def log(self): + try: + with open(os.path.join(self.home, "fw", "logs", "runner-api.log")) as f: + return f.read() + except OSError: + return "" + + def test_claim_beat_done(self): + r = self.api("claim r1 docker") + self.assertEqual(json.loads(r.stdout), {"id": JOB}) + self.assertTrue(os.path.exists(self.path("running", JOB + ".json"))) + self.assertEqual(self.api(f"beat {JOB} r1").returncode, 0) + self.assertNotEqual(self.api(f"beat {JOB} other").returncode, 0) # not the holder + self.assertNotEqual(self.api(f"done demo {SHA} {JOB} other").returncode, 0) + self.assertTrue(os.path.exists(self.path("running", JOB + ".json"))) # the stale caller changed nothing + + def test_release_declines_for_that_runner_only(self): + self.api("claim r1 docker") + self.assertEqual(self.api(f"release {JOB} r1").returncode, 0) + self.assertTrue(os.path.exists(self.path("pending", JOB + ".json"))) + self.assertEqual(self.api("claim r1 docker").stdout, "") # r1 is not offered it again + self.assertEqual(json.loads(self.api("claim r2 docker").stdout), {"id": JOB}) + + def test_quiet_runner_loses_the_lease(self): + self.api("claim r1 docker") + old = time.time() - 10 * 60 + os.utime(self.path("running", JOB + ".beat"), (old, old)) + r = self.api("claim r2 docker") # the job goes back, and r2 takes it + self.assertEqual(json.loads(r.stdout), {"id": JOB}) + self.assertNotEqual(self.api(f"beat {JOB} r1").returncode, 0) # r1 finds out on its next beat + with open(self.path("running", JOB + ".beat")) as f: + self.assertEqual(f.read().strip(), "r2") + + def test_refused_beat_is_logged(self): + self.api("claim r1 docker") + self.assertNotEqual(self.api(f"beat {JOB} other").returncode, 0) + self.assertIn("refused", self.log()) + + def test_beats_and_empty_claims_are_not_logged(self): + self.api("claim r1 docker") + self.api(f"beat {JOB} r1") + self.api("claim r1 docker") + log = self.log() + self.assertNotIn("beat", log) + self.assertIn("claim r1 docker: " + JOB + ".json", log) # a successful claim is + + +if __name__ == "__main__": + unittest.main()