fuzzyworld

fuzzyworld-tools / files / server / runner-api

runner-api

7.9 KB · history · blame

1#!/bin/bash
2# Forced command for CI runner keys in ~/.ssh/authorized_keys on DreamHost:
3#
4# restrict,command="/home/fuzzygamer/fw/tools/server/runner-api" ssh-ed25519 AAAA... truenas-runner
5#
6# A runner (runner/fw-runner.py) can then only:
7# claim <runner> [tags] take the oldest queued job this runner hasn't declined; prints its
8# JSON, or nothing. `tags` (comma-separated) is what the runner can do.
9# beat <job> <runner> "still working on it": a job whose runner is quiet for ~2 minutes
10# (lid closed, network gone) goes back in the queue
11# release <job> <runner> "I can't run this one" (missing tag or secrets): back in the queue,
12# and this runner is not offered it again
13# git-upload-pack '<repo>' clone/fetch ~/git/<name>.git (the job's source)
14# put <repo> <sha> <file> store one result file (stdin) in ~/fw/ci/<repo>/<sha>/
15# done <repo> <sha> <job-id> [runner]
16# the job is finished: remove it from the queue, refresh pages. With
17# `runner`, refused if the job was meanwhile given to another runner
18# put-shot <app> <file> store one screenshot (stdin) in apps.fuzzyworld.net/_shots/<app>/
19# shots-done <app> <repo> refresh the apps index and the repo's page
20# Anything else is refused. Every call is logged to ~/fw/logs/runner-api.log, except a claim that
21# finds nothing (runners ask every few minutes) and heartbeats. A claim also records the runner in ~/fw/runners/<runner>
22# (its file's time is when it last checked in; the CI dashboard says "waiting for a runner" when none has).
23set -u
24cmd=${SSH_ORIGINAL_COMMAND:-}
25QUEUE=$HOME/fw/queue
26RUNNERS=$HOME/fw/runners
27LEASE_MIN=2 # a job whose runner has not sent a heartbeat for this long goes back to pending/
28CI=$HOME/fw/ci
29SHOTS=$HOME/apps.fuzzyworld.net/_shots
30MAX_FILE=$((20 * 1024 * 1024))
31log="$HOME/fw/logs/runner-api.log"
32mkdir -p "$(dirname "$log")" "$QUEUE/pending" "$QUEUE/running" "$QUEUE/declined" "$RUNNERS"
33note() { echo "$(date -u '+%F %T') ${SSH_CLIENT%% *} $*" >> "$log"; }
34[[ "$cmd" == claim* || "$cmd" == beat* ]] || note "$cmd" # not the polling and heartbeats (they would fill the log)
35
36die() { echo "runner-api: $*" >&2; exit 1; }
37# Page updates run in the background, so the runner never waits for them.
38bg() { setsid nohup "$@" >> "$HOME/fw/logs/runner-api-pages.log" 2>&1 < /dev/null & }
39valid_name() { [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] && [[ "$1" != *.git ]]; }
40valid_repo() { valid_name "$1" && [ -d "$HOME/git/$1.git" ]; }
41valid_sha() { [[ "$1" =~ ^[0-9a-f]{40}$ ]]; }
42valid_tags() { [[ "$1" =~ ^[A-Za-z0-9,_-]*$ ]]; }
43valid_job() { [[ "$1" =~ ^[0-9]+-[A-Za-z0-9._-]+-[0-9a-f]{12}$ ]]; }
44
45# stdin to a file, refusing anything over MAX_FILE; written beside the target, then renamed.
46store() {
47 local dest=$1 tmp="$1.part"
48 head -c $((MAX_FILE + 1)) > "$tmp"
49 if [ "$(stat -c %s "$tmp")" -gt "$MAX_FILE" ]; then rm -f "$tmp"; die "file too large"; fi
50 mv "$tmp" "$dest"
51}
52
53read -r -a w <<< "$cmd"
54case "${w[0]:-}" in
55 claim)
56 runner=${w[1]:-} tags=${w[2]:-}
57 valid_name "$runner" && valid_tags "$tags" || die "usage: claim <runner> [tag,tag,...]"
58 echo "$tags" > "$RUNNERS/$runner"
59 # Jobs whose runner went quiet go back in the queue. A runner that sends heartbeats (it claimed with
60 # tags) has a <job>.beat file, and is given LEASE_MIN minutes; an older runner is given 2 hours.
61 for f in "$QUEUE"/running/*.json; do
62 [ -e "$f" ] || continue
63 id=$(basename "$f" .json)
64 if [ -e "$QUEUE/running/$id.beat" ]; then
65 [ -n "$(find "$QUEUE/running/$id.beat" -mmin +$LEASE_MIN)" ] || continue
66 else
67 [ -n "$(find "$f" -mmin +120)" ] || continue
68 fi
69 mv "$f" "$QUEUE/pending/" 2>/dev/null && { rm -f "$QUEUE/running/$id.beat"; note "requeued $id (its runner went quiet)"; }
70 done
71 for f in $(ls "$QUEUE/pending" 2>/dev/null | sort); do
72 id=${f%.json}
73 [ -e "$QUEUE/declined/$id.$runner" ] && continue
74 # rename(2) is atomic: if two runners race for the same job, only one mv succeeds.
75 if mv "$QUEUE/pending/$f" "$QUEUE/running/$f" 2>/dev/null; then
76 touch "$QUEUE/running/$f"
77 [ -n "$tags" ] && echo "$runner" > "$QUEUE/running/$id.beat"
78 note "$cmd: $f"
79 bg "$HOME/bin/fw" ci-dashboard
80 cat "$QUEUE/running/$f"
81 exit 0
82 fi
83 done ;;
84 beat)
85 job=${w[1]:-} runner=${w[2]:-}
86 valid_job "$job" && valid_name "$runner" || die "usage: beat <job-id> <runner>"
87 [ -e "$QUEUE/running/$job.json" ] && [ "$(cat "$QUEUE/running/$job.beat" 2>/dev/null)" = "$runner" ] \
88 || { note "$cmd refused (not running on $runner)"; die "job $job is not running on $runner (it was put back in the queue)"; }
89 touch "$QUEUE/running/$job.beat" ;;
90 release)
91 job=${w[1]:-} runner=${w[2]:-}
92 valid_job "$job" && valid_name "$runner" || die "usage: release <job-id> <runner>"
93 [ -e "$QUEUE/running/$job.json" ] && [ "$(cat "$QUEUE/running/$job.beat" 2>/dev/null)" = "$runner" ] \
94 || die "job $job is not running on $runner"
95 : > "$QUEUE/declined/$job.$runner"
96 mv "$QUEUE/running/$job.json" "$QUEUE/pending/$job.json" && rm -f "$QUEUE/running/$job.beat"
97 bg "$HOME/bin/fw" ci-dashboard ;;
98 git-upload-pack)
99 repo=${cmd#git-upload-pack \'}; repo=${repo%\'}; repo=${repo#/home/fuzzygamer/}; repo=${repo#\~/}
100 [[ "$repo" =~ ^git/([A-Za-z0-9][A-Za-z0-9._-]*)\.git$ ]] && valid_repo "${BASH_REMATCH[1]}" || die "no such repo"
101 exec git-upload-pack "$HOME/$repo" ;;
102 put)
103 repo=${w[1]:-} sha=${w[2]:-} file=${w[3]:-}
104 valid_repo "$repo" && valid_sha "$sha" && [[ "$file" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*\.(log|json)$ ]] || die "usage: put <repo> <sha> <file.log|file.json>"
105 mkdir -p "$CI/$repo/$sha"
106 store "$CI/$repo/$sha/$file"
107 # A finished pipeline refreshes all the repo's pages (✓/✗ on its log and summary); anything else while
108 # it runs (a job starting, a growing log) only its CI pages and the dashboard. Both are locked per repo.
109 if [ "$file" = pipeline.json ] && ! grep -q '"status": "running"' "$CI/$repo/$sha/pipeline.json"; then
110 bg "$HOME/bin/fw" publish "$repo"
111 else
112 bg "$HOME/bin/fw" ci-pages "$repo"
113 fi ;;
114 done)
115 repo=${w[1]:-} sha=${w[2]:-} job=${w[3]:-} runner=${w[4]:-}
116 valid_repo "$repo" && valid_sha "$sha" && valid_job "$job" && { [ -z "$runner" ] || valid_name "$runner"; } \
117 || die "usage: done <repo> <sha> <job-id> [runner]"
118 # A runner that lost its lease (laptop slept, job re-claimed by another runner) must not remove the new holder's claim.
119 holder=$(cat "$QUEUE/running/$job.beat" 2>/dev/null)
120 # (Every current runner has a .beat file, even without tags; only an old runner, which sends no runner name, has none.)
121 [ -z "$runner" ] || [ "$holder" = "$runner" ] || die "job $job is not held by $runner (held by: ${holder:-nobody})"
122 rm -f "$QUEUE/running/$job.json" "$QUEUE/running/$job.beat" "$QUEUE/declined/$job".*
123 bg "$HOME/bin/fw" ci-dashboard ;;
124 put-shot)
125 app=${w[1]:-} file=${w[2]:-}
126 valid_name "$app" && [[ "$file" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*\.(png|json)$ ]] || die "usage: put-shot <app> <file.png|shots.json>"
127 mkdir -p "$SHOTS/$app"
128 store "$SHOTS/$app/$file" ;;
129 shots-done)
130 app=${w[1]:-} repo=${w[2]:-}
131 valid_name "$app" && valid_repo "$repo" || die "usage: shots-done <app> <repo>"
132 # Screenshots from earlier runs that shots.json no longer lists are removed.
133 python3 - "$SHOTS/$app" <<'PY'
134import json, os, sys
135d = sys.argv[1]
136keep = {s["file"] for s in json.load(open(os.path.join(d, "shots.json"))).get("shots", [])}
137for f in os.listdir(d):
138 if f.endswith(".png") and f not in keep:
139 os.remove(os.path.join(d, f))
140PY
141 setsid nohup "$HOME/bin/fw" app-deployed "$app" "$repo" >> "$HOME/fw/logs/$repo.log" 2>&1 < /dev/null & ;;
142 *)
143 die "this key may only run CI jobs (refused: $cmd)" ;;
144esac