fuzzyworld

fuzzyworld-tools / files / server / fw

fw

10.2 KB · history · blame

1#!/bin/bash
2# fw: manage fuzzyworld git repos on DreamHost.
3#
4# fw list every repo with its visibility and last commit
5# fw new <name> [--public] [--description TEXT]
6# create ~/git/<name>.git (private unless --public)
7# fw public <name> publish source on code.fuzzyworld.net
8# fw private <name> unpublish: remove public clone copy and pages
9# fw describe <name> TEXT set the description shown on the index
10# fw publish <name> refresh public copy + pages (the push hook runs this)
11# fw publish-all publish every public repo, then the index
12# fw install-hooks point every repo at the shared hooks
13# fw apps-index rebuild the apps.fuzzyworld.net index
14# fw content render the Markdown site pages (server/content/*.md, e.g. /why/)
15# fw app-deployed <app> <repo> after `fw deploy`: apps index, plus the repo's code page
16# fw ci <name> [branch] queue a CI run of a branch's latest commit (default: the default branch)
17# fw ci-queue <name> <ref> <sha> queue a CI run (the push hook runs this)
18# fw ci-list queued and running CI jobs
19# fw ci-watch follow CI (all repos, private too) until nothing is queued or running
20# fw ci-pages <name> rebuild a repo's CI pages and the dashboard (runner-api runs this)
21# fw ci-dashboard rebuild code.fuzzyworld.net/ci/
22# fw comments <cmd> moderate blog comments (list, show, approve, reject, ... ingest, imap-check)
23# fw issues <repo> public|reviewed|private who sees a repo's bug-report issue pages (default private)
24# fw issues <cmd> moderate issues (list, show, record, approve, hide, close, reopen, publish, publish-all, ingest, imap-check)
25# fw deploys [--json] what is deployed where: the deploy ledger and each deployed app against its repo's HEAD
26#
27# Visibility is `git config fuzzyworld.visibility` inside each repo. Anything other
28# than exactly "public" is treated as private.
29set -euo pipefail
30
31GIT_ROOT=${FW_GIT_ROOT:-$HOME/git}
32SITE=${FW_CODE_SITE:-$HOME/code.fuzzyworld.net}
33TOOLS=${FW_TOOLS:-$HOME/fw/tools}
34STATE=${FW_STATE:-$HOME/fw/state}
35LOGS=${FW_LOGS:-$HOME/fw/logs}
36QUEUE=${FW_QUEUE:-$HOME/fw/queue}
37HOOKS=$TOOLS/server/hooks
38GEN=(python3 "$TOOLS/server/gen.py")
39
40die() { echo "fw: $*" >&2; exit 1; }
41
42valid_name() {
43 [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] && [[ "$1" != *.git ]] || die "invalid repo name: '$1'"
44}
45
46repo_dir() { valid_name "$1"; echo "$GIT_ROOT/$1.git"; }
47
48need_repo() { [ -d "$(repo_dir "$1")" ] || die "no such repo: $1"; }
49
50visibility() {
51 local v; v=$(git -C "$(repo_dir "$1")" config --get fuzzyworld.visibility || true)
52 [ "$v" = public ] && echo public || echo private
53}
54
55# Serialise work on one repo (a push and a manual publish could overlap).
56with_lock() {
57 local name=$1; shift
58 mkdir -p "$STATE/locks"
59 exec 9>"$STATE/locks/$name.lock"
60 flock -w 600 9 || die "timed out waiting for lock on $name"
61 "$@"
62}
63
64unpublish() {
65 local name=$1
66 valid_name "$name"
67 rm -rf -- "${SITE:?}/${name:?}" "${SITE:?}/${name:?}.git"
68 rm -f -- "${STATE:?}/${name:?}.json"
69}
70
71publish_one() {
72 local name=$1 src pub
73 src=$(repo_dir "$name"); pub="$SITE/$name.git"
74 if [ "$(visibility "$name")" != public ]; then
75 unpublish "$name"
76 echo "$name: private, not published"
77 return
78 fi
79 # Public clone copy for dumb-HTTP `git clone https://code.fuzzyworld.net/<name>.git`.
80 # Objects are hardlinked on first copy; later fetches add only new packs.
81 if [ ! -d "$pub" ]; then
82 git clone --quiet --mirror --local "$src" "$pub"
83 else
84 git -C "$pub" fetch --quiet --prune origin '+refs/*:refs/*'
85 fi
86 git -C "$pub" symbolic-ref HEAD "$(git -C "$src" symbolic-ref HEAD)"
87 git -C "$pub" update-server-info
88 # Issue pages (bug reports) live in the repo's folder; rebuilt under the lock we already hold.
89 python3 "$TOOLS/server/issues.py" publish "$name" --locked || echo "fw: issues publish failed for $name" >&2
90 "${GEN[@]}" repo "$name"
91}
92
93cmd_list() {
94 local r name
95 for r in "$GIT_ROOT"/*.git; do
96 [ -d "$r" ] || continue
97 name=$(basename "$r" .git)
98 printf '%-32s %-8s %s\n' "$name" "$(visibility "$name")" \
99 "$(git -C "$r" log -1 --format='%cs %s' HEAD 2>/dev/null | cut -c1-60 || echo '(empty)')"
100 done
101}
102
103cmd_new() {
104 local name="" vis=private desc=""
105 while [ $# -gt 0 ]; do
106 case "$1" in
107 --public) vis=public ;;
108 --description) desc=${2:-}; shift ;;
109 -*) die "unknown option $1" ;;
110 *) [ -z "$name" ] && name=$1 || die "unexpected argument $1" ;;
111 esac
112 shift
113 done
114 [ -n "$name" ] || die "usage: fw new <name> [--public] [--description TEXT]"
115 local r; r=$(repo_dir "$name")
116 [ -e "$r" ] && die "$name already exists"
117 git init --quiet --bare --initial-branch=main "$r"
118 git -C "$r" config fuzzyworld.visibility "$vis"
119 git -C "$r" config core.hooksPath "$HOOKS"
120 [ -n "$desc" ] && git -C "$r" config fuzzyworld.description "$desc"
121 echo "created $name ($vis)"
122 echo " push to: dh:git/$name.git"
123 [ "$vis" = public ] && echo " browse: https://code.fuzzyworld.net/$name/ (after the first push)"
124 return 0
125}
126
127cmd_set_visibility() {
128 local vis=$1 name=${2:-}
129 [ -n "$name" ] || die "usage: fw $vis <name>"
130 need_repo "$name"
131 git -C "$(repo_dir "$name")" config fuzzyworld.visibility "$vis"
132 with_lock "$name" publish_one "$name"
133 "${GEN[@]}" index
134}
135
136cmd_describe() {
137 local name=${1:-}; shift || true
138 [ -n "$name" ] && [ $# -gt 0 ] || die "usage: fw describe <name> TEXT"
139 need_repo "$name"
140 git -C "$(repo_dir "$name")" config fuzzyworld.description "$*"
141 printf '%s\n' "$*" > "$(repo_dir "$name")/description"
142 with_lock "$name" publish_one "$name"
143 "${GEN[@]}" index
144}
145
146cmd_publish() {
147 local name=${1:-}
148 [ -n "$name" ] || die "usage: fw publish <name>"
149 need_repo "$name"
150 with_lock "$name" publish_one "$name"
151 "${GEN[@]}" index
152}
153
154cmd_publish_all() {
155 local r name n=0 total started
156 total=$(ls -d "$GIT_ROOT"/*.git | wc -l | tr -d ' ')
157 for r in "$GIT_ROOT"/*.git; do
158 name=$(basename "$r" .git)
159 n=$((n + 1))
160 started=$SECONDS
161 ( with_lock "$name" publish_one "$name" ) || echo "fw: publish failed for $name" >&2
162 echo "fw: [$n/$total] $name ($((SECONDS - started))s)"
163 done
164 "${GEN[@]}" index
165 "${GEN[@]}" ci-dashboard
166}
167
168cmd_install_hooks() {
169 local r
170 for r in "$GIT_ROOT"/*.git; do git -C "$r" config core.hooksPath "$HOOKS"; done
171 echo "hooks installed on $(ls -d "$GIT_ROOT"/*.git | wc -l) repos"
172}
173
174cmd_app_deployed() {
175 local app=${1:-} name=${2:-}
176 valid_name "$app"
177 "${GEN[@]}" apps
178 # The repo's summary page links to its app; refresh it if the source is public.
179 if [ -n "$name" ] && [ -d "$(repo_dir "$name")" ] && [ "$(visibility "$name")" = public ]; then
180 with_lock "$name" publish_one "$name" >/dev/null
181 "${GEN[@]}" index >/dev/null
182 fi
183}
184
185cmd_ci_queue() {
186 local name=${1:-} ref=${2:-} sha=${3:-}
187 [ -n "$name" ] && [ -n "$ref" ] && [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || die "usage: fw ci-queue <name> <ref> <sha>"
188 need_repo "$name"
189 mkdir -p "$QUEUE/pending" "$QUEUE/running"
190 local id; id="$(date +%s%N | cut -c1-16)-$name-${sha:0:12}"
191 local default; default=$(git -C "$(repo_dir "$name")" symbolic-ref --short HEAD 2>/dev/null || echo main)
192 # Written beside the queue, then renamed in: a runner never sees a half-written job.
193 python3 -c 'import json, sys, time
194id, name, ref, sha, default, out = sys.argv[1:]
195json.dump({"id": id, "repo": name, "ref": ref, "sha": sha, "default_branch": default,
196 "queued": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, open(out, "w"))' \
197 "$id" "$name" "$ref" "$sha" "$default" "$QUEUE/$id.json.tmp"
198 mv "$QUEUE/$id.json.tmp" "$QUEUE/pending/$id.json"
199 echo "fw: CI queued for $name ${ref#refs/*/} ${sha:0:12} ($id)"
200 "${GEN[@]}" ci-dashboard || true
201}
202
203cmd_ci() {
204 local name=${1:-} branch=${2:-}
205 [ -n "$name" ] || die "usage: fw ci <name> [branch]"
206 need_repo "$name"
207 local r; r=$(repo_dir "$name")
208 [ -n "$branch" ] || branch=$(git -C "$r" symbolic-ref --short HEAD)
209 local sha; sha=$(git -C "$r" rev-parse -q --verify "refs/heads/$branch^{commit}") || die "no branch $branch in $name"
210 cmd_ci_queue "$name" "refs/heads/$branch" "$sha"
211}
212
213cmd_ci_list() {
214 local d f
215 for d in pending running; do
216 for f in "$QUEUE/$d"/*.json; do
217 [ -e "$f" ] || continue
218 printf '%-8s %s %s\n' "$d" "$(date -r "$f" '+%F %T')" "$(basename "$f" .json)"
219 done
220 done
221}
222
223cmd_issues() {
224 case "${1:-}" in
225 ingest|imap-check|list|show|record|approve|hide|close|reopen|publish|publish-all|help|-h|--help|"")
226 python3 "$TOOLS/server/issues.py" "$@" ;;
227 *)
228 # fw issues <repo> <mode>
229 [ $# -eq 2 ] && [[ "$2" =~ ^(public|reviewed|private)$ ]] || die "usage: fw issues <repo> public|reviewed|private (or fw issues help)"
230 need_repo "$1"
231 git -C "$(repo_dir "$1")" config fuzzyworld.issues "$2"
232 python3 "$TOOLS/server/issues.py" publish "$1"
233 echo "$1: issues $2" ;;
234 esac
235}
236
237cmd_ci_watch() {
238 local last="" now
239 while true; do
240 now=$("${GEN[@]}" ci-status)
241 if [ "$now" != "$last" ]; then
242 printf '\n%s\n%s\n' "$(date '+%T')" "$now"
243 last=$now
244 fi
245 grep -qE '^(running|queued) ' <<< "$now" || { echo "(nothing queued or running)"; return; }
246 sleep 10
247 done
248}
249
250mkdir -p "$STATE" "$LOGS"
251case "${1:-}" in
252 apps-index) "${GEN[@]}" apps ;;
253 content) "${GEN[@]}" content ;;
254 app-deployed) cmd_app_deployed "${2:-}" "${3:-}" ;;
255 list) cmd_list ;;
256 new) shift; cmd_new "$@" ;;
257 public|private) cmd_set_visibility "$1" "${2:-}" ;;
258 describe) shift; cmd_describe "$@" ;;
259 publish) cmd_publish "${2:-}" ;;
260 publish-all) cmd_publish_all ;;
261 install-hooks) cmd_install_hooks ;;
262 ci) cmd_ci "${2:-}" "${3:-}" ;;
263 ci-queue) cmd_ci_queue "${2:-}" "${3:-}" "${4:-}" ;;
264 ci-list) cmd_ci_list ;;
265 ci-watch) cmd_ci_watch ;;
266 ci-pages) need_repo "${2:-}"; with_lock "$2" "${GEN[@]}" ci "$2" ;;
267 ci-dashboard) "${GEN[@]}" ci-dashboard ;;
268 comments) shift; python3 "$TOOLS/server/comments.py" "$@" ;;
269 issues) shift; cmd_issues "$@" ;;
270 deploys) shift; python3 "$TOOLS/server/deploys.py" "$@" ;;
271 ""|-h|--help|help) sed -n '2,28p' "$0" | sed 's/^# \{0,1\}//' ;;
272 *) die "unknown command: $1 (try: fw help)" ;;
273esac