fuzzyworld-tools / commit
Add contribute page, read-only backup access, TrueNAS mirror script
server/gen.py | 7 ++- server/install.sh | 1 + server/mirror-only | 30 ++++++++++++ server/site/contribute/index.html | 52 +++++++++++++++++++++ server/site/how/index.html | 97 +++++++++++++++++++++++++++++++++++++++ server/site/why/index.html | 7 ++- server/truenas/mirror-all.sh | 46 +++++++++++++++++++ 7 files changed, 234 insertions(+), 6 deletions(-) diff --git a/server/gen.py b/server/gen.py index dd6fa3f..fd0aa52 100755 --- a/server/gen.py +++ b/server/gen.py @@ -33,7 +33,7 @@ BASE_URL = "https://code.fuzzyworld.net" APPS_URL = "https://apps.fuzzyworld.net" # Bump when page templates change, so every repo gets a full rebuild once. -GEN_VERSION = 5 +GEN_VERSION = 6 LOG_LIMIT = 100 MAX_TEXT_BYTES = 1024 * 1024 @@ -114,7 +114,7 @@ def page(title, body, crumbs=None, repo=None, site="code"): f'<a href="{BASE_URL}/why/">Why this site exists</a>') else: host, footer = "code.fuzzyworld.net", (f'<a href="/">All repositories</a> · <a href="{APPS_URL}/">Apps</a> · ' - f'<a href="/why/">Why this site exists</a>') + f'<a href="/contribute/">Contribute</a> · <a href="/why/">Why this site exists</a>') nav = "" if repo: nav = (f'<nav class="repo-nav"><a href="{url(repo)}/">Summary</a>' @@ -404,6 +404,9 @@ def build_summary(repo, out, name, desc, files, ctx, commits, empty): clone = f"git clone {BASE_URL}/{name}.git" app = app_url(name) links = f'<p class="meta">App: <a href="{app}">{esc(app)}</a></p>' if app else "" + contact = f"bugs-k7q4+{name}@fuzzyworld.net" + links += (f'<p class="meta">Contribute or report a bug: <a href="mailto:{esc(contact)}">{esc(contact)}</a>' + f' · <a href="/contribute/">how</a></p>') readme_html = "" if not empty: lower = {p.lower(): p for p in files if "/" not in p} diff --git a/server/install.sh b/server/install.sh index abcffaa..4c73798 100755 --- a/server/install.sh +++ b/server/install.sh @@ -24,6 +24,7 @@ site=~/code.fuzzyworld.net cp ~/fw/tools/server/site/htaccess "$site/.htaccess" cp ~/fw/tools/server/site/style.css ~/fw/tools/server/site/robots.txt ~/fw/tools/server/site/404.html "$site/" mkdir -p "$site/why" && cp ~/fw/tools/server/site/why/index.html "$site/why/" +mkdir -p "$site/contribute" && cp ~/fw/tools/server/site/contribute/index.html "$site/contribute/" apps=~/apps.fuzzyworld.net cp ~/fw/tools/server/site-apps/htaccess "$apps/.htaccess" cp ~/fw/tools/server/site/style.css "$apps/" diff --git a/server/mirror-only b/server/mirror-only new file mode 100755 index 0000000..1644bc8 --- /dev/null +++ b/server/mirror-only @@ -0,0 +1,30 @@ +#!/bin/bash +# Forced command for backup keys in ~/.ssh/authorized_keys on DreamHost: +# +# restrict,command="/home/fuzzygamer/fw/tools/server/mirror-only" ssh-ed25519 AAAA... truenas-mirror +# +# The key can then only read repos: `git fetch`/`git clone` of ~/git/<name>.git, plus +# `list` (repo names, one per line) and `visibility <name>`. Anything else is refused. +set -u +cmd=${SSH_ORIGINAL_COMMAND:-} +log="$HOME/fw/logs/mirror-only.log" +mkdir -p "$(dirname "$log")" +echo "$(date -u '+%F %T') ${SSH_CLIENT%% *} $cmd" >> "$log" + +valid_repo() { [[ "$1" =~ ^git/[A-Za-z0-9][A-Za-z0-9._-]*\.git$ ]] && [ -d "$HOME/$1" ]; } + +case "$cmd" in + "git-upload-pack '"*"'") + repo=${cmd#git-upload-pack \'}; repo=${repo%\'}; repo=${repo#/home/fuzzygamer/}; repo=${repo#\~/} + valid_repo "$repo" || { echo "mirror-only: no such repo" >&2; exit 1; } + exec git-upload-pack "$HOME/$repo" ;; + list) + for r in "$HOME"/git/*.git; do basename "$r" .git; done ;; + "visibility "*) + name=${cmd#visibility } + valid_repo "git/$name.git" || { echo "mirror-only: no such repo" >&2; exit 1; } + v=$(git -C "$HOME/git/$name.git" config --get fuzzyworld.visibility || true) + [ "$v" = public ] && echo public || echo private ;; + *) + echo "mirror-only: this key may only fetch repos (refused: $cmd)" >&2; exit 1 ;; +esac diff --git a/server/site/contribute/index.html b/server/site/contribute/index.html new file mode 100644 index 0000000..2cec09d --- /dev/null +++ b/server/site/contribute/index.html @@ -0,0 +1,52 @@ +<!doctype html> +<html lang="en"> + +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>How to contribute · code.fuzzyworld.net</title> + <link rel="stylesheet" href="/style.css"> +</head> + +<body> + <header class="site"><a class="home" href="/">code.fuzzyworld.net</a></header> + <main> + <h1>How to contribute</h1> + <p>Thanks for wanting to help! This site has no accounts, issue tracker or merge requests, so contributions + arrive by email. Two ways work, both with plain git.</p> + + <h2>1. Share a branch</h2> + <p>Best for anything bigger than a small fix.</p> + <ol> + <li>Clone the repository: <code>git clone https://code.fuzzyworld.net/<repo>.git</code></li> + <li>Make your change on a branch, and push that branch to <strong>any</strong> public git host you like + (GitHub, Codeberg, GitLab, your own server).</li> + <li>Email the repository URL, the branch name and a short description to + <code>bugs-k7q4+<repo>@fuzzyworld.net</code>, using the repository's name in place of + <code><repo></code>.</li> + </ol> + <p>I fetch the branch, review it locally, and reply by email.</p> + + <h2>2. Email a patch</h2> + <p>Good for small changes. Commit your change, then create a patch file:</p> + <pre>git format-patch origin/main</pre> + <p>and attach the <code>.patch</code> file(s) to an email to <code>bugs-k7q4+<repo>@fuzzyworld.net</code>. + If you use <code>git send-email</code>, that address works there too.</p> + + <h2>Bug reports and ideas</h2> + <p>Same address, no git needed. Many of the apps have a <strong>report</strong> link that fills in the + details for you.</p> + + <h2>Before you send</h2> + <ul> + <li>Check the repository's README (and CONTRIBUTING, if it has one) for how to build and test it.</li> + <li>One change per branch or patch, please; it makes review much quicker.</li> + <li>By contributing you agree that your change is shared under the repository's license.</li> + </ul> + + <p><a href="/why/">Why there are no merge requests here</a></p> + </main> + <footer class="site"><a href="/">All repositories</a> · <a href="https://apps.fuzzyworld.net/">Apps</a></footer> +</body> + +</html> diff --git a/server/site/how/index.html b/server/site/how/index.html new file mode 100644 index 0000000..b9f9c83 --- /dev/null +++ b/server/site/how/index.html @@ -0,0 +1,97 @@ +<!doctype html> +<html lang="en"> + +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>How I moved off GitLab · code.fuzzyworld.net</title> + <link rel="stylesheet" href="/style.css"> +</head> + +<body> + <header class="site"><a class="home" href="/">code.fuzzyworld.net</a></header> + <main> + <!-- DRAFT 2026-10-27 for code.fuzzyworld.net/how/ (plan Backlog item 3). Pavel edits before it goes live; on hold until the item 2 features. Not installed or linked. --> + <h1>How I moved off GitLab</h1> + <p><a href="/why/">Why</a> I left is short. <em>How</em> took about a day, 89 repositories and 35 web + apps, and turned up enough surprises that it seemed worth writing down for anyone thinking about the same + move.</p> + + <h2>What I ended up with</h2> + <ul> + <li><strong>Bare git repos on cheap shared hosting</strong> (DreamHost). I push over SSH; nothing else + runs there permanently.</li> + <li><strong>code.fuzzyworld.net</strong>: static browse pages, generated by a git hook when I push, plus + read-only <code>git clone https://…</code>. No app server, no accounts, no rate limits beyond what + Apache can serve.</li> + <li><strong>apps.fuzzyworld.net</strong>: every former GitLab Pages site, at the same relative path, built + and uploaded from my laptop with one command.</li> + <li><strong>A backup mirror at home</strong> that can read, but not write.</li> + </ul> + <p>What I gave up: issues, merge requests and search. Only the default branch can be browsed. Cloning over + plain HTTP is slower for big repos (a 244 MB repo takes about 50 seconds).</p> + + <h2>Getting everything out of GitLab</h2> + <p>The code is the easy part: <code>git clone --mirror</code> gets every branch and tag. The rest lives in + GitLab's settings and is easy to forget:</p> + <ul> + <li><strong>CI/CD variables.</strong> Signing keys, API tokens, deploy keys. Export them before anything + else; losing an Android signing key means you can't update that app in the Play Store again.</li> + <li><strong>Release files.</strong> APKs and tarballs attached to releases are often CI <em>job + artifacts</em>, and those expire. Five of mine already had.</li> + <li><strong>Pipeline schedules</strong>, issues, merge requests and wiki pages. Confidential issues in a + public project need separate handling, or the export makes them public.</li> + <li><strong>Merge-request commits</strong> live under <code>refs/merge-requests/*</code>, which a normal + clone doesn't fetch. I kept them as <code>refs/gitlab/merge-requests/*</code>.</li> + </ul> + + <h2>Traps I fell into</h2> + <ul> + <li><strong>The rate limit bit me during the export.</strong> The <code>glab</code> CLI's login is an + OAuth token, which has to be sent as <code>Authorization: Bearer</code>. Sent as + <code>PRIVATE-TOKEN</code>, every call counted as anonymous, and my home IP was blocked within + minutes. Web URLs for job artifacts and uploads count as anonymous even with a token; use the API + endpoints instead.</li> + <li><strong>My own git hook refused my own history.</strong> A global pre-push hook that rejects + lightweight tags (good for new releases) blocked mirroring old ones. Historical tags have to move + unchanged, so the migration pushes skip the hook.</li> + <li><strong>Shared hosting kills big git operations.</strong> Cloning a 284 MB repo over SSH failed with + "git-pack-objects died": the host's watchdog killed git's packing step. Limiting git's threads and + memory (<code>pack.threads 1</code>, <code>pack.windowMemory 64m</code>) fixed it for good.</li> + <li><strong>The host executes things you might not expect.</strong> DreamHost runs <code>.php</code> files + and server-side includes by default. A code browser that wrote repository files into the web root + with their real names would have run them. Browse pages are pre-generated HTML, raw copies are + images only, and every response says <code>script-src 'none'</code>.</li> + <li><strong>Node on a hardened kernel.</strong> The host's kernel refuses executable memory, so modern + Node crashes as soon as its JIT starts. <code>--jitless</code> works; native npm modules don't build, + so installs use <code>--ignore-scripts</code>.</li> + </ul> + + <h2>Moving the apps</h2> + <p>Every app moved to the same path under a new domain, and they all share one origin, exactly as they did on + <code>*.gitlab.io</code>. That means two apps can still clash over <code>localStorage</code> keys, a + service worker can claim more than its own folder, and two installed web apps can merge on Android if + their manifests look alike. So the deploy script refuses to upload anything whose URLs, manifest or + service worker reach outside the app's own path.</p> + <p>Small things broke in surprising places: GitLab-style links (<code>/-/blob/main/…</code>, + <code>/-/commit/…</code>) baked into apps, short commit IDs where the new site expected long ones, a + vite config whose root was a subfolder, Google sign-in that only works on domains Firebase knows about, + and OAuth redirect URIs registered with the old address.</p> + + <h2>Saying goodbye without breaking links</h2> + <p>Each GitLab project got one last commit: a README pointing to the new home, and, for sites, a redirect + page in place of the app. The redirect keeps the path, query and <code>#hash</code>, so deep links land on + the matching new page. It also retires any old service worker, which would otherwise keep serving the + cached old app to returning visitors forever.</p> + + <h2>Would I do it again?</h2> + <p>Yes. Everything is static, it costs nothing extra on hosting I already paid for, and nobody needs an + account to read my code. The one real loss is easy contributions: there are no merge requests here + yet.</p> + + <p><a href="/why/">← Why I left GitLab</a></p> + </main> + <footer class="site"><a href="/">All repositories</a> · <a href="https://apps.fuzzyworld.net/">Apps</a></footer> +</body> + +</html> diff --git a/server/site/why/index.html b/server/site/why/index.html index cf0190d..74b60d0 100644 --- a/server/site/why/index.html +++ b/server/site/why/index.html @@ -14,12 +14,11 @@ <h1>Why I left GitLab</h1> <p>On Monday, October 26, 2026, I came back to work after a week's vacation to Friday Harbor. I had some spare time, so I made a turn-based rogue-like Bomberman clone. I was thinking of sharing it with some coworkers, - but very quickly GitLab showed me a "Rate limit exceeded" error. And rapidly learned that GitLab enacted - some rate-limiting policies recently that effectively blocked repo access at work, and, very quickly, even + but very quickly GitLab showed me a "Rate limit exceeded" error. And I rapidly learned that GitLab enacted + some rate-limiting policies recently that blocked repo access at work, and, very quickly, even access to GitLab itself!</p> <p>So yeah, I couldn't look at my personal code at work, and that pissed me off, and now all of my code and projects are hosted on Dreamhost, where I already keep a blog and a lightweight webpage.</p> - <ul> <li>code: <a href="https://code.fuzzyworld.net/">https://code.fuzzyworld.net/</a> </li> @@ -28,7 +27,7 @@ <p>No rate limiting, no sign-in, no "modern" limits. This is a static website! What more do I really need?</p> <p></p> <p>I'm currently trying to formalize the process of the move from GitLab to Dreamhost, and eventually my - learning are going to be at /how/ on this website.</p> + learnings are going to be at /how/ on this website.</p> </main> <footer class="site"><a href="/">All repositories</a> · <a href="https://apps.fuzzyworld.net/">Apps</a></footer> </body> diff --git a/server/truenas/mirror-all.sh b/server/truenas/mirror-all.sh new file mode 100755 index 0000000..3a52509 --- /dev/null +++ b/server/truenas/mirror-all.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# Backup mirror of every DreamHost repo, run on TrueNAS from a cron job (System → Advanced → Cron Jobs). +# +# mirror-all.sh /mnt/<pool>/git-mirrors +# +# DEST is a ZFS dataset (snapshotted by a periodic snapshot task). DEST/.ssh/id_ed25519 is a key that +# DreamHost only lets run server/mirror-only (fetch + list), so this box can read repos but never write. +# New repos are cloned with --mirror; existing ones get `remote update --prune`. Repos that disappear +# from DreamHost are kept here (this is a backup) and listed in the summary. Each mirror's config +# carries fuzzyworld.visibility, so a restore keeps the private/public marking. +set -u +DEST=${1:?usage: mirror-all.sh /mnt/<pool>/git-mirrors} +HOST=${FW_MIRROR_HOST:-fuzzygamer@iad1-shared-b7-30.dreamhost.com} +SSH="ssh -i $DEST/.ssh/id_ed25519 -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$DEST/.ssh/known_hosts" +export GIT_SSH_COMMAND="$SSH" +LOG=$DEST/.logs/mirror-$(date -u +%Y%m%d).log +mkdir -p "$DEST/.logs" +exec 3>&2 # keep the cron job's stderr: failures are reported there too +exec >> "$LOG" 2>&1 +echo "== $(date -u '+%F %T UTC') mirror from $HOST" + +names=$($SSH "$HOST" list) || { echo "FAIL: cannot list repos on $HOST"; echo "git mirror: cannot list repos on $HOST (log: $LOG)" >&3; exit 1; } +ok=0; failed=0 +for name in $names; do + m="$DEST/$name.git" + if [ -d "$m" ]; then + git -C "$m" remote update --prune >/dev/null 2>&1 && r=0 || r=1 + else + git clone --quiet --mirror "$HOST:git/$name.git" "$m" && r=0 || r=1 + fi + if [ $r = 0 ]; then + v=$($SSH "$HOST" "visibility $name") && git -C "$m" config fuzzyworld.visibility "$v" + ok=$((ok + 1)) + else + echo "FAIL: $name"; failed=$((failed + 1)) + fi +done +gone="" +for m in "$DEST"/*.git; do + [ -d "$m" ] || continue + n=$(basename "$m" .git) + echo "$names" | tr ' ' '\n' | grep -qxF "$n" || gone="$gone $n" +done +echo "mirrored $ok, failed $failed${gone:+, kept but no longer on DreamHost:$gone}" +echo "== done $(date -u '+%F %T UTC')" +[ $failed = 0 ] || { echo "git mirror: $failed repo(s) failed (log: $LOG)" >&3; exit 1; }