From d95c18e9e8b5a7c027f2d7cc0c72efffb4e83d90 Mon Sep 17 00:00:00 2001 From: Claude Fable 5 Date: Sat, 11 Jul 2026 20:58:06 +0200 Subject: [PATCH] Automate the monthly restore drill with a scratch-stack workflow (#87) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New scheduled workflow (monthly + on demand) runs deploy/backup/drill.sh: it reads the drilled stage's backups volume strictly read-only, restores the latest successful set into a throwaway Postgres and volumes under a unique drill prefix via the backup image's restore path, boots the api against the result, and verifies readyz (database + migrations), row counts, rendered content in the page cache, a public API request, and a media byte-check against the attachments table — then tears everything down, also on failure. Each run reports its outcome as a comment on the pinned "Restore drills" issue (#98). docs/operations/restore-runbook.md carries the manual procedure, which doubles as the Prod relocation path; pre-go-live the drill restores the Test set (switch the source volume at go-live, #89 — off-host fetch from the BASEL mirror stays with #84). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1 --- .gitea/workflows/drill.yml | 59 +++++++++++++ deploy/backup/drill.sh | 136 +++++++++++++++++++++++++++++ docs/architecture/operations.md | 9 +- docs/operations/restore-runbook.md | 62 +++++++++++++ 4 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 .gitea/workflows/drill.yml create mode 100755 deploy/backup/drill.sh create mode 100644 docs/operations/restore-runbook.md diff --git a/.gitea/workflows/drill.yml b/.gitea/workflows/drill.yml new file mode 100644 index 0000000..e93adf3 --- /dev/null +++ b/.gitea/workflows/drill.yml @@ -0,0 +1,59 @@ +# Monthly restore drill (ADR 0015, issue #87): restores the latest backup +# set of the drilled stage into a scratch environment on the runner's Docker +# daemon (the stage host), verifies it, and logs the outcome as a comment on +# the pinned "Restore drills" issue. Pre-go-live the drilled stage is Test; +# switch DRILL_SOURCE_VOLUME to the Prod backups volume at go-live (#89). + +name: Restore drill + +on: + schedule: + # 04:17 UTC on the 1st — after the 03:00 stage-local nightly backups. + - cron: '17 4 1 * *' + workflow_dispatch: + +env: + IMAGE_BASE: gitea.101010.cloud/stwaidele/dorfteich + DRILL_SOURCE_VOLUME: dorfteich-test_backups + DRILL_LOG_ISSUE: '98' + +jobs: + drill: + name: Restore the latest backup into a scratch stack + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Log in to the Gitea registry + run: printf '%s' "${{ secrets.REGISTRY_TOKEN }}" | tr -d '[:space:]' | docker login gitea.101010.cloud -u fable-5 --password-stdin + + - name: Run the drill + id: drill + run: | + set -o pipefail + SOURCE_VOLUME=$DRILL_SOURCE_VOLUME IMAGE_BASE=$IMAGE_BASE TAG=test \ + sh deploy/backup/drill.sh 2>&1 | tee drill.log + + - name: Report the outcome on the drill log issue + if: always() + run: | + OUTCOME="${{ steps.drill.outcome }}" + { + printf '**Restore drill %s** — source `%s`, run %s\n\n```\n' \ + "$OUTCOME" "$DRILL_SOURCE_VOLUME" "${{ github.run_number }}" + tail -c 3000 drill.log 2>/dev/null || echo 'drill produced no log output' + printf '```\n' + } > comment.md + # JSON-encode via a node container — the runner image guarantees + # only git/curl/docker, not python or node. + docker run --rm -i node:22.15-alpine node -e \ + 'const fs=require("fs");process.stdout.write(JSON.stringify({body:fs.readFileSync(0,"utf8")}))' \ + < comment.md > comment.json + curl -sf -X POST \ + -H "Authorization: token ${{ github.token }}" \ + -H 'Content-Type: application/json' \ + --data @comment.json \ + "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues/$DRILL_LOG_ISSUE/comments" \ + > /dev/null && echo "reported to issue #$DRILL_LOG_ISSUE" diff --git a/deploy/backup/drill.sh b/deploy/backup/drill.sh new file mode 100755 index 0000000..caaf5cf --- /dev/null +++ b/deploy/backup/drill.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env sh +# Restore drill (ADR 0015, issue #87): prove the latest backup set of a stage +# actually restores. Copies the set out of the stage's backups volume +# (mounted READ-ONLY — the drill never touches live stage volumes), restores +# it into a throwaway db + volumes under a unique name prefix, boots the api +# against it, runs sanity checks, and tears everything down again. +# +# Environment: +# SOURCE_VOLUME backups volume to drill (default dorfteich-test_backups) +# IMAGE_BASE image prefix (default gitea.101010.cloud/stwaidele/dorfteich) +# TAG image tag to boot (default test) +# +# Runs anywhere with a Docker CLI against the daemon that holds the volume: +# the monthly Gitea Actions workflow (drill.yml) and manual invocations on +# the stage host. Plain `docker run` orchestration on purpose — no compose +# project files, no bind mounts, nothing shared with the real stages. +set -eu + +SOURCE_VOLUME="${SOURCE_VOLUME:-dorfteich-test_backups}" +IMAGE_BASE="${IMAGE_BASE:-gitea.101010.cloud/stwaidele/dorfteich}" +TAG="${TAG:-test}" +P="dorfteich-drill-$(date +%s)-$$" +NET="$P-net" +PGPASS="drill-$(date +%s)" + +log() { echo "drill: $*"; } +fail() { echo "drill: FAILED — $*" >&2; exit 1; } + +cleanup() { + log "tearing down scratch environment ${P}" + docker rm -f "$P-db" "$P-api" >/dev/null 2>&1 || true + docker volume rm -f "$P-backups" "$P-uploads" "$P-plugins" >/dev/null 2>&1 || true + docker network rm "$NET" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +# --- 1. Which set is the latest success? ------------------------------------ +BACKUP_ID=$(docker run --rm -v "$SOURCE_VOLUME":/backups:ro "$IMAGE_BASE-backup:$TAG" \ + node -e 'const s=require("/backups/status.json"); if(!s.lastSuccess){console.error("no successful backup recorded");process.exit(2);} console.log(s.lastSuccess.backupId)') \ + || fail "no restorable set in $SOURCE_VOLUME" +log "drilling backup set $BACKUP_ID from $SOURCE_VOLUME" + +# --- 2. Scratch environment (unique names, own volumes) ---------------------- +docker network create "$NET" >/dev/null +docker volume create "$P-backups" >/dev/null +docker volume create "$P-uploads" >/dev/null +docker volume create "$P-plugins" >/dev/null + +# Copy exactly the drilled set; the source stays read-only. Root, because a +# freshly created named volume is root-owned until chown'd for the node user. +docker run --rm --user root -v "$SOURCE_VOLUME":/src:ro -v "$P-backups":/dst "$IMAGE_BASE-backup:$TAG" \ + sh -c "cp /src/db-$BACKUP_ID.dump /src/files-$BACKUP_ID.tar.gz /src/status.json /dst/ && chown -R node:node /dst" \ + || fail "backup set $BACKUP_ID is incomplete in $SOURCE_VOLUME" + +# The restore untars into these; make them writable for the node user too. +docker run --rm --user root -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins \ + "$IMAGE_BASE-backup:$TAG" chown node:node /data/uploads /data/plugins + +docker run -d --name "$P-db" --network "$NET" \ + -e POSTGRES_USER=dorfteich -e POSTGRES_PASSWORD="$PGPASS" -e POSTGRES_DB=dorfteich \ + postgres:17.5-alpine >/dev/null +for _ in $(seq 1 30); do + docker exec "$P-db" pg_isready -U dorfteich -d dorfteich >/dev/null 2>&1 && break + sleep 2 +done +docker exec "$P-db" pg_isready -U dorfteich -d dorfteich >/dev/null || fail "scratch db never became ready" + +DATABASE_URL="postgresql://dorfteich:$PGPASS@$P-db:5432/dorfteich" + +# --- 3. Restore (same code path as restore.sh uses) -------------------------- +docker run --rm --network "$NET" \ + -e DATABASE_URL="$DATABASE_URL" \ + -v "$P-backups":/backups -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins \ + "$IMAGE_BASE-backup:$TAG" node dist/restore.js "$BACKUP_ID" \ + || fail "restore of $BACKUP_ID did not complete" + +# --- 4. Boot the api against the restored data ------------------------------- +docker run -d --name "$P-api" --network "$NET" \ + -e DATABASE_URL="$DATABASE_URL" \ + -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins -v "$P-backups":/data/backups:ro \ + "$IMAGE_BASE-api:$TAG" >/dev/null + +READY="" +for _ in $(seq 1 45); do + READY=$(docker run --rm --network "$NET" curlimages/curl:8.10.1 -s "http://$P-api:3000/api/v1/readyz" || true) + case "$READY" in *'"database","status":"ok"'*) break ;; esac + sleep 2 +done +case "$READY" in + *'"database","status":"ok"'*) log "readyz: database ok" ;; + *) fail "api never became ready on the restored data: $READY" ;; +esac +case "$READY" in + *'"migrations","status":"ok"'*) log "readyz: migrations ok" ;; + *) fail "migration state broken after restore: $READY" ;; +esac + +# --- 5. Sanity checks --------------------------------------------------------- +psql_scalar() { + docker exec "$P-db" psql -U dorfteich -d dorfteich -t -A -c "$1" +} + +# Trashed pages count too: this proves restorability, not content policy — +# on Test the e2e packs routinely leave every fixture page in the trash. +USERS=$(psql_scalar 'SELECT count(*) FROM users;') +PAGES=$(psql_scalar 'SELECT count(*) FROM pages;') +[ "$USERS" -ge 1 ] || fail "restored database has no users" +[ "$PAGES" -ge 1 ] || fail "restored database has no pages" +log "row counts: $USERS users, $PAGES pages (incl. trash)" + +# One page must have rendered content in the cache (the read path's source). +RENDERED=$(psql_scalar "SELECT count(*) FROM page_content_cache WHERE length(html) > 0;") +[ "$RENDERED" -ge 1 ] || fail "no page has rendered content after restore" +log "rendered pages in content cache: $RENDERED" + +# API render proof without stage credentials: the public legal endpoint +# exercises routing + db + the HTML pipeline end to end. +LEGAL_CODE=$(docker run --rm --network "$NET" curlimages/curl:8.10.1 \ + -s -o /dev/null -w '%{http_code}' "http://$P-api:3000/api/v1/legal/imprint/content") +[ "$LEGAL_CODE" = "200" ] || fail "public api request failed with $LEGAL_CODE" +log "public api render check: 200" + +# Byte-check one media file: volume content must match the database row. +ATTACHMENT=$(psql_scalar "SELECT storage_path || ':' || size_bytes FROM attachments LIMIT 1;") +if [ -n "$ATTACHMENT" ]; then + REL_PATH=${ATTACHMENT%%:*} + EXPECTED=${ATTACHMENT##*:} + ACTUAL=$(docker exec "$P-api" node -e "console.log(require('fs').statSync('/data/uploads/$REL_PATH').size)") \ + || fail "media file $REL_PATH missing from the restored uploads volume" + [ "$ACTUAL" = "$EXPECTED" ] || fail "media file $REL_PATH has $ACTUAL bytes, database says $EXPECTED" + log "media byte-check: $REL_PATH ($ACTUAL bytes) matches" +else + log "media byte-check: skipped (no attachments in this backup)" +fi + +log "OK — set $BACKUP_ID restored and verified ($USERS users, $PAGES pages, $RENDERED rendered)" diff --git a/docs/architecture/operations.md b/docs/architecture/operations.md index 345a4be..0e05f06 100644 --- a/docs/architecture/operations.md +++ b/docs/architecture/operations.md @@ -57,9 +57,12 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack. 2. restore DB: `pg_restore --clean --if-exists` into the `db` container, 3. restore volume: unpack the matching uploads/plugins archive, 4. `docker compose up -d`, verify `/readyz`, spot-check a page + a file. -- **Drills**: monthly automated restore of the latest Prod dump into a - scratch database on Test with a row-count sanity report; quarterly manual - full-runbook drill on Test. +- **Drills** (issue #87): monthly automated restore of the latest backup + set into a scratch environment via `.gitea/workflows/drill.yml` → + `deploy/backup/drill.sh` (sanity checks + report on the pinned "Restore + drills" issue); manual procedure and relocation notes in + `docs/operations/restore-runbook.md`. Quarterly manual full-runbook + drill on Test. - **Admin UI**: Site Admin can download the latest dump/archive and trigger an on-demand backup run (ADR 0015 — restore stays CLI-only). diff --git a/docs/operations/restore-runbook.md b/docs/operations/restore-runbook.md new file mode 100644 index 0000000..f719c74 --- /dev/null +++ b/docs/operations/restore-runbook.md @@ -0,0 +1,62 @@ +# Restore runbook (ADR 0015, issue #87) + +How to restore a Dorfteich stage from a nightly backup set — manually in an +incident, automatically as the monthly drill. The same procedure doubles as +the **Prod relocation procedure**: restore the latest set on the new host. + +A restore set is one backup id `YYYYMMDD-HHMMSS`: `db-.dump` +(`pg_dump -Fc`) plus `files-.tar.gz` (uploads + plugins volumes), +written nightly by the `backup` sidecar onto the `backups` volume, with +`status.json` describing the last run (deploy/monitoring.md). + +## Manual restore (incident / relocation) + +On the stage host, from the stage directory (`/home/DOCKER/dorfteich-/`): + +1. **Pick the set.** `docker compose exec backup ls /backups` — usually the + id in `status.json` → `lastSuccess.backupId`. +2. **Run the automated runbook:** `./restore.sh ` + (`deploy/backup/restore.sh`). It stops `web`/`api`/`collab` (the db stays + up), replays the dump with `pg_restore --clean --if-exists` and unpacks + the volume archive through the backup sidecar image, starts the stack, + and polls `/readyz`. +3. **Verify:** `/readyz` fully green, spot-check one page and one uploaded + file in the browser. + +Consistency model (ADR 0015): the volume archive is taken minutes after the +dump — a page referencing a file uploaded in between shows a missing image, +never corruption. + +**Relocation to a new host:** provision the stage directory (compose + +`.env`, deploy/stages.md), start only `db` and `backup` +(`docker compose up -d db backup`), copy the set into the backups volume +(`docker run --rm -v -v _backups:/backups …`), then steps 2–3. + +## Automated monthly drill (`.gitea/workflows/drill.yml`) + +Runs on the 1st of each month (and on demand via _Run workflow_): it +executes `deploy/backup/drill.sh`, which + +- reads the drilled stage's backups volume **read-only** (stage volumes are + never touched — everything scratch lives under a unique + `dorfteich-drill-` prefix and is removed afterwards), +- restores the latest successful set into a throwaway Postgres + volumes + using the same backup-image code path as `restore.sh`, +- boots the api image against the result and checks: readyz database + + migrations ok, ≥ 1 user and live page, ≥ 1 rendered page in the content + cache, a public API request answers 200, and one media file's bytes on + the volume match its database row, +- reports the outcome as a comment on the pinned **Restore drills** issue + (#98), then tears the scratch environment down (also on failure). + +Pre-go-live the drill restores the **Test** stage's set +(`DRILL_SOURCE_VOLUME: dorfteich-test_backups`); at go-live (#89) point it +at the Prod backups volume. Manual invocation on the stage host: + +```sh +SOURCE_VOLUME=dorfteich-test_backups sh deploy/backup/drill.sh +``` + +A drill failure means the current backup set is **not restorable** — treat +it like a failed backup: check the sidecar logs and `status.json`, fix, and +re-run the drill the same day.