Claude Fable 5
d95c18e9e8
All checks were successful
CD / Build and push images (push) Successful in 1m5s
CD / Deploy to Test (push) Successful in 10s
CD / Smoke tests against Test (push) Successful in 1m7s
CD / Promote to Int (push) Successful in 11s
CI / Lint, typecheck, test (push) Successful in 3m18s
CI / Build container images (push) Has been skipped
CI / Auth e2e pack (push) Successful in 5m14s
CI / Import/export fidelity gate (push) Successful in 45s
New scheduled workflow (monthly + on demand) runs deploy/backup/drill.sh: it reads the drilled stage's backups volume strictly read-only, restores the latest successful set into a throwaway Postgres and volumes under a unique drill prefix via the backup image's restore path, boots the api against the result, and verifies readyz (database + migrations), row counts, rendered content in the page cache, a public API request, and a media byte-check against the attachments table — then tears everything down, also on failure. Each run reports its outcome as a comment on the pinned "Restore drills" issue (#98). docs/operations/restore-runbook.md carries the manual procedure, which doubles as the Prod relocation path; pre-go-live the drill restores the Test set (switch the source volume at go-live, #89 — off-host fetch from the BASEL mirror stays with #84). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1
137 lines
6.5 KiB
Bash
Executable File
137 lines
6.5 KiB
Bash
Executable File
#!/usr/bin/env sh
|
|
# Restore drill (ADR 0015, issue #87): prove the latest backup set of a stage
|
|
# actually restores. Copies the set out of the stage's backups volume
|
|
# (mounted READ-ONLY — the drill never touches live stage volumes), restores
|
|
# it into a throwaway db + volumes under a unique name prefix, boots the api
|
|
# against it, runs sanity checks, and tears everything down again.
|
|
#
|
|
# Environment:
|
|
# SOURCE_VOLUME backups volume to drill (default dorfteich-test_backups)
|
|
# IMAGE_BASE image prefix (default gitea.101010.cloud/stwaidele/dorfteich)
|
|
# TAG image tag to boot (default test)
|
|
#
|
|
# Runs anywhere with a Docker CLI against the daemon that holds the volume:
|
|
# the monthly Gitea Actions workflow (drill.yml) and manual invocations on
|
|
# the stage host. Plain `docker run` orchestration on purpose — no compose
|
|
# project files, no bind mounts, nothing shared with the real stages.
|
|
set -eu
|
|
|
|
SOURCE_VOLUME="${SOURCE_VOLUME:-dorfteich-test_backups}"
|
|
IMAGE_BASE="${IMAGE_BASE:-gitea.101010.cloud/stwaidele/dorfteich}"
|
|
TAG="${TAG:-test}"
|
|
P="dorfteich-drill-$(date +%s)-$$"
|
|
NET="$P-net"
|
|
PGPASS="drill-$(date +%s)"
|
|
|
|
log() { echo "drill: $*"; }
|
|
fail() { echo "drill: FAILED — $*" >&2; exit 1; }
|
|
|
|
cleanup() {
|
|
log "tearing down scratch environment ${P}"
|
|
docker rm -f "$P-db" "$P-api" >/dev/null 2>&1 || true
|
|
docker volume rm -f "$P-backups" "$P-uploads" "$P-plugins" >/dev/null 2>&1 || true
|
|
docker network rm "$NET" >/dev/null 2>&1 || true
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# --- 1. Which set is the latest success? ------------------------------------
|
|
BACKUP_ID=$(docker run --rm -v "$SOURCE_VOLUME":/backups:ro "$IMAGE_BASE-backup:$TAG" \
|
|
node -e 'const s=require("/backups/status.json"); if(!s.lastSuccess){console.error("no successful backup recorded");process.exit(2);} console.log(s.lastSuccess.backupId)') \
|
|
|| fail "no restorable set in $SOURCE_VOLUME"
|
|
log "drilling backup set $BACKUP_ID from $SOURCE_VOLUME"
|
|
|
|
# --- 2. Scratch environment (unique names, own volumes) ----------------------
|
|
docker network create "$NET" >/dev/null
|
|
docker volume create "$P-backups" >/dev/null
|
|
docker volume create "$P-uploads" >/dev/null
|
|
docker volume create "$P-plugins" >/dev/null
|
|
|
|
# Copy exactly the drilled set; the source stays read-only. Root, because a
|
|
# freshly created named volume is root-owned until chown'd for the node user.
|
|
docker run --rm --user root -v "$SOURCE_VOLUME":/src:ro -v "$P-backups":/dst "$IMAGE_BASE-backup:$TAG" \
|
|
sh -c "cp /src/db-$BACKUP_ID.dump /src/files-$BACKUP_ID.tar.gz /src/status.json /dst/ && chown -R node:node /dst" \
|
|
|| fail "backup set $BACKUP_ID is incomplete in $SOURCE_VOLUME"
|
|
|
|
# The restore untars into these; make them writable for the node user too.
|
|
docker run --rm --user root -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins \
|
|
"$IMAGE_BASE-backup:$TAG" chown node:node /data/uploads /data/plugins
|
|
|
|
docker run -d --name "$P-db" --network "$NET" \
|
|
-e POSTGRES_USER=dorfteich -e POSTGRES_PASSWORD="$PGPASS" -e POSTGRES_DB=dorfteich \
|
|
postgres:17.5-alpine >/dev/null
|
|
for _ in $(seq 1 30); do
|
|
docker exec "$P-db" pg_isready -U dorfteich -d dorfteich >/dev/null 2>&1 && break
|
|
sleep 2
|
|
done
|
|
docker exec "$P-db" pg_isready -U dorfteich -d dorfteich >/dev/null || fail "scratch db never became ready"
|
|
|
|
DATABASE_URL="postgresql://dorfteich:$PGPASS@$P-db:5432/dorfteich"
|
|
|
|
# --- 3. Restore (same code path as restore.sh uses) --------------------------
|
|
docker run --rm --network "$NET" \
|
|
-e DATABASE_URL="$DATABASE_URL" \
|
|
-v "$P-backups":/backups -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins \
|
|
"$IMAGE_BASE-backup:$TAG" node dist/restore.js "$BACKUP_ID" \
|
|
|| fail "restore of $BACKUP_ID did not complete"
|
|
|
|
# --- 4. Boot the api against the restored data -------------------------------
|
|
docker run -d --name "$P-api" --network "$NET" \
|
|
-e DATABASE_URL="$DATABASE_URL" \
|
|
-v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins -v "$P-backups":/data/backups:ro \
|
|
"$IMAGE_BASE-api:$TAG" >/dev/null
|
|
|
|
READY=""
|
|
for _ in $(seq 1 45); do
|
|
READY=$(docker run --rm --network "$NET" curlimages/curl:8.10.1 -s "http://$P-api:3000/api/v1/readyz" || true)
|
|
case "$READY" in *'"database","status":"ok"'*) break ;; esac
|
|
sleep 2
|
|
done
|
|
case "$READY" in
|
|
*'"database","status":"ok"'*) log "readyz: database ok" ;;
|
|
*) fail "api never became ready on the restored data: $READY" ;;
|
|
esac
|
|
case "$READY" in
|
|
*'"migrations","status":"ok"'*) log "readyz: migrations ok" ;;
|
|
*) fail "migration state broken after restore: $READY" ;;
|
|
esac
|
|
|
|
# --- 5. Sanity checks ---------------------------------------------------------
|
|
psql_scalar() {
|
|
docker exec "$P-db" psql -U dorfteich -d dorfteich -t -A -c "$1"
|
|
}
|
|
|
|
# Trashed pages count too: this proves restorability, not content policy —
|
|
# on Test the e2e packs routinely leave every fixture page in the trash.
|
|
USERS=$(psql_scalar 'SELECT count(*) FROM users;')
|
|
PAGES=$(psql_scalar 'SELECT count(*) FROM pages;')
|
|
[ "$USERS" -ge 1 ] || fail "restored database has no users"
|
|
[ "$PAGES" -ge 1 ] || fail "restored database has no pages"
|
|
log "row counts: $USERS users, $PAGES pages (incl. trash)"
|
|
|
|
# One page must have rendered content in the cache (the read path's source).
|
|
RENDERED=$(psql_scalar "SELECT count(*) FROM page_content_cache WHERE length(html) > 0;")
|
|
[ "$RENDERED" -ge 1 ] || fail "no page has rendered content after restore"
|
|
log "rendered pages in content cache: $RENDERED"
|
|
|
|
# API render proof without stage credentials: the public legal endpoint
|
|
# exercises routing + db + the HTML pipeline end to end.
|
|
LEGAL_CODE=$(docker run --rm --network "$NET" curlimages/curl:8.10.1 \
|
|
-s -o /dev/null -w '%{http_code}' "http://$P-api:3000/api/v1/legal/imprint/content")
|
|
[ "$LEGAL_CODE" = "200" ] || fail "public api request failed with $LEGAL_CODE"
|
|
log "public api render check: 200"
|
|
|
|
# Byte-check one media file: volume content must match the database row.
|
|
ATTACHMENT=$(psql_scalar "SELECT storage_path || ':' || size_bytes FROM attachments LIMIT 1;")
|
|
if [ -n "$ATTACHMENT" ]; then
|
|
REL_PATH=${ATTACHMENT%%:*}
|
|
EXPECTED=${ATTACHMENT##*:}
|
|
ACTUAL=$(docker exec "$P-api" node -e "console.log(require('fs').statSync('/data/uploads/$REL_PATH').size)") \
|
|
|| fail "media file $REL_PATH missing from the restored uploads volume"
|
|
[ "$ACTUAL" = "$EXPECTED" ] || fail "media file $REL_PATH has $ACTUAL bytes, database says $EXPECTED"
|
|
log "media byte-check: $REL_PATH ($ACTUAL bytes) matches"
|
|
else
|
|
log "media byte-check: skipped (no attachments in this backup)"
|
|
fi
|
|
|
|
log "OK — set $BACKUP_ID restored and verified ($USERS users, $PAGES pages, $RENDERED rendered)"
|