diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 875dd85..3aec54d 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -63,3 +63,25 @@ jobs: --data @release.json \ "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases" \ > /dev/null && echo "release $TAG published" + + # Operations QA (issue #90): the pre-approval gate. A human pushes the + # prod-vX.Y.Z tag only after BOTH jobs of this release run are green. + ops-qa: + name: Release-candidate operations QA + needs: build-release + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Log in to the Gitea registry + run: printf '%s' "${{ secrets.REGISTRY_TOKEN }}" | tr -d '[:space:]' | docker login gitea.101010.cloud -u fable-5 --password-stdin + + - name: Run update simulation, degraded readiness, and backup roundtrip + run: | + PREV=$(git tag --list 'v*.*.*' --sort=-v:refname | grep -vx "$GITHUB_REF_NAME" | head -n1 || true) + NEXT_TAG=$GITHUB_REF_NAME PREV_TAG=$PREV IMAGE_BASE=$IMAGE_BASE \ + sh deploy/release-qa.sh diff --git a/deploy/release-qa.sh b/deploy/release-qa.sh new file mode 100755 index 0000000..a73367b --- /dev/null +++ b/deploy/release-qa.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env sh +# Release-candidate operations QA (issue #90): the automated proof, run on +# every release BEFORE the manual prod gate (release.yml job "ops-qa"), that +# the operational promises hold for exactly the images a human is about to +# approve: +# +# 1. UPDATE SIMULATION — boot the PREVIOUS release with pre-seeded fixture +# content, swap the api to the CANDIDATE against the same database: +# migrations apply automatically, readiness is green, the content +# survives. Future migrations extend `seed_fixture`/`assert_fixture` +# below with their own fixtures — that pair IS the update-fixture +# contract (one write on N-1, one read on N). +# 2. BACKUP/RESTORE ROUNDTRIP — the candidate's backup sidecar dumps the +# stack, the set restores into a second, empty database, and the +# candidate api serves the content from it. Distinct from the monthly +# drill (#87): this runs per release candidate, against candidate +# images, never against stage data. +# 3. DEGRADED READYZ — without converter/renderer sidecars and before the +# first backup, readyz must answer 200 with `degraded` and warn-level +# checks (never 503) — the #85 semantics, proven on the candidate. +# +# Environment: +# NEXT_TAG the candidate release tag (required), e.g. v0.1.2 +# PREV_TAG the previous release (empty skips the update simulation) +# IMAGE_BASE default gitea.101010.cloud/stwaidele/dorfteich +set -eu + +NEXT_TAG="${NEXT_TAG:?set NEXT_TAG}" +PREV_TAG="${PREV_TAG:-}" +IMAGE_BASE="${IMAGE_BASE:-gitea.101010.cloud/stwaidele/dorfteich}" +P="dorfteich-rqa-$(date +%s)-$$" +NET="$P-net" +PGPASS="rqa-$(date +%s)" +ADMIN_PASS="release qa admin pass 1" +SESSION="" + +log() { echo "release-qa: $*"; } +fail() { echo "release-qa: FAILED — $*" >&2; exit 1; } + +cleanup() { + log "tearing down $P" + docker rm -f "$P-db" "$P-db2" "$P-api" >/dev/null 2>&1 || true + docker volume rm -f "$P-backups" "$P-uploads" "$P-plugins" >/dev/null 2>&1 || true + docker network rm "$NET" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +# No cookie file: the CI job talks to the HOST docker daemon, so bind +# mounts would resolve host paths — the session travels as a header instead. +api_curl() { # api_curl [json-body] + METHOD=$1; APIPATH=$2; BODY=${3:-} + docker run --rm --network "$NET" curlimages/curl:8.10.1 \ + -s ${SESSION:+-H "Cookie: $SESSION"} -X "$METHOD" \ + ${BODY:+-H 'Content-Type: application/json' --data "$BODY"} \ + "http://$P-api:3000/api/v1$APIPATH" +} + +start_api() { # start_api + docker rm -f "$P-api" >/dev/null 2>&1 || true + docker run -d --name "$P-api" --network "$NET" \ + -e DATABASE_URL="$1" \ + -e SETUP_ADMIN_USERNAME=qa-admin -e SETUP_ADMIN_EMAIL=qa-admin@example.org \ + -e SETUP_ADMIN_PASSWORD="$ADMIN_PASS" \ + -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins -v "$P-backups":/data/backups:ro \ + "$IMAGE_BASE-api:$2" >/dev/null + for _ in $(seq 1 45); do + READY=$(docker run --rm --network "$NET" curlimages/curl:8.10.1 -s "http://$P-api:3000/api/v1/readyz" || true) + case "$READY" in *'"database","status":"ok"'*) return 0 ;; esac + sleep 2 + done + docker logs "$P-api" | tail -20 + fail "api ($2) never became ready" +} + +login() { + RAW=$(docker run --rm --network "$NET" curlimages/curl:8.10.1 \ + -si -X POST -H 'Content-Type: application/json' \ + --data '{"usernameOrEmail":"qa-admin","password":"release qa admin pass 1"}' \ + "http://$P-api:3000/api/v1/auth/login") + SESSION=$(printf '%s' "$RAW" | tr -d '\r' | grep -i '^set-cookie: dt_session=' | head -1 | sed 's/^[Ss]et-[Cc]ookie: //; s/;.*//') + [ -n "$SESSION" ] || fail "login failed: $(printf '%s' "$RAW" | tail -1)" +} + +seed_fixture() { + # A fresh instance defaults additional_ponds to 0 — the preseeded Site + # Admin lifts their own quota through the real admin API first. + ME=$(api_curl GET /auth/me) + MY_ID=$(echo "$ME" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) + [ -n "$MY_ID" ] || fail "could not resolve the qa-admin id: $ME" + QUOTA=$(api_curl PUT "/admin/quotas/user/$MY_ID/additional_ponds" '{"value":5}') + case "$QUOTA" in *'"lines"'*|*additional_ponds*) ;; *) fail "quota override failed: $QUOTA" ;; esac + POND=$(api_curl POST /ponds '{"name":"Release QA"}') + POND_ID=$(echo "$POND" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) + [ -n "$POND_ID" ] || fail "pond creation failed: $POND" + PAGE=$(api_curl POST "/ponds/$POND_ID/pages" '{"title":"Update fixture page"}') + PAGE_ID=$(echo "$PAGE" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) + [ -n "$PAGE_ID" ] || fail "page creation failed: $PAGE" + log "fixture seeded on $1 (pond $POND_ID, page $PAGE_ID)" +} + +assert_fixture() { + RES=$(api_curl GET "/pages/$PAGE_ID/comments") + case "$RES" in *'"threads"'*) ;; *) fail "fixture page unreadable on $1: $RES" ;; esac + log "fixture content intact on $1" +} + +docker network create "$NET" >/dev/null +docker volume create "$P-backups" >/dev/null +docker volume create "$P-uploads" >/dev/null +docker volume create "$P-plugins" >/dev/null +docker run --rm --user root -v "$P-backups":/b -v "$P-uploads":/u -v "$P-plugins":/p \ + "$IMAGE_BASE-api:$NEXT_TAG" chown node:node /b /u /p +docker run -d --name "$P-db" --network "$NET" \ + -e POSTGRES_USER=dorfteich -e POSTGRES_PASSWORD="$PGPASS" -e POSTGRES_DB=dorfteich \ + postgres:17.5-alpine >/dev/null +for _ in $(seq 1 30); do + docker exec "$P-db" pg_isready -U dorfteich >/dev/null 2>&1 && break + sleep 2 +done +DB1="postgresql://dorfteich:$PGPASS@$P-db:5432/dorfteich" + +# --- 1. Update simulation ---------------------------------------------------- +if [ -n "$PREV_TAG" ]; then + log "update simulation: $PREV_TAG → $NEXT_TAG" + start_api "$DB1" "$PREV_TAG" + login + seed_fixture "$PREV_TAG" + start_api "$DB1" "$NEXT_TAG" # same database: migrations must auto-apply + case "$READY" in + *'"migrations","status":"ok"'*) log "migrations applied cleanly on upgrade" ;; + *) fail "migration state broken after upgrade: $READY" ;; + esac + login + assert_fixture "$NEXT_TAG" +else + log "no previous release — skipping the update simulation (first release)" + start_api "$DB1" "$NEXT_TAG" + login + seed_fixture "$NEXT_TAG" +fi + +# --- 3. Degraded readyz (no sidecars, no backup yet) -------------------------- +case "$READY" in + *'"status":"degraded"'*) log "readyz degraded (200) without sidecars — as specified" ;; + *) fail "expected degraded readiness without sidecars: $READY" ;; +esac +case "$READY" in + *'"converter","status":"warn"'*) ;; + *) fail "converter check should warn without a sidecar: $READY" ;; +esac + +# --- 2. Backup/restore roundtrip ---------------------------------------------- +log "backup/restore roundtrip with $NEXT_TAG" +docker run --rm --network "$NET" -e BACKUP_RUN_ONCE=1 \ + -e DATABASE_URL="$DB1" \ + -v "$P-backups":/backups -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins \ + "$IMAGE_BASE-backup:$NEXT_TAG" || fail "candidate backup run failed" +BACKUP_ID=$(docker run --rm -v "$P-backups":/backups:ro "$IMAGE_BASE-backup:$NEXT_TAG" \ + node -e 'console.log(require("/backups/status.json").lastSuccess.backupId)') + +docker run -d --name "$P-db2" --network "$NET" \ + -e POSTGRES_USER=dorfteich -e POSTGRES_PASSWORD="$PGPASS" -e POSTGRES_DB=dorfteich \ + postgres:17.5-alpine >/dev/null +for _ in $(seq 1 30); do + docker exec "$P-db2" pg_isready -U dorfteich >/dev/null 2>&1 && break + sleep 2 +done +DB2="postgresql://dorfteich:$PGPASS@$P-db2:5432/dorfteich" +docker run --rm --network "$NET" -e DATABASE_URL="$DB2" \ + -v "$P-backups":/backups -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins \ + "$IMAGE_BASE-backup:$NEXT_TAG" node dist/restore.js "$BACKUP_ID" \ + || fail "restore into the scratch database failed" +start_api "$DB2" "$NEXT_TAG" +login +assert_fixture "restored-$NEXT_TAG" + +log "OK — update simulation, degraded readiness, and backup roundtrip all green for $NEXT_TAG" diff --git a/docs/architecture/deployment.md b/docs/architecture/deployment.md index a6e0b40..2dd85d2 100644 --- a/docs/architecture/deployment.md +++ b/docs/architecture/deployment.md @@ -115,4 +115,8 @@ flowchart LR proxy or the `caddy` profile). Setup = compose up + browser wizard. - Updates: `docker compose pull && up -d` on a new semver tag; migrations run automatically; the release notes flag anything manual. Downgrades are - supported one release back. + supported one release back. Every release candidate passes the automated + operations QA (`deploy/release-qa.sh`, issue #90: update simulation from + the previous release, degraded-readyz semantics, backup/restore + roundtrip) before the manual prod gate; new migrations add their own + fixture to the script's `seed_fixture`/`assert_fixture` pair.