#!/usr/bin/env sh # Release-candidate operations QA (issue #90): the automated proof, run on # every release BEFORE the manual prod gate (release.yml job "ops-qa"), that # the operational promises hold for exactly the images a human is about to # approve: # # 1. UPDATE SIMULATION — boot the PREVIOUS release with pre-seeded fixture # content, swap the api to the CANDIDATE against the same database: # migrations apply automatically, readiness is green, the content # survives. Future migrations extend `seed_fixture`/`assert_fixture` # below with their own fixtures — that pair IS the update-fixture # contract (one write on N-1, one read on N). # 2. BACKUP/RESTORE ROUNDTRIP — the candidate's backup sidecar dumps the # stack, the set restores into a second, empty database, and the # candidate api serves the content from it. Distinct from the monthly # drill (#87): this runs per release candidate, against candidate # images, never against stage data. # 3. DEGRADED READYZ — without converter/renderer sidecars and before the # first backup, readyz must answer 200 with `degraded` and warn-level # checks (never 503) — the #85 semantics, proven on the candidate. # # Environment: # NEXT_TAG the candidate release tag (required), e.g. v0.1.2 # PREV_TAG the previous release (empty skips the update simulation) # IMAGE_BASE default gitea.101010.cloud/stwaidele/dorfteich set -eu NEXT_TAG="${NEXT_TAG:?set NEXT_TAG}" PREV_TAG="${PREV_TAG:-}" IMAGE_BASE="${IMAGE_BASE:-gitea.101010.cloud/stwaidele/dorfteich}" P="dorfteich-rqa-$(date +%s)-$$" NET="$P-net" PGPASS="rqa-$(date +%s)" ADMIN_PASS="release qa admin pass 1" SESSION="" log() { echo "release-qa: $*"; } fail() { echo "release-qa: FAILED — $*" >&2; exit 1; } cleanup() { log "tearing down $P" docker rm -f "$P-db" "$P-db2" "$P-api" >/dev/null 2>&1 || true docker volume rm -f "$P-backups" "$P-uploads" "$P-plugins" >/dev/null 2>&1 || true docker network rm "$NET" >/dev/null 2>&1 || true } trap cleanup EXIT # No cookie file: the CI job talks to the HOST docker daemon, so bind # mounts would resolve host paths — the session travels as a header instead. api_curl() { # api_curl [json-body] METHOD=$1; APIPATH=$2; BODY=${3:-} docker run --rm --network "$NET" curlimages/curl:8.10.1 \ -s ${SESSION:+-H "Cookie: $SESSION"} -X "$METHOD" \ ${BODY:+-H 'Content-Type: application/json' --data "$BODY"} \ "http://$P-api:3000/api/v1$APIPATH" } start_api() { # start_api docker rm -f "$P-api" >/dev/null 2>&1 || true docker run -d --name "$P-api" --network "$NET" \ -e DATABASE_URL="$1" \ -e SETUP_ADMIN_USERNAME=qa-admin -e SETUP_ADMIN_EMAIL=qa-admin@example.org \ -e SETUP_ADMIN_PASSWORD="$ADMIN_PASS" \ -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins -v "$P-backups":/data/backups:ro \ "$IMAGE_BASE-api:$2" >/dev/null for _ in $(seq 1 45); do READY=$(docker run --rm --network "$NET" curlimages/curl:8.10.1 -s "http://$P-api:3000/api/v1/readyz" || true) case "$READY" in *'"database","status":"ok"'*) return 0 ;; esac sleep 2 done docker logs "$P-api" | tail -20 fail "api ($2) never became ready" } login() { RAW=$(docker run --rm --network "$NET" curlimages/curl:8.10.1 \ -si -X POST -H 'Content-Type: application/json' \ --data '{"usernameOrEmail":"qa-admin","password":"release qa admin pass 1"}' \ "http://$P-api:3000/api/v1/auth/login") SESSION=$(printf '%s' "$RAW" | tr -d '\r' | grep -i '^set-cookie: dt_session=' | head -1 | sed 's/^[Ss]et-[Cc]ookie: //; s/;.*//') [ -n "$SESSION" ] || fail "login failed: $(printf '%s' "$RAW" | tail -1)" } seed_fixture() { # A fresh instance defaults additional_ponds to 0 — the preseeded Site # Admin lifts their own quota through the real admin API first. ME=$(api_curl GET /auth/me) MY_ID=$(echo "$ME" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) [ -n "$MY_ID" ] || fail "could not resolve the qa-admin id: $ME" QUOTA=$(api_curl PUT "/admin/quotas/user/$MY_ID/additional_ponds" '{"value":5}') case "$QUOTA" in *'"lines"'*|*additional_ponds*) ;; *) fail "quota override failed: $QUOTA" ;; esac POND=$(api_curl POST /ponds '{"name":"Release QA"}') POND_ID=$(echo "$POND" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) [ -n "$POND_ID" ] || fail "pond creation failed: $POND" PAGE=$(api_curl POST "/ponds/$POND_ID/pages" '{"title":"Update fixture page"}') PAGE_ID=$(echo "$PAGE" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) [ -n "$PAGE_ID" ] || fail "page creation failed: $PAGE" log "fixture seeded on $1 (pond $POND_ID, page $PAGE_ID)" } assert_fixture() { RES=$(api_curl GET "/pages/$PAGE_ID/comments") case "$RES" in *'"threads"'*) ;; *) fail "fixture page unreadable on $1: $RES" ;; esac log "fixture content intact on $1" } docker network create "$NET" >/dev/null docker volume create "$P-backups" >/dev/null docker volume create "$P-uploads" >/dev/null docker volume create "$P-plugins" >/dev/null docker run --rm --user root -v "$P-backups":/b -v "$P-uploads":/u -v "$P-plugins":/p \ "$IMAGE_BASE-api:$NEXT_TAG" chown node:node /b /u /p docker run -d --name "$P-db" --network "$NET" \ -e POSTGRES_USER=dorfteich -e POSTGRES_PASSWORD="$PGPASS" -e POSTGRES_DB=dorfteich \ postgres:17.5-alpine >/dev/null for _ in $(seq 1 30); do docker exec "$P-db" pg_isready -U dorfteich >/dev/null 2>&1 && break sleep 2 done DB1="postgresql://dorfteich:$PGPASS@$P-db:5432/dorfteich" # --- 1. Update simulation ---------------------------------------------------- if [ -n "$PREV_TAG" ]; then log "update simulation: $PREV_TAG → $NEXT_TAG" start_api "$DB1" "$PREV_TAG" login seed_fixture "$PREV_TAG" start_api "$DB1" "$NEXT_TAG" # same database: migrations must auto-apply case "$READY" in *'"migrations","status":"ok"'*) log "migrations applied cleanly on upgrade" ;; *) fail "migration state broken after upgrade: $READY" ;; esac login assert_fixture "$NEXT_TAG" else log "no previous release — skipping the update simulation (first release)" start_api "$DB1" "$NEXT_TAG" login seed_fixture "$NEXT_TAG" fi # --- 3. Degraded readyz (no sidecars, no backup yet) -------------------------- case "$READY" in *'"status":"degraded"'*) log "readyz degraded (200) without sidecars — as specified" ;; *) fail "expected degraded readiness without sidecars: $READY" ;; esac case "$READY" in *'"converter","status":"warn"'*) ;; *) fail "converter check should warn without a sidecar: $READY" ;; esac # --- 2. Backup/restore roundtrip ---------------------------------------------- log "backup/restore roundtrip with $NEXT_TAG" docker run --rm --network "$NET" -e BACKUP_RUN_ONCE=1 \ -e DATABASE_URL="$DB1" \ -v "$P-backups":/backups -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins \ "$IMAGE_BASE-backup:$NEXT_TAG" || fail "candidate backup run failed" BACKUP_ID=$(docker run --rm -v "$P-backups":/backups:ro "$IMAGE_BASE-backup:$NEXT_TAG" \ node -e 'console.log(require("/backups/status.json").lastSuccess.backupId)') docker run -d --name "$P-db2" --network "$NET" \ -e POSTGRES_USER=dorfteich -e POSTGRES_PASSWORD="$PGPASS" -e POSTGRES_DB=dorfteich \ postgres:17.5-alpine >/dev/null for _ in $(seq 1 30); do docker exec "$P-db2" pg_isready -U dorfteich >/dev/null 2>&1 && break sleep 2 done DB2="postgresql://dorfteich:$PGPASS@$P-db2:5432/dorfteich" docker run --rm --network "$NET" -e DATABASE_URL="$DB2" \ -v "$P-backups":/backups -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins \ "$IMAGE_BASE-backup:$NEXT_TAG" node dist/restore.js "$BACKUP_ID" \ || fail "restore into the scratch database failed" start_api "$DB2" "$NEXT_TAG" login assert_fixture "restored-$NEXT_TAG" log "OK — update simulation, degraded readiness, and backup roundtrip all green for $NEXT_TAG"