dorfteich/deploy/release-qa.sh

#!/usr/bin/env sh
# Release-candidate operations QA (issue #90): the automated proof, run on
# every release BEFORE the manual prod gate (release.yml job "ops-qa"), that
# the operational promises hold for exactly the images a human is about to
# approve:
#
#   1. UPDATE SIMULATION  — boot the PREVIOUS release with pre-seeded fixture
#      content, swap the api to the CANDIDATE against the same database:
#      migrations apply automatically, readiness is green, the content
#      survives. Future migrations extend `seed_fixture`/`assert_fixture`
#      below with their own fixtures — that pair IS the update-fixture
#      contract (one write on N-1, one read on N).
#   2. BACKUP/RESTORE ROUNDTRIP — the candidate's backup sidecar dumps the
#      stack, the set restores into a second, empty database, and the
#      candidate api serves the content from it. Distinct from the monthly
#      drill (#87): this runs per release candidate, against candidate
#      images, never against stage data.
#   3. DEGRADED READYZ — without converter/renderer sidecars and before the
#      first backup, readyz must answer 200 with `degraded` and warn-level
#      checks (never 503) — the #85 semantics, proven on the candidate.
#
# Environment:
#   NEXT_TAG   the candidate release tag (required), e.g. v0.1.2
#   PREV_TAG   the previous release (empty skips the update simulation)
#   IMAGE_BASE default gitea.101010.cloud/stwaidele/dorfteich
set -eu

NEXT_TAG="${NEXT_TAG:?set NEXT_TAG}"
PREV_TAG="${PREV_TAG:-}"
IMAGE_BASE="${IMAGE_BASE:-gitea.101010.cloud/stwaidele/dorfteich}"
P="dorfteich-rqa-$(date +%s)-$$"
NET="$P-net"
PGPASS="rqa-$(date +%s)"
ADMIN_PASS="release qa admin pass 1"
SESSION=""
# Cookie mutations must prove their origin since #189 (CSRF fail-closed).
# The api pins APP_BASE_URL to this value and every api_curl sends it.
QA_ORIGIN="http://release-qa.local"

log() { echo "release-qa: $*"; }
fail() { echo "release-qa: FAILED — $*" >&2; exit 1; }

cleanup() {
  log "tearing down $P"
  docker rm -f "$P-db" "$P-db2" "$P-api" >/dev/null 2>&1 || true
  docker volume rm -f "$P-backups" "$P-uploads" "$P-plugins" >/dev/null 2>&1 || true
  docker network rm "$NET" >/dev/null 2>&1 || true
}
trap cleanup EXIT

# No cookie file: the CI job talks to the HOST docker daemon, so bind
# mounts would resolve host paths — the session travels as a header instead.
api_curl() { # api_curl <method> <path> [json-body]
  METHOD=$1; APIPATH=$2; BODY=${3:-}
  docker run --rm --network "$NET" curlimages/curl:8.10.1 \
    -s ${SESSION:+-H "Cookie: $SESSION"} -H "Origin: $QA_ORIGIN" -X "$METHOD" \
    ${BODY:+-H 'Content-Type: application/json' --data "$BODY"} \
    "http://$P-api:3000/api/v1$APIPATH"
}

start_api() { # start_api <tag>
  docker rm -f "$P-api" >/dev/null 2>&1 || true
  docker run -d --name "$P-api" --network "$NET" \
    -e DATABASE_URL="$1" -e APP_BASE_URL="$QA_ORIGIN" \
    -e SETUP_ADMIN_USERNAME=qa-admin -e SETUP_ADMIN_EMAIL=qa-admin@example.org \
    -e SETUP_ADMIN_PASSWORD="$ADMIN_PASS" \
    -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins -v "$P-backups":/data/backups:ro \
    "$IMAGE_BASE-api:$2" >/dev/null
  for _ in $(seq 1 45); do
    READY=$(docker run --rm --network "$NET" curlimages/curl:8.10.1 -s "http://$P-api:3000/api/v1/readyz" || true)
    case "$READY" in *'"database","status":"ok"'*) return 0 ;; esac
    sleep 2
  done
  docker logs "$P-api" | tail -20
  fail "api ($2) never became ready"
}

login() {
  RAW=$(docker run --rm --network "$NET" curlimages/curl:8.10.1 \
    -si -X POST -H 'Content-Type: application/json' \
    --data '{"usernameOrEmail":"qa-admin","password":"release qa admin pass 1"}' \
    "http://$P-api:3000/api/v1/auth/login")
  SESSION=$(printf '%s' "$RAW" | tr -d '\r' | grep -i '^set-cookie: dt_session=' | head -1 | sed 's/^[Ss]et-[Cc]ookie: //; s/;.*//')
  [ -n "$SESSION" ] || fail "login failed: $(printf '%s' "$RAW" | tail -1)"
}

seed_fixture() {
  # A fresh instance defaults additional_ponds to 0 — the preseeded Site
  # Admin lifts their own quota through the real admin API first.
  ME=$(api_curl GET /auth/me)
  MY_ID=$(echo "$ME" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4)
  [ -n "$MY_ID" ] || fail "could not resolve the qa-admin id: $ME"
  QUOTA=$(api_curl PUT "/admin/quotas/user/$MY_ID/additional_ponds" '{"value":5}')
  case "$QUOTA" in *'"lines"'*|*additional_ponds*) ;; *) fail "quota override failed: $QUOTA" ;; esac
  POND=$(api_curl POST /ponds '{"name":"Release QA"}')
  POND_ID=$(echo "$POND" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4)
  [ -n "$POND_ID" ] || fail "pond creation failed: $POND"
  PAGE=$(api_curl POST "/ponds/$POND_ID/pages" '{"title":"Update fixture page"}')
  PAGE_ID=$(echo "$PAGE" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4)
  [ -n "$PAGE_ID" ] || fail "page creation failed: $PAGE"
  log "fixture seeded on $1 (pond $POND_ID, page $PAGE_ID)"
}

assert_fixture() {
  RES=$(api_curl GET "/pages/$PAGE_ID/comments")
  case "$RES" in *'"threads"'*) ;; *) fail "fixture page unreadable on $1: $RES" ;; esac
  log "fixture content intact on $1"
}

docker network create "$NET" >/dev/null
docker volume create "$P-backups" >/dev/null
docker volume create "$P-uploads" >/dev/null
docker volume create "$P-plugins" >/dev/null
docker run --rm --user root -v "$P-backups":/b -v "$P-uploads":/u -v "$P-plugins":/p \
  "$IMAGE_BASE-api:$NEXT_TAG" chown node:node /b /u /p
docker run -d --name "$P-db" --network "$NET" \
  -e POSTGRES_USER=dorfteich -e POSTGRES_PASSWORD="$PGPASS" -e POSTGRES_DB=dorfteich \
  postgres:17.5-alpine >/dev/null
for _ in $(seq 1 30); do
  docker exec "$P-db" pg_isready -U dorfteich >/dev/null 2>&1 && break
  sleep 2
done
DB1="postgresql://dorfteich:$PGPASS@$P-db:5432/dorfteich"

# --- 1. Update simulation ----------------------------------------------------
if [ -n "$PREV_TAG" ]; then
  log "update simulation: $PREV_TAG → $NEXT_TAG"
  start_api "$DB1" "$PREV_TAG"
  login
  seed_fixture "$PREV_TAG"
  start_api "$DB1" "$NEXT_TAG" # same database: migrations must auto-apply
  case "$READY" in
    *'"migrations","status":"ok"'*) log "migrations applied cleanly on upgrade" ;;
    *) fail "migration state broken after upgrade: $READY" ;;
  esac
  login
  assert_fixture "$NEXT_TAG"
else
  log "no previous release — skipping the update simulation (first release)"
  start_api "$DB1" "$NEXT_TAG"
  login
  seed_fixture "$NEXT_TAG"
fi

# --- 3. Degraded readyz (no sidecars, no backup yet) --------------------------
case "$READY" in
  *'"status":"degraded"'*) log "readyz degraded (200) without sidecars — as specified" ;;
  *) fail "expected degraded readiness without sidecars: $READY" ;;
esac
case "$READY" in
  *'"converter","status":"warn"'*) ;;
  *) fail "converter check should warn without a sidecar: $READY" ;;
esac

# --- 2. Backup/restore roundtrip ----------------------------------------------
log "backup/restore roundtrip with $NEXT_TAG"
docker run --rm --network "$NET" -e BACKUP_RUN_ONCE=1 \
  -e DATABASE_URL="$DB1" \
  -v "$P-backups":/backups -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins \
  "$IMAGE_BASE-backup:$NEXT_TAG" || fail "candidate backup run failed"
BACKUP_ID=$(docker run --rm -v "$P-backups":/backups:ro "$IMAGE_BASE-backup:$NEXT_TAG" \
  node -e 'console.log(require("/backups/status.json").lastSuccess.backupId)')

docker run -d --name "$P-db2" --network "$NET" \
  -e POSTGRES_USER=dorfteich -e POSTGRES_PASSWORD="$PGPASS" -e POSTGRES_DB=dorfteich \
  postgres:17.5-alpine >/dev/null
for _ in $(seq 1 30); do
  docker exec "$P-db2" pg_isready -U dorfteich >/dev/null 2>&1 && break
  sleep 2
done
DB2="postgresql://dorfteich:$PGPASS@$P-db2:5432/dorfteich"
docker run --rm --network "$NET" -e DATABASE_URL="$DB2" \
  -v "$P-backups":/backups -v "$P-uploads":/data/uploads -v "$P-plugins":/data/plugins \
  "$IMAGE_BASE-backup:$NEXT_TAG" node dist/restore.js "$BACKUP_ID" \
  || fail "restore into the scratch database failed"
start_api "$DB2" "$NEXT_TAG"
login
assert_fixture "restored-$NEXT_TAG"

log "OK — update simulation, degraded readiness, and backup roundtrip all green for $NEXT_TAG"