diff --git a/.gitea/workflows/cd.yml b/.gitea/workflows/cd.yml index 10f8e9e..7e30f38 100644 --- a/.gitea/workflows/cd.yml +++ b/.gitea/workflows/cd.yml @@ -47,6 +47,13 @@ jobs: docker push $IMAGE_BASE-collab:${{ github.sha }} docker push $IMAGE_BASE-collab:test + - name: Build and push backup image + run: | + docker build -f apps/backup/Dockerfile --build-arg APP_VERSION=${{ github.sha }} \ + -t $IMAGE_BASE-backup:${{ github.sha }} -t $IMAGE_BASE-backup:test . + docker push $IMAGE_BASE-backup:${{ github.sha }} + docker push $IMAGE_BASE-backup:test + deploy-test: name: Deploy to Test needs: build-push @@ -120,6 +127,7 @@ jobs: docker buildx imagetools create -t $IMAGE_BASE-web:int $IMAGE_BASE-web:${{ github.sha }} docker buildx imagetools create -t $IMAGE_BASE-api:int $IMAGE_BASE-api:${{ github.sha }} docker buildx imagetools create -t $IMAGE_BASE-collab:int $IMAGE_BASE-collab:${{ github.sha }} + docker buildx imagetools create -t $IMAGE_BASE-backup:int $IMAGE_BASE-backup:${{ github.sha }} - name: Set up SSH run: | diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 9c482ff..00dd2b0 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -538,3 +538,6 @@ jobs: - name: Build collab image run: docker build -f apps/collab/Dockerfile --build-arg APP_VERSION=${{ github.sha }} -t dorfteich-collab:ci . + + - name: Build backup image + run: docker build -f apps/backup/Dockerfile --build-arg APP_VERSION=${{ github.sha }} -t dorfteich-backup:ci . diff --git a/apps/api/src/config/secret-store.ts b/apps/api/src/config/secret-store.ts index 88da4f3..b9ac2cb 100644 --- a/apps/api/src/config/secret-store.ts +++ b/apps/api/src/config/secret-store.ts @@ -2,44 +2,16 @@ import { existsSync, readFileSync } from 'node:fs'; import { chmod, mkdir, rename, writeFile } from 'node:fs/promises'; import { dirname } from 'node:path'; +import { parseSecretsFile, serializeSecrets } from '@dorfteich/shared'; + /** - * The env-backed secret store (security.md §Secrets, issue #80): secrets the - * setup wizard collects in the browser (SMTP credentials) are persisted as a - * mode-600 dotenv-style file on a volume — never as database rows. The file - * extends the environment: `overlayEnv` fills only variables the process - * environment does not set, so the stage `.env` always stays authoritative. + * File I/O for the env-backed secret store (security.md §Secrets, issue #80). + * The pure format/merge helpers (`parseSecretsFile`, `serializeSecrets`, + * `overlayEnv`) moved to `@dorfteich/shared` in issue #83 so the backup + * sidecar can read the same store; they are re-exported here to keep the + * api-internal import paths stable. */ - -/** Parses the dotenv-style store content. Ignores blank lines and comments. */ -export function parseSecretsFile(content: string): Record { - const secrets: Record = {}; - for (const line of content.split('\n')) { - const trimmed = line.trim(); - if (!trimmed || trimmed.startsWith('#')) continue; - const eq = trimmed.indexOf('='); - if (eq <= 0) continue; - const key = trimmed.slice(0, eq).trim(); - let value = trimmed.slice(eq + 1).trim(); - if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) { - value = value.slice(1, -1).replace(/\\n/g, '\n').replace(/\\"/g, '"').replace(/\\\\/g, '\\'); - } - secrets[key] = value; - } - return secrets; -} - -/** Serializes secrets with double-quoted, escaped values (dotenv-compatible). */ -export function serializeSecrets(secrets: Record): string { - const lines = [ - '# Managed by Dorfteich (setup wizard). Values here fill environment', - '# variables that the container environment does not set explicitly.', - ]; - for (const [key, value] of Object.entries(secrets)) { - const escaped = value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n'); - lines.push(`${key}="${escaped}"`); - } - return lines.join('\n') + '\n'; -} +export { overlayEnv, parseSecretsFile, serializeSecrets } from '@dorfteich/shared'; /** Reads the store file; a missing file is an empty store, not an error. */ export function readSecretsFile(path: string): Record { @@ -47,27 +19,6 @@ export function readSecretsFile(path: string): Record { return parseSecretsFile(readFileSync(path, 'utf8')); } -/** - * Merges the store under the real environment: explicit env vars win, store - * values fill the gaps (and Zod defaults fill whatever remains at parse - * time). Empty strings count as unset on both sides — compose passes - * `${SMTP_HOST:-}` as `""` for variables the stage `.env` does not define, - * and those must not shadow wizard-written store values or schema defaults. - */ -export function overlayEnv( - env: Record, - secrets: Record, -): Record { - const merged: Record = {}; - for (const [key, value] of Object.entries(secrets)) { - if (value !== '') merged[key] = value; - } - for (const [key, value] of Object.entries(env)) { - if (value !== undefined && value !== '') merged[key] = value; - } - return merged; -} - /** * Merges entries into the store file atomically (staging file + rename, so a * crash mid-write never leaves a torn file) and keeps it owner-only readable. diff --git a/apps/backup/Dockerfile b/apps/backup/Dockerfile new file mode 100644 index 0000000..49d7634 --- /dev/null +++ b/apps/backup/Dockerfile @@ -0,0 +1,31 @@ +# Build context is the repository root (workspace build): +# docker build -f apps/backup/Dockerfile . + +FROM node:22.15-alpine AS build +WORKDIR /repo +RUN npm install -g pnpm@11 +COPY pnpm-workspace.yaml pnpm-lock.yaml package.json tsconfig.base.json ./ +COPY packages/shared ./packages/shared +COPY apps/backup ./apps/backup +RUN pnpm install --frozen-lockfile --filter @dorfteich/backup... \ + && pnpm --filter @dorfteich/shared build \ + && pnpm --filter @dorfteich/backup build \ + # Self-contained production bundle (prod deps only) at /out. + && pnpm --filter @dorfteich/backup deploy --prod --legacy /out \ + && cp -r apps/backup/dist /out/dist + +FROM node:22.15-alpine +ARG APP_VERSION=0.0.0-dev +ENV NODE_ENV=production APP_VERSION=${APP_VERSION} \ + # Baked-in volume paths (self-sufficient without compose env, like the + # api image's PLUGINS_DIR — issue #71's lesson). + BACKUPS_DIR=/backups UPLOADS_DIR=/data/uploads PLUGINS_DIR=/data/plugins \ + SECRETS_FILE=/data/secrets/secrets.env +# pg_dump/pg_restore matching the stack's postgres:17 server, GNU tar for the +# volume archives, tzdata so BACKUP_TIME honors a configured TZ. +RUN apk add --no-cache postgresql17-client tar tzdata \ + && mkdir -p /backups && chown node:node /backups +WORKDIR /app +COPY --from=build --chown=node:node /out /app +USER node +CMD ["node", "dist/index.js"] diff --git a/apps/backup/package.json b/apps/backup/package.json new file mode 100644 index 0000000..a9b648b --- /dev/null +++ b/apps/backup/package.json @@ -0,0 +1,28 @@ +{ + "name": "@dorfteich/backup", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Dorfteich backup sidecar — nightly pg_dump + volume archive with prune, status.json, and failure mail (ADR 0015)", + "license": "MIT", + "main": "dist/index.js", + "scripts": { + "build": "tsc -p tsconfig.json", + "start": "node dist/index.js", + "start:dev": "tsx watch src/index.ts", + "typecheck": "tsc --noEmit", + "test": "vitest run --passWithNoTests" + }, + "dependencies": { + "@dorfteich/shared": "workspace:*", + "nodemailer": "^9.0.3", + "pino": "^9.6.0", + "zod": "^3.25.76" + }, + "devDependencies": { + "@types/node": "^26.1.0", + "@types/nodemailer": "^8.0.1", + "tsx": "^4.19.0", + "vitest": "^3.0.0" + } +} diff --git a/apps/backup/src/archive.test.ts b/apps/backup/src/archive.test.ts new file mode 100644 index 0000000..b73b453 --- /dev/null +++ b/apps/backup/src/archive.test.ts @@ -0,0 +1,55 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { archiveBase, createArchive, extractArchive } from './archive.js'; + +let dir: string; + +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'dorfteich-archive-')); +}); + +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +describe('archiveBase', () => { + it('requires a shared parent directory', () => { + expect(() => archiveBase(['/data/uploads', '/other/plugins'])).toThrow(/share one parent/); + expect(archiveBase(['/data/uploads', '/data/plugins'])).toEqual({ + base: '/data', + names: ['uploads', 'plugins'], + }); + }); +}); + +describe('create/extract round-trip (real tar)', () => { + it('restores the archived files byte-identically', async () => { + const source = join(dir, 'source'); + await mkdir(join(source, 'uploads', 'ab'), { recursive: true }); + await mkdir(join(source, 'plugins'), { recursive: true }); + await writeFile(join(source, 'uploads', 'ab', 'file.png'), 'png-bytes'); + await writeFile(join(source, 'plugins', 'manifest.json'), '{"id":"toc"}'); + + const archive = join(dir, 'files.tar.gz'); + await createArchive(archive, [join(source, 'uploads'), join(source, 'plugins')]); + + const target = join(dir, 'target'); + await mkdir(target, { recursive: true }); + await extractArchive(archive, [join(target, 'uploads'), join(target, 'plugins')]); + + expect(await readFile(join(target, 'uploads', 'ab', 'file.png'), 'utf8')).toBe('png-bytes'); + expect(await readFile(join(target, 'plugins', 'manifest.json'), 'utf8')).toBe('{"id":"toc"}'); + }); + + it('produces a valid empty archive when no data directory exists yet', async () => { + const archive = join(dir, 'empty.tar.gz'); + await createArchive(archive, [join(dir, 'none', 'uploads'), join(dir, 'none', 'plugins')]); + const target = join(dir, 'extract'); + await mkdir(target, { recursive: true }); + await extractArchive(archive, [join(target, 'uploads'), join(target, 'plugins')]); + }); +}); diff --git a/apps/backup/src/archive.ts b/apps/backup/src/archive.ts new file mode 100644 index 0000000..d762dc6 --- /dev/null +++ b/apps/backup/src/archive.ts @@ -0,0 +1,50 @@ +import { execFile } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { basename, dirname } from 'node:path'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +/** + * The uploads and plugins directories travel in one tar archive per restore + * set (ADR 0015). Both must share a parent directory (in the compose stack + * that is `/data`): entries are stored relative to it (`uploads/…`, + * `plugins/…`), so extraction lands exactly where the api reads. + */ +export function archiveBase(dataDirs: string[]): { base: string; names: string[] } { + const bases = new Set(dataDirs.map((dir) => dirname(dir))); + if (bases.size !== 1) { + throw new Error( + `data directories must share one parent to form a single archive, got: ${dataDirs.join(', ')}`, + ); + } + return { base: [...bases][0]!, names: dataDirs.map((dir) => basename(dir)) }; +} + +async function runTar(args: string[]): Promise { + try { + await execFileAsync('tar', args, { maxBuffer: 16 * 1024 * 1024 }); + } catch (error) { + const stderr = (error as { stderr?: string }).stderr?.trim(); + throw new Error(`tar failed: ${stderr || (error as Error).message}`); + } +} + +/** Creates the gzip'd volume archive; directories that do not exist yet + * (fresh instance without uploads) are skipped, an empty set still produces + * a valid empty archive. */ +export async function createArchive(outFile: string, dataDirs: string[]): Promise { + const { base, names } = archiveBase(dataDirs); + const existing = names.filter((name) => existsSync(`${base}/${name}`)); + if (existing.length === 0) { + await runTar(['-czf', outFile, '--files-from', '/dev/null']); + return; + } + await runTar(['-czf', outFile, '-C', base, ...existing]); +} + +/** Unpacks a volume archive back over the data directories (restore path). */ +export async function extractArchive(archiveFile: string, dataDirs: string[]): Promise { + const { base } = archiveBase(dataDirs); + await runTar(['-xzf', archiveFile, '-C', base]); +} diff --git a/apps/backup/src/backup-set.test.ts b/apps/backup/src/backup-set.test.ts new file mode 100644 index 0000000..ab8a951 --- /dev/null +++ b/apps/backup/src/backup-set.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest'; + +import { + archiveFileName, + backupIdTime, + dumpFileName, + expiredSets, + listSets, + newBackupId, +} from './backup-set.js'; + +const DAY = 24 * 60 * 60 * 1000; + +function setFiles(id: string): string[] { + return [dumpFileName(id), archiveFileName(id)]; +} + +describe('backup ids', () => { + it('round-trips through the id format', () => { + const now = new Date('2026-07-11T03:00:00Z'); + const id = newBackupId(now); + expect(id).toBe('20260711-030000'); + expect(backupIdTime(id)?.toISOString()).toBe(now.toISOString()); + }); + + it('rejects malformed ids', () => { + expect(backupIdTime('not-a-backup')).toBeNull(); + expect(backupIdTime('20260711')).toBeNull(); + }); +}); + +describe('listSets', () => { + it('groups files by id and flags completeness', () => { + const sets = listSets([ + 'status.json', + 'db-20260701-030000.dump.partial', + ...setFiles('20260710-030000'), + 'db-20260711-030000.dump', + ]); + expect(sets.map((set) => set.id)).toEqual(['20260710-030000', '20260711-030000']); + expect(sets[0]?.complete).toBe(true); + expect(sets[1]?.complete).toBe(false); + }); +}); + +describe('expiredSets', () => { + const now = new Date('2026-07-31T04:00:00Z'); + + it('removes only sets past retention', () => { + const fresh = newBackupId(new Date(now.getTime() - 2 * DAY)); + const old = newBackupId(new Date(now.getTime() - 10 * DAY)); + const newest = newBackupId(new Date(now.getTime() - 1 * DAY)); + const sets = listSets([...setFiles(old), ...setFiles(fresh), ...setFiles(newest)]); + + const expired = expiredSets(sets, now, 7); + expect(expired.map((set) => set.id)).toEqual([old]); + }); + + it('never removes the newest complete set, even when expired', () => { + const older = newBackupId(new Date(now.getTime() - 40 * DAY)); + const newestComplete = newBackupId(new Date(now.getTime() - 20 * DAY)); + // A newer but incomplete set (crashed run) must not shield the complete one. + const newerIncomplete = newBackupId(new Date(now.getTime() - 9 * DAY)); + const sets = listSets([ + ...setFiles(older), + ...setFiles(newestComplete), + dumpFileName(newerIncomplete), + ]); + + const expired = expiredSets(sets, now, 7); + expect(expired.map((set) => set.id)).toEqual([older, newerIncomplete]); + }); +}); diff --git a/apps/backup/src/backup-set.ts b/apps/backup/src/backup-set.ts new file mode 100644 index 0000000..6254134 --- /dev/null +++ b/apps/backup/src/backup-set.ts @@ -0,0 +1,89 @@ +/** + * A restore set (ADR 0015) is one nightly `pg_dump` plus the matching + * uploads/plugins archive, tied together by a shared backup id derived from + * the run's UTC start time. This module owns the naming scheme and the pure + * prune decision; the runner applies it to the filesystem. + */ + +export interface BackupSet { + id: string; + /** File names (not paths) present in the backups directory. */ + files: string[]; + /** Complete = both the dump and the volume archive exist. */ + complete: boolean; +} + +const ID_PATTERN = /^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})$/; +const SET_FILE_PATTERN = /^(?:db-|files-)(\d{8}-\d{6})\.(?:dump|tar\.gz)$/; + +/** Backup id for a run starting now: UTC timestamp, filesystem-safe. */ +export function newBackupId(now: Date): string { + const pad = (value: number): string => String(value).padStart(2, '0'); + return ( + `${now.getUTCFullYear()}${pad(now.getUTCMonth() + 1)}${pad(now.getUTCDate())}` + + `-${pad(now.getUTCHours())}${pad(now.getUTCMinutes())}${pad(now.getUTCSeconds())}` + ); +} + +/** The UTC time encoded in a backup id, or null for a malformed id. */ +export function backupIdTime(id: string): Date | null { + const match = ID_PATTERN.exec(id); + if (!match) return null; + const [, year, month, day, hour, minute, second] = match; + return new Date( + Date.UTC( + Number(year), + Number(month) - 1, + Number(day), + Number(hour), + Number(minute), + Number(second), + ), + ); +} + +export function dumpFileName(id: string): string { + return `db-${id}.dump`; +} + +export function archiveFileName(id: string): string { + return `files-${id}.tar.gz`; +} + +/** + * Groups the backup directory's file names into sets, oldest first. Files + * that do not belong to the naming scheme (status.json, `.partial` staging + * files of a running or crashed run) are ignored — prune never touches them. + */ +export function listSets(fileNames: string[]): BackupSet[] { + const byId = new Map(); + for (const name of fileNames) { + const match = SET_FILE_PATTERN.exec(name); + if (!match || !backupIdTime(match[1]!)) continue; + const files = byId.get(match[1]!) ?? []; + files.push(name); + byId.set(match[1]!, files); + } + return [...byId.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([id, files]) => ({ + id, + files: files.sort(), + complete: files.includes(dumpFileName(id)) && files.includes(archiveFileName(id)), + })); +} + +/** + * The sets prune may delete: older than the retention cutoff — but never + * the newest complete set, even when it is expired. A stalled instance must + * always keep one restorable set (issue #83 acceptance criteria). + */ +export function expiredSets(sets: BackupSet[], now: Date, retentionDays: number): BackupSet[] { + const cutoff = now.getTime() - retentionDays * 24 * 60 * 60 * 1000; + const newestComplete = [...sets].reverse().find((set) => set.complete); + return sets.filter((set) => { + if (set === newestComplete) return false; + const time = backupIdTime(set.id); + return time !== null && time.getTime() < cutoff; + }); +} diff --git a/apps/backup/src/config.test.ts b/apps/backup/src/config.test.ts new file mode 100644 index 0000000..b75dc7d --- /dev/null +++ b/apps/backup/src/config.test.ts @@ -0,0 +1,56 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { loadBackupEnv } from './config.js'; + +const DATABASE_URL = 'postgresql://dorfteich:pw@db:5432/dorfteich'; + +let dir: string; + +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'dorfteich-backup-env-')); +}); + +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +describe('loadBackupEnv', () => { + it('applies defaults and drops compose empty strings', () => { + // Compose passes optional variables as "" — they must fall through to + // the schema defaults, not fail enum/number parsing (issue #80 lesson). + const env = loadBackupEnv({ + DATABASE_URL, + BACKUP_TIME: '', + BACKUP_RETENTION_DAYS: '', + BACKUP_MAIL_LOCALE: '', + SECRETS_FILE: join(dir, 'missing.env'), + }); + expect(env.BACKUP_TIME).toBe('03:00'); + expect(env.BACKUP_RETENTION_DAYS).toBe(30); + expect(env.BACKUP_MAIL_LOCALE).toBe('en'); + }); + + it('fills SMTP settings from the wizard-written secret store', () => { + const secretsFile = join(dir, 'secrets.env'); + return writeFile(secretsFile, 'SMTP_HOST="relay.example.com"\nSMTP_PORT="2525"\n').then(() => { + const env = loadBackupEnv({ + DATABASE_URL, + SECRETS_FILE: secretsFile, + // Explicit container env must win over the store. + SMTP_PORT: '465', + }); + expect(env.SMTP_HOST).toBe('relay.example.com'); + expect(env.SMTP_PORT).toBe(465); + }); + }); + + it('rejects a malformed BACKUP_TIME', () => { + expect(() => loadBackupEnv({ DATABASE_URL, BACKUP_TIME: '25:99' })).toThrow( + /BACKUP_TIME.*HH:MM/s, + ); + }); +}); diff --git a/apps/backup/src/config.ts b/apps/backup/src/config.ts new file mode 100644 index 0000000..18e4abb --- /dev/null +++ b/apps/backup/src/config.ts @@ -0,0 +1,24 @@ +import { existsSync, readFileSync } from 'node:fs'; + +import { + backupEnvSchema, + overlayEnv, + parseEnv, + parseSecretsFile, + type BackupEnv, +} from '@dorfteich/shared'; + +/** + * Loads the sidecar configuration the same way the api does (issue #80): + * the wizard-written secret store fills SMTP variables the container env + * does not set, and empty strings count as unset — compose passes optional + * variables as `""`. Without this, a wizard-configured relay would never + * reach the failure mail. + */ +export function loadBackupEnv(env: Record = process.env): BackupEnv { + const secretsFile = backupEnvSchema.shape.SECRETS_FILE.parse(env.SECRETS_FILE || undefined); + const secrets = existsSync(secretsFile) + ? parseSecretsFile(readFileSync(secretsFile, 'utf8')) + : {}; + return parseEnv(backupEnvSchema, overlayEnv(env, secrets)); +} diff --git a/apps/backup/src/index.ts b/apps/backup/src/index.ts new file mode 100644 index 0000000..7eae44d --- /dev/null +++ b/apps/backup/src/index.ts @@ -0,0 +1,49 @@ +import { pino } from 'pino'; + +import { createArchive } from './archive.js'; +import { loadBackupEnv } from './config.js'; +import { sendFailureMail } from './mail.js'; +import { pgDump } from './pg.js'; +import { runBackup } from './runner.js'; +import { scheduleDaily } from './scheduler.js'; +import type { RunnerDeps } from './runner.js'; + +/** + * Sidecar entrypoint: runs the nightly backup at BACKUP_TIME (ADR 0015). + * `BACKUP_RUN_ONCE=1` runs a single backup and exits with the outcome as + * the exit code — the on-demand path (`docker compose run backup`) and what + * the admin panel's manual trigger (#86) will call. + */ +const env = loadBackupEnv(); +const log = pino({ level: env.LOG_LEVEL, base: { service: 'backup' } }); + +const deps: RunnerDeps = { + backupsDir: env.BACKUPS_DIR, + retentionDays: env.BACKUP_RETENTION_DAYS, + now: () => new Date(), + dump: (outFile) => pgDump(env.DATABASE_URL, outFile), + archive: (outFile) => createArchive(outFile, [env.UPLOADS_DIR, env.PLUGINS_DIR]), + onFailure: async (run) => { + const sent = await sendFailureMail(env, run); + if (!sent) log.warn({ backupId: run.backupId }, 'no BACKUP_MAIL_TO configured, alert not sent'); + }, + log, +}; + +if (process.env.BACKUP_RUN_ONCE === '1') { + const status = await runBackup(deps); + process.exit(status.lastRun.outcome === 'succeeded' ? 0 : 1); +} + +log.info( + { time: env.BACKUP_TIME, retentionDays: env.BACKUP_RETENTION_DAYS, dir: env.BACKUPS_DIR }, + 'backup sidecar started', +); +const schedule = scheduleDaily(env.BACKUP_TIME, async () => void (await runBackup(deps)), log); + +for (const signal of ['SIGTERM', 'SIGINT'] as const) { + process.on(signal, () => { + schedule.stop(); + process.exit(0); + }); +} diff --git a/apps/backup/src/mail.test.ts b/apps/backup/src/mail.test.ts new file mode 100644 index 0000000..86b5020 --- /dev/null +++ b/apps/backup/src/mail.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest'; + +import { renderFailureMail } from './mail.js'; + +const input = { + backupId: '20260711-030000', + error: 'pg_dump failed: connection refused', + lastSuccessAt: '2026-07-10T03:00:12.000Z', +}; + +describe('renderFailureMail', () => { + it('renders the English alert with all details interpolated', () => { + const mail = renderFailureMail(input, 'en', 'dorfteich-test'); + expect(mail.subject).toBe('[dorfteich-test] Backup failed (20260711-030000)'); + expect(mail.text).toContain('Backup id: 20260711-030000'); + expect(mail.text).toContain('Error: pg_dump failed: connection refused'); + expect(mail.text).toContain('Last successful backup: 2026-07-10T03:00:12.000Z'); + }); + + it('renders the German alert', () => { + const mail = renderFailureMail(input, 'de', 'dorfteich-test'); + expect(mail.subject).toBe('[dorfteich-test] Backup fehlgeschlagen (20260711-030000)'); + expect(mail.text).toContain('Fehler: pg_dump failed: connection refused'); + }); + + it('states when no backup ever succeeded', () => { + const mail = renderFailureMail({ ...input, lastSuccessAt: null }, 'en', 'Dorfteich'); + expect(mail.text).toContain('Last successful backup: none yet'); + }); +}); diff --git a/apps/backup/src/mail.ts b/apps/backup/src/mail.ts new file mode 100644 index 0000000..1fff616 --- /dev/null +++ b/apps/backup/src/mail.ts @@ -0,0 +1,93 @@ +import { createTransport } from 'nodemailer'; + +import type { BackupEnv } from '@dorfteich/shared'; + +import deMails from '@dorfteich/shared/i18n/de/mails.json' with { type: 'json' }; +import enMails from '@dorfteich/shared/i18n/en/mails.json' with { type: 'json' }; + +/** + * The failure alert goes out through nodemailer directly instead of the + * api's mail outbox — when backups fail, the api may be the broken part + * (issue #83). Texts live in the shared `mails` i18n namespace (ADR 0012, + * de + en); the operator picks the language via BACKUP_MAIL_LOCALE. The + * catalog uses i18next's `{{var}}` syntax, interpolated here without + * pulling i18next into the sidecar. + */ + +export interface FailureMailInput { + backupId: string; + error: string; + lastSuccessAt: string | null; +} + +export interface RenderedFailureMail { + subject: string; + text: string; +} + +const CATALOGS = { de: deMails, en: enMails } as const; + +function t( + locale: 'de' | 'en', + key: keyof (typeof CATALOGS)['en']['backupFailed'], + params: Record = {}, +): string { + let text: string = CATALOGS[locale].backupFailed[key]; + for (const [name, value] of Object.entries(params)) { + text = text.replaceAll(`{{${name}}}`, value); + } + return text; +} + +export function renderFailureMail( + input: FailureMailInput, + locale: 'de' | 'en', + instanceLabel: string, +): RenderedFailureMail { + const lastSuccess = input.lastSuccessAt + ? t(locale, 'lastSuccess', { finishedAt: input.lastSuccessAt }) + : t(locale, 'lastSuccessNever'); + return { + subject: t(locale, 'subject', { instance: instanceLabel, backupId: input.backupId }), + text: [ + t(locale, 'intro', { instance: instanceLabel }), + '', + t(locale, 'backupId', { backupId: input.backupId }), + t(locale, 'error', { error: input.error }), + lastSuccess, + '', + t(locale, 'hint'), + ].join('\n'), + }; +} + +/** Sends the alert; returns false (after logging upstream) when no relay or + * recipient is configured — a missing mail must never fail the run. */ +export async function sendFailureMail(env: BackupEnv, input: FailureMailInput): Promise { + if (!env.BACKUP_MAIL_TO) return false; + const mail = renderFailureMail( + input, + env.BACKUP_MAIL_LOCALE, + env.BACKUP_INSTANCE_LABEL || 'Dorfteich', + ); + const transport = createTransport({ + host: env.SMTP_HOST, + port: env.SMTP_PORT, + secure: env.SMTP_SECURE, + auth: env.SMTP_USER ? { user: env.SMTP_USER, pass: env.SMTP_PASS } : undefined, + connectionTimeout: 10_000, + greetingTimeout: 10_000, + socketTimeout: 20_000, + }); + try { + await transport.sendMail({ + from: env.SMTP_FROM, + to: env.BACKUP_MAIL_TO, + subject: mail.subject, + text: mail.text, + }); + return true; + } finally { + transport.close(); + } +} diff --git a/apps/backup/src/pg.ts b/apps/backup/src/pg.ts new file mode 100644 index 0000000..1ec49c7 --- /dev/null +++ b/apps/backup/src/pg.ts @@ -0,0 +1,54 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +/** + * Connection settings for the libpq CLI tools as environment variables. + * Deliberately not `--dbname=`: the URL carries the password, and argv + * is world-readable inside the container (`/proc//cmdline`) — the + * credential-handling rule is env/file only, never argv. + */ +export function pgEnvFromUrl(databaseUrl: string): Record { + const url = new URL(databaseUrl); + const env: Record = { + PGHOST: url.hostname, + PGDATABASE: decodeURIComponent(url.pathname.replace(/^\//, '')), + }; + if (url.port) env.PGPORT = url.port; + if (url.username) env.PGUSER = decodeURIComponent(url.username); + if (url.password) env.PGPASSWORD = decodeURIComponent(url.password); + return env; +} + +async function runPgTool(tool: string, args: string[], databaseUrl: string): Promise { + try { + await execFileAsync(tool, args, { + env: { ...process.env, ...pgEnvFromUrl(databaseUrl) }, + maxBuffer: 16 * 1024 * 1024, + }); + } catch (error) { + const stderr = (error as { stderr?: string }).stderr?.trim(); + throw new Error(`${tool} failed: ${stderr || (error as Error).message}`); + } +} + +/** `pg_dump -Fc` of the whole database into `outFile` (ADR 0015). */ +export async function pgDump(databaseUrl: string, outFile: string): Promise { + await runPgTool('pg_dump', ['--format=custom', '--file', outFile], databaseUrl); +} + +/** + * Restores a custom-format dump into the live database. `--clean + * --if-exists` drops recreated objects first, so the restore lands on a + * database that may still hold newer state (the runbook stops the app + * services, not the db container). + */ +export async function pgRestore(databaseUrl: string, dumpFile: string): Promise { + const database = pgEnvFromUrl(databaseUrl).PGDATABASE ?? ''; + await runPgTool( + 'pg_restore', + ['--clean', '--if-exists', '--no-owner', '--dbname', database, dumpFile], + databaseUrl, + ); +} diff --git a/apps/backup/src/restore.ts b/apps/backup/src/restore.ts new file mode 100644 index 0000000..6f99582 --- /dev/null +++ b/apps/backup/src/restore.ts @@ -0,0 +1,40 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; + +import { pino } from 'pino'; + +import { extractArchive } from './archive.js'; +import { archiveFileName, backupIdTime, dumpFileName } from './backup-set.js'; +import { loadBackupEnv } from './config.js'; +import { pgRestore } from './pg.js'; + +/** + * In-container half of the restore runbook (operations.md §Backup & restore): + * `pg_restore --clean --if-exists` of the set's dump, then the volume archive + * back over the uploads/plugins mounts. The host-side `restore.sh` wraps this + * with stopping/starting the app services — run through that, not directly, + * unless you know the api and collab are down. + */ +const env = loadBackupEnv(); +const log = pino({ level: env.LOG_LEVEL, base: { service: 'backup-restore' } }); + +const backupId = process.argv[2]; +if (!backupId || !backupIdTime(backupId)) { + log.error({ backupId }, 'usage: node dist/restore.js (e.g. 20260711-030000)'); + process.exit(2); +} + +const dumpFile = join(env.BACKUPS_DIR, dumpFileName(backupId)); +const archiveFile = join(env.BACKUPS_DIR, archiveFileName(backupId)); +for (const file of [dumpFile, archiveFile]) { + if (!existsSync(file)) { + log.error({ file }, 'restore set is incomplete — file not found'); + process.exit(2); + } +} + +log.info({ backupId }, 'restoring database dump'); +await pgRestore(env.DATABASE_URL, dumpFile); +log.info({ backupId }, 'restoring uploads/plugins archive'); +await extractArchive(archiveFile, [env.UPLOADS_DIR, env.PLUGINS_DIR]); +log.info({ backupId }, 'restore complete — start the stack and verify /readyz'); diff --git a/apps/backup/src/runner.test.ts b/apps/backup/src/runner.test.ts new file mode 100644 index 0000000..e1220cc --- /dev/null +++ b/apps/backup/src/runner.test.ts @@ -0,0 +1,139 @@ +import { mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { archiveFileName, dumpFileName } from './backup-set.js'; +import { runBackup, type RunnerDeps } from './runner.js'; +import { readStatus } from './status.js'; + +/** + * Time-accelerated nightly runs (issue #83 acceptance criteria): the clock + * is injected, so "a month of nights" is a loop, with real files in a temp + * backups directory and dump/archive doubles standing in for pg_dump/tar + * (their real counterparts are covered by pg.ts/archive.ts on the stage). + */ + +const DAY = 24 * 60 * 60 * 1000; +const START = new Date('2026-07-01T03:00:00Z').getTime(); + +let dir: string; +let clock: Date; +let failures: Array<{ backupId: string; error: string; lastSuccessAt: string | null }>; + +function makeDeps(overrides: Partial = {}): RunnerDeps { + return { + backupsDir: dir, + retentionDays: 7, + now: () => new Date(clock), + dump: (outFile) => writeFile(outFile, 'dump-bytes'), + archive: (outFile) => writeFile(outFile, 'archive-bytes!'), + onFailure: async (run) => { + failures.push(run); + }, + log: { info: () => {}, error: () => {} }, + ...overrides, + }; +} + +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'dorfteich-backup-')); + clock = new Date(START); + failures = []; +}); + +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +describe('runBackup', () => { + it('produces a dump, a matching archive, and status.json', async () => { + const status = await runBackup(makeDeps()); + + const names = await readdir(dir); + expect(names).toContain(dumpFileName('20260701-030000')); + expect(names).toContain(archiveFileName('20260701-030000')); + expect(status.lastRun.outcome).toBe('succeeded'); + expect(status.lastRun.sizes).toEqual({ dumpBytes: 10, archiveBytes: 14 }); + expect(status.lastSuccess?.backupId).toBe('20260701-030000'); + expect(readStatus(dir)).toEqual(status); + }); + + it('keeps exactly the retention window across simulated nights', async () => { + for (let night = 0; night < 30; night += 1) { + clock = new Date(START + night * DAY); + await runBackup(makeDeps()); + } + + const names = (await readdir(dir)).filter((name) => name.startsWith('db-')); + // Retention 7 days: the last 7 nights survive plus the current night's set. + expect(names.length).toBe(8); + expect(names).toContain(dumpFileName('20260730-030000')); + expect(names).not.toContain(dumpFileName('20260722-030000')); + expect(names).toContain(dumpFileName('20260723-030000')); + }); + + it('records a failed run, keeps lastSuccess, alerts, and cleans partials', async () => { + await runBackup(makeDeps()); + + clock = new Date(START + DAY); + const status = await runBackup( + makeDeps({ + dump: async () => { + throw new Error('connection refused'); + }, + }), + ); + + expect(status.lastRun.outcome).toBe('failed'); + expect(status.lastRun.error).toContain('connection refused'); + expect(status.lastSuccess?.backupId).toBe('20260701-030000'); + expect(failures).toHaveLength(1); + expect(failures[0]?.lastSuccessAt).toBe(new Date(START).toISOString()); + + const names = await readdir(dir); + expect(names.filter((name) => name.endsWith('.partial'))).toHaveLength(0); + // The last good set survives a failed night. + expect(names).toContain(dumpFileName('20260701-030000')); + }); + + it('a failure alert that itself fails never breaks the run', async () => { + const status = await runBackup( + makeDeps({ + dump: async () => { + throw new Error('disk full'); + }, + onFailure: async () => { + throw new Error('smtp down'); + }, + }), + ); + expect(status.lastRun.outcome).toBe('failed'); + expect(readStatus(dir)?.lastRun.error).toContain('disk full'); + }); + + it('after long downtime the newest complete set survives pruning', async () => { + await runBackup(makeDeps()); + + // 60 days later (retention 7): the old set is expired but must survive + // as the newest complete one until a new success replaces it. + clock = new Date(START + 60 * DAY); + const status = await runBackup( + makeDeps({ + dump: async () => { + throw new Error('db gone'); + }, + }), + ); + expect(status.lastRun.outcome).toBe('failed'); + expect(await readdir(dir)).toContain(dumpFileName('20260701-030000')); + + // The next success prunes it. + clock = new Date(START + 61 * DAY); + await runBackup(makeDeps()); + const names = await readdir(dir); + expect(names).not.toContain(dumpFileName('20260701-030000')); + expect(names).toContain(dumpFileName('20260831-030000')); + }); +}); diff --git a/apps/backup/src/runner.ts b/apps/backup/src/runner.ts new file mode 100644 index 0000000..fd0e5ff --- /dev/null +++ b/apps/backup/src/runner.ts @@ -0,0 +1,132 @@ +import { mkdir, readdir, rename, rm, stat } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { archiveFileName, dumpFileName, expiredSets, listSets, newBackupId } from './backup-set.js'; +import { + readStatus, + writeStatus, + type BackupRun, + type BackupSizes, + type BackupStatus, +} from './status.js'; + +/** + * One nightly run (ADR 0015): dump first, then the volume archive (files may + * be minutes newer than the dump — documented consistency model), then prune, + * then `status.json`. Artifacts are written under a `.partial` suffix and + * renamed on completion, so a crash never leaves a file that looks like a + * restorable artifact, and prune (which ignores `.partial`) never counts a + * torn set as the "newest complete" one. + */ + +export interface RunnerDeps { + backupsDir: string; + retentionDays: number; + now(): Date; + /** Real implementation: pg_dump -Fc (pg.ts). */ + dump(outFile: string): Promise; + /** Real implementation: tar of the uploads/plugins mounts (archive.ts). */ + archive(outFile: string): Promise; + /** Failure alert (mail.ts); errors here are logged, never rethrown. */ + onFailure(run: { backupId: string; error: string; lastSuccessAt: string | null }): Promise; + log: { + info(details: object, message: string): void; + error(details: object, message: string): void; + }; +} + +export async function runBackup(deps: RunnerDeps): Promise { + const startedAt = deps.now(); + const backupId = newBackupId(startedAt); + const previous = readStatus(deps.backupsDir); + const lastSuccess = previous?.lastSuccess ?? null; + await mkdir(deps.backupsDir, { recursive: true }); + + const dumpFile = join(deps.backupsDir, dumpFileName(backupId)); + const archiveFile = join(deps.backupsDir, archiveFileName(backupId)); + + let run: BackupRun; + let success: BackupStatus['lastSuccess'] = lastSuccess; + try { + const sizes = await produceArtifacts(deps, dumpFile, archiveFile); + const finishedAt = deps.now(); + run = { + backupId, + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + durationMs: finishedAt.getTime() - startedAt.getTime(), + outcome: 'succeeded', + sizes, + }; + success = { backupId, finishedAt: run.finishedAt, sizes }; + deps.log.info({ backupId, sizes }, 'backup run succeeded'); + } catch (error) { + await removePartials(deps.backupsDir); + const message = error instanceof Error ? error.message : String(error); + const finishedAt = deps.now(); + run = { + backupId, + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + durationMs: finishedAt.getTime() - startedAt.getTime(), + outcome: 'failed', + error: message, + }; + deps.log.error({ backupId, error: message }, 'backup run failed'); + try { + await deps.onFailure({ + backupId, + error: message, + lastSuccessAt: lastSuccess?.finishedAt ?? null, + }); + } catch (mailError) { + deps.log.error({ backupId, error: String(mailError) }, 'failure alert could not be sent'); + } + } + + await prune(deps); + + const status: BackupStatus = { + schemaVersion: 1, + updatedAt: deps.now().toISOString(), + retentionDays: deps.retentionDays, + lastRun: run, + lastSuccess: success, + }; + await writeStatus(deps.backupsDir, status); + return status; +} + +/** Dump, then archive — each staged as `.partial` and renamed only when done. */ +async function produceArtifacts( + deps: RunnerDeps, + dumpFile: string, + archiveFile: string, +): Promise { + await deps.dump(`${dumpFile}.partial`); + await rename(`${dumpFile}.partial`, dumpFile); + await deps.archive(`${archiveFile}.partial`); + await rename(`${archiveFile}.partial`, archiveFile); + return { + dumpBytes: (await stat(dumpFile)).size, + archiveBytes: (await stat(archiveFile)).size, + }; +} + +/** Leftover staging files: from this failed run or an earlier crash — no + * resumption path exists, so they are dead weight either way. */ +async function removePartials(backupsDir: string): Promise { + for (const name of await readdir(backupsDir)) { + if (name.endsWith('.partial')) await rm(join(backupsDir, name), { force: true }); + } +} + +async function prune(deps: RunnerDeps): Promise { + const names = await readdir(deps.backupsDir); + for (const set of expiredSets(listSets(names), deps.now(), deps.retentionDays)) { + for (const file of set.files) { + await rm(join(deps.backupsDir, file), { force: true }); + } + deps.log.info({ backupId: set.id }, 'pruned expired backup set'); + } +} diff --git a/apps/backup/src/scheduler.test.ts b/apps/backup/src/scheduler.test.ts new file mode 100644 index 0000000..ba36a38 --- /dev/null +++ b/apps/backup/src/scheduler.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest'; + +import { msUntilNext } from './scheduler.js'; + +// msUntilNext works in local time (the container's TZ); the tests build +// their expectations with local-time Dates, so they hold in any zone. +describe('msUntilNext', () => { + it('targets today when the time is still ahead', () => { + const now = new Date(2026, 6, 11, 1, 30, 0); + expect(msUntilNext(now, '03:00')).toBe(90 * 60 * 1000); + }); + + it('targets tomorrow when the time has passed', () => { + const now = new Date(2026, 6, 11, 3, 0, 1); + expect(msUntilNext(now, '03:00')).toBe(24 * 60 * 60 * 1000 - 1000); + }); + + it('an exact hit schedules a full day ahead (never a zero delay)', () => { + const now = new Date(2026, 6, 11, 3, 0, 0); + expect(msUntilNext(now, '03:00')).toBe(24 * 60 * 60 * 1000); + }); +}); diff --git a/apps/backup/src/scheduler.ts b/apps/backup/src/scheduler.ts new file mode 100644 index 0000000..5ddf231 --- /dev/null +++ b/apps/backup/src/scheduler.ts @@ -0,0 +1,32 @@ +/** + * Daily scheduling without a cron dependency: one `setTimeout` to the next + * HH:MM occurrence (container-local time via TZ), re-armed after every run. + * A run that crosses midnight simply shifts the next one — backups are + * hours apart, drift by seconds is irrelevant. + */ + +/** Milliseconds from `now` to the next local-time occurrence of `time` (HH:MM). */ +export function msUntilNext(now: Date, time: string): number { + const [hours = 0, minutes = 0] = time.split(':').map(Number); + const next = new Date(now); + next.setHours(hours, minutes, 0, 0); + if (next.getTime() <= now.getTime()) next.setDate(next.getDate() + 1); + return next.getTime() - now.getTime(); +} + +export function scheduleDaily( + time: string, + task: () => Promise, + log: { info(details: object, message: string): void }, +): { stop(): void } { + let timer: NodeJS.Timeout; + const arm = (): void => { + const delay = msUntilNext(new Date(), time); + log.info({ nextRunInMs: delay }, 'next backup run scheduled'); + timer = setTimeout(() => { + void task().finally(arm); + }, delay); + }; + arm(); + return { stop: () => clearTimeout(timer) }; +} diff --git a/apps/backup/src/status.ts b/apps/backup/src/status.ts new file mode 100644 index 0000000..07438bc --- /dev/null +++ b/apps/backup/src/status.ts @@ -0,0 +1,57 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { rename, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +/** + * `status.json` on the backups volume is the machine-readable outcome of the + * last run (ADR 0015): the api's backup-freshness readiness check (#85) and + * the admin panel's backup card (#86) consume it. Keep the shape additive — + * bump `schemaVersion` on breaking changes. + */ + +export const STATUS_FILE = 'status.json'; + +export interface BackupSizes { + dumpBytes: number; + archiveBytes: number; +} + +export interface BackupRun { + backupId: string; + startedAt: string; + finishedAt: string; + durationMs: number; + outcome: 'succeeded' | 'failed'; + /** Present on failure: the first error the run hit, as a plain string. */ + error?: string; + /** Present on success. */ + sizes?: BackupSizes; +} + +export interface BackupStatus { + schemaVersion: 1; + updatedAt: string; + retentionDays: number; + lastRun: BackupRun; + /** Carried across failed runs so freshness checks see the real gap. */ + lastSuccess: { backupId: string; finishedAt: string; sizes: BackupSizes } | null; +} + +/** Reads the previous status; a missing or torn file is simply "no status". */ +export function readStatus(backupsDir: string): BackupStatus | null { + const path = join(backupsDir, STATUS_FILE); + if (!existsSync(path)) return null; + try { + return JSON.parse(readFileSync(path, 'utf8')) as BackupStatus; + } catch { + return null; + } +} + +/** Atomic write (staging + rename) so readers never see a torn file. */ +export async function writeStatus(backupsDir: string, status: BackupStatus): Promise { + const path = join(backupsDir, STATUS_FILE); + const staging = `${path}.tmp-${process.pid}`; + await writeFile(staging, JSON.stringify(status, null, 2) + '\n'); + await rename(staging, path); +} diff --git a/apps/backup/tsconfig.json b/apps/backup/tsconfig.json new file mode 100644 index 0000000..1e20cd9 --- /dev/null +++ b/apps/backup/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src"] +} diff --git a/apps/backup/vitest.config.ts b/apps/backup/vitest.config.ts new file mode 100644 index 0000000..4ac6027 --- /dev/null +++ b/apps/backup/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + environment: 'node', + }, +}); diff --git a/deploy/backup/restore.sh b/deploy/backup/restore.sh new file mode 100755 index 0000000..3baa1f9 --- /dev/null +++ b/deploy/backup/restore.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env sh +# Restore a nightly backup set (ADR 0015, operations.md §Backup & restore). +# +# Run from the stage directory (where docker-compose.yml and .env live): +# deploy/backup/restore.sh # e.g. 20260711-030000 +# +# Steps (the documented runbook, automated): stop the app services (the db +# stays up — pg_restore needs it), replay the dump and the uploads/plugins +# archive through the backup sidecar image, start the stack, check /readyz. +set -eu + +if [ $# -ne 1 ]; then + echo "usage: $0 (list sets: docker compose exec backup ls /backups)" >&2 + exit 2 +fi +backup_id="$1" + +echo "Stopping app services (db keeps running) ..." +docker compose stop web api collab + +echo "Restoring set ${backup_id} ..." +docker compose run --rm --no-deps backup node dist/restore.js "${backup_id}" + +echo "Starting the stack ..." +docker compose up -d + +api_port="$(docker compose port api 3000 2>/dev/null | head -n1)" +if [ -n "${api_port}" ]; then + echo "Waiting for readiness on ${api_port} ..." + for _ in $(seq 1 30); do + if curl -sf "http://${api_port}/api/v1/readyz" >/dev/null 2>&1; then + echo "Restore complete — /readyz is green. Spot-check a page and a file." + exit 0 + fi + sleep 2 + done + echo "Restore applied, but /readyz did not turn green within 60 s — check 'docker compose logs api'." >&2 + exit 1 +fi +echo "Restore applied. Verify /readyz through your reverse proxy." diff --git a/deploy/compose/.env.example b/deploy/compose/.env.example index 7b4a9ed..7a6177e 100644 --- a/deploy/compose/.env.example +++ b/deploy/compose/.env.example @@ -48,6 +48,23 @@ SMTP_USER=wiki@example.com SMTP_PASS=change-me SMTP_FROM=Dorfteich +# --- backups (ADR 0015, issue #83) -------------------------------------------- +# The backup sidecar dumps the database and archives the uploads/plugins +# volumes nightly onto the `backups` volume; restore via +# deploy/backup/restore.sh . All values optional. +# Daily run time HH:MM in TZ (default 03:00; set TZ for stage-local time, +# e.g. TZ=Europe/Berlin — unset means UTC). +#TZ=Europe/Berlin +#BACKUP_TIME=03:00 +# Local retention in days: 30 (default) for Prod, 7 for Test/Int (ADR 0015). +#BACKUP_RETENTION_DAYS=30 +# Failure alert: recipient (unset = no mail, failures only in the logs and +# status.json), mail language (de|en), and the label used in the subject +# (defaults to the compose project name). +#BACKUP_MAIL_TO=ops@example.com +#BACKUP_MAIL_LOCALE=en +#BACKUP_INSTANCE_LABEL=dorfteich-test + # --- first-run setup (optional pre-seeding, issue #80) ------------------------ # A fresh (empty) database makes the instance require the browser setup # wizard. Automated deploys can skip it entirely by pre-seeding the Site diff --git a/deploy/compose/docker-compose.yml b/deploy/compose/docker-compose.yml index 8a34757..8773f91 100644 --- a/deploy/compose/docker-compose.yml +++ b/deploy/compose/docker-compose.yml @@ -127,6 +127,52 @@ services: condition: service_healthy <<: *logging + # Backup sidecar (ADR 0015, issue #83): nightly `pg_dump -Fc` + one tar of + # the uploads/plugins volumes as a consistent restore set on the `backups` + # volume, prune by retention, `status.json`, failure mail directly via SMTP + # (the api may be the broken part). Restore runs through + # deploy/backup/restore.sh, which drives this same image. + backup: + image: ${IMAGE_PREFIX:-dorfteich}-backup:${TAG:-latest} + build: + context: ../.. + dockerfile: apps/backup/Dockerfile + args: + APP_VERSION: ${TAG:-latest} + restart: unless-stopped + environment: + NODE_ENV: production + LOG_LEVEL: ${LOG_LEVEL:-info} + DATABASE_URL: postgresql://dorfteich:${POSTGRES_PASSWORD:?set in .env}@db:5432/dorfteich + # Daily run time (HH:MM) in TZ; retention 30 d default, 7 d on Test/Int. + TZ: ${TZ:-} + BACKUP_TIME: ${BACKUP_TIME:-} + BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-} + # Failure-alert recipient; empty disables the mail (logged instead). + BACKUP_MAIL_TO: ${BACKUP_MAIL_TO:-} + BACKUP_MAIL_LOCALE: ${BACKUP_MAIL_LOCALE:-} + BACKUP_INSTANCE_LABEL: ${BACKUP_INSTANCE_LABEL:-${COMPOSE_PROJECT_NAME:-dorfteich}} + # Same SMTP resolution as the api: explicit env wins, the wizard-written + # secret store fills the gaps (issue #80). + SMTP_HOST: ${SMTP_HOST:-} + SMTP_PORT: ${SMTP_PORT:-} + SMTP_SECURE: ${SMTP_SECURE:-} + SMTP_USER: ${SMTP_USER:-} + SMTP_PASS: ${SMTP_PASS:-} + SMTP_FROM: ${SMTP_FROM:-} + networks: [internal] + volumes: + # Write access to uploads/plugins is for the restore path only; the + # nightly run just reads them into the archive. + - uploads:/data/uploads + - plugins:/data/plugins + - secrets:/data/secrets:ro + - backups:/backups + depends_on: + db: + condition: service_healthy + <<: *logging + db: image: postgres:17.5-alpine restart: unless-stopped @@ -182,3 +228,4 @@ volumes: uploads: plugins: secrets: + backups: diff --git a/docs/architecture/deployment.md b/docs/architecture/deployment.md index 3c9fbca..a6e0b40 100644 --- a/docs/architecture/deployment.md +++ b/docs/architecture/deployment.md @@ -23,7 +23,9 @@ Every stage (and every self-hosted instance) runs the same services: | `gotenberg` | `gotenberg/gotenberg:` | internal only | | `backup` | `dorfteich-backup` | cron sidecar: pg_dump, volume archive, prune, mirror (ADR 0015) | -Volumes: `db-data`, `uploads` (uploads + installed plugins), `backups`. +Volumes: `db-data`, `uploads`, `plugins` (installed plugin bundles), +`secrets` (wizard-written secret store), `backups` (restore sets + +`status.json`). Networks: `frontend` (reverse proxy ↔ web/api/collab) and `internal` (api/collab ↔ db/pandoc/gotenberg); db and converters are never exposed. diff --git a/docs/architecture/operations.md b/docs/architecture/operations.md index 1218698..039dbba 100644 --- a/docs/architecture/operations.md +++ b/docs/architecture/operations.md @@ -36,14 +36,19 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack. ## Backup & restore (operational view of ADR 0015) -- Nightly at 03:00 stage-local time: `pg_dump -Fc` → uploads/plugins volume - archive → prune (> 30 days Prod, > 7 days Test/Int) → rsync mirror to - BASEL `/home/RAID/BACKUPS/dorfteich-prod/` (Prod only, dedicated - `dorfteich-backup` user). -- **Restore runbook** (also the Prod-relocation procedure): - 1. `docker compose down` (keep volumes), +- Nightly at 03:00 stage-local time (sidecar `backup`, issue #83; env + `BACKUP_TIME`/`TZ`): `pg_dump -Fc` → uploads/plugins volume archive (one + tar, same backup id `YYYYMMDD-HHMMSS`) → prune (`BACKUP_RETENTION_DAYS`, + 30 default / 7 Test+Int; the newest complete set always survives) → + `status.json` on the `backups` volume → on failure a mail directly via + the instance SMTP to `BACKUP_MAIL_TO`. Mirror to BASEL is issue #84. +- **On-demand run**: `docker compose run --rm -e BACKUP_RUN_ONCE=1 backup` + (exit code = outcome); list sets with `docker compose exec backup ls /backups`. +- **Restore runbook** (also the Prod-relocation procedure) — automated by + `deploy/backup/restore.sh `, run from the stage directory: + 1. stop the app services (`web`, `api`, `collab`; the db stays up), 2. restore DB: `pg_restore --clean --if-exists` into the `db` container, - 3. restore volume: unpack the matching uploads archive, + 3. restore volume: unpack the matching uploads/plugins archive, 4. `docker compose up -d`, verify `/readyz`, spot-check a page + a file. - **Drills**: monthly automated restore of the latest Prod dump into a scratch database on Test with a row-count sanity report; quarterly manual diff --git a/packages/shared/i18n/de/mails.json b/packages/shared/i18n/de/mails.json index 7a7ffbd..367b41a 100644 --- a/packages/shared/i18n/de/mails.json +++ b/packages/shared/i18n/de/mails.json @@ -21,5 +21,14 @@ "body": "diese Testnachricht bestätigt, dass dein Dorfteich E-Mails über den konfigurierten SMTP-Server versenden kann. Deine Instanz erreichst du hier:", "action": "Dorfteich öffnen", "expiry": "Du kannst diese E-Mail einfach löschen." + }, + "backupFailed": { + "subject": "[{{instance}}] Backup fehlgeschlagen ({{backupId}})", + "intro": "Der nächtliche Backup-Lauf auf {{instance}} ist fehlgeschlagen.", + "backupId": "Backup-ID: {{backupId}}", + "error": "Fehler: {{error}}", + "lastSuccess": "Letztes erfolgreiches Backup: {{finishedAt}}", + "lastSuccessNever": "Letztes erfolgreiches Backup: noch keines", + "hint": "Prüfe die Sidecar-Logs (docker compose logs backup) und die status.json auf dem Backups-Volume." } } diff --git a/packages/shared/i18n/en/mails.json b/packages/shared/i18n/en/mails.json index fa6fdde..cb0c3ad 100644 --- a/packages/shared/i18n/en/mails.json +++ b/packages/shared/i18n/en/mails.json @@ -21,5 +21,14 @@ "body": "this test message confirms that your Dorfteich can send e-mail through the configured SMTP server. You can reach your instance here:", "action": "Open Dorfteich", "expiry": "You can simply delete this e-mail." + }, + "backupFailed": { + "subject": "[{{instance}}] Backup failed ({{backupId}})", + "intro": "The nightly backup run on {{instance}} failed.", + "backupId": "Backup id: {{backupId}}", + "error": "Error: {{error}}", + "lastSuccess": "Last successful backup: {{finishedAt}}", + "lastSuccessNever": "Last successful backup: none yet", + "hint": "Check the sidecar logs (docker compose logs backup) and status.json on the backups volume." } } diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts index 84de051..ceb7c80 100644 --- a/packages/shared/src/env.ts +++ b/packages/shared/src/env.ts @@ -27,6 +27,25 @@ const databaseUrl = z */ const collabTokenSecret = z.string().min(16).default('dev-insecure-collab-token-secret-change-me'); +/** + * SMTP delivery fields, shared by the api (transactional mail) and the + * backup sidecar (failure alert mail, issue #83). Defaults match the + * Mailpit container from the dev overlay; production instances configure + * their real relay in the stage `.env` or through the M8 setup wizard, + * whose secret store both services overlay the same way. + */ +const smtpFields = { + SMTP_HOST: z.string().default('localhost'), + SMTP_PORT: z.coerce.number().int().default(1025), + SMTP_SECURE: z + .enum(['true', 'false']) + .default('false') + .transform((value) => value === 'true'), + SMTP_USER: z.string().optional(), + SMTP_PASS: z.string().optional(), + SMTP_FROM: z.string().default('Dorfteich '), +}; + export const apiEnvSchema = z.object({ NODE_ENV: nodeEnv, PORT: z.coerce.number().int().min(1).max(65535).default(3000), @@ -41,20 +60,7 @@ export const apiEnvSchema = z.object({ .transform((value) => value === 'true'), /** Public base URL of this instance — used in e-mail links. */ APP_BASE_URL: z.string().url().default('http://localhost:5173'), - /** - * SMTP delivery. Defaults match the Mailpit container from the dev - * overlay; production instances configure their real relay here (the - * M8 setup wizard writes these). - */ - SMTP_HOST: z.string().default('localhost'), - SMTP_PORT: z.coerce.number().int().default(1025), - SMTP_SECURE: z - .enum(['true', 'false']) - .default('false') - .transform((value) => value === 'true'), - SMTP_USER: z.string().optional(), - SMTP_PASS: z.string().optional(), - SMTP_FROM: z.string().default('Dorfteich '), + ...smtpFields, /** * Filesystem root for uploaded files (ADR 0011). The compose stack * mounts the `uploads` volume at `/data/uploads` and sets this @@ -132,6 +138,41 @@ export const collabEnvSchema = z.object({ export type CollabEnv = z.infer; +/** + * Configuration for the backup sidecar (ADR 0015, issue #83). It talks to + * the same database and mounts the same data volumes as the api, plus its + * own `backups` volume for the nightly restore sets and `status.json`. + */ +export const backupEnvSchema = z.object({ + NODE_ENV: nodeEnv, + LOG_LEVEL: logLevel, + APP_VERSION: appVersion, + DATABASE_URL: databaseUrl, + /** Where restore sets and `status.json` are written (the `backups` volume). */ + BACKUPS_DIR: z.string().min(1).default('./data/backups'), + /** Same mounts as the api — archived together as one restore set. */ + UPLOADS_DIR: z.string().min(1).default('./data/uploads'), + PLUGINS_DIR: z.string().min(1).default('./data/plugins'), + /** Daily run time as HH:MM, interpreted in the container's TZ. */ + BACKUP_TIME: z + .string() + .regex(/^([01]\d|2[0-3]):[0-5]\d$/, 'must be HH:MM (24h)') + .default('03:00'), + /** Local retention in days: 30 for Prod, 7 for Test/Int (ADR 0015). */ + BACKUP_RETENTION_DAYS: z.coerce.number().int().min(1).default(30), + /** Failure-alert recipient; unset disables the mail (logged instead). */ + BACKUP_MAIL_TO: z.string().optional(), + /** Language of the failure mail (ADR 0012 — both exist, operator picks). */ + BACKUP_MAIL_LOCALE: z.enum(['de', 'en']).default('en'), + /** Instance label in the mail subject, e.g. "dorfteich-test". */ + BACKUP_INSTANCE_LABEL: z.string().optional(), + ...smtpFields, + /** Read-only view of the wizard-written secret store (issue #80). */ + SECRETS_FILE: z.string().min(1).default('./data/secrets.env'), +}); + +export type BackupEnv = z.infer; + export function parseEnv( schema: Schema, env: Record, diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index fe5427c..f8895a6 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -17,6 +17,7 @@ export * from './pages'; export * from './permissions'; export * from './plugins'; export * from './search'; +export * from './secret-store'; export * from './setup'; export * from './ponds'; export * from './quotas'; diff --git a/packages/shared/src/secret-store.ts b/packages/shared/src/secret-store.ts new file mode 100644 index 0000000..db67e31 --- /dev/null +++ b/packages/shared/src/secret-store.ts @@ -0,0 +1,62 @@ +/** + * The env-backed secret store (security.md §Secrets, issue #80): secrets the + * setup wizard collects in the browser (SMTP credentials) are persisted as a + * mode-600 dotenv-style file on a volume — never as database rows. The file + * extends the environment: `overlayEnv` fills only variables the process + * environment does not set. These pure format/merge helpers live in shared + * because two services read the store the api writes: the api itself and the + * backup sidecar (issue #83), which needs the wizard's SMTP relay for its + * failure mail. File I/O stays with each service. + */ + +/** Parses the dotenv-style store content. Ignores blank lines and comments. */ +export function parseSecretsFile(content: string): Record { + const secrets: Record = {}; + for (const line of content.split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const eq = trimmed.indexOf('='); + if (eq <= 0) continue; + const key = trimmed.slice(0, eq).trim(); + let value = trimmed.slice(eq + 1).trim(); + if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) { + value = value.slice(1, -1).replace(/\\n/g, '\n').replace(/\\"/g, '"').replace(/\\\\/g, '\\'); + } + secrets[key] = value; + } + return secrets; +} + +/** Serializes secrets with double-quoted, escaped values (dotenv-compatible). */ +export function serializeSecrets(secrets: Record): string { + const lines = [ + '# Managed by Dorfteich (setup wizard). Values here fill environment', + '# variables that the container environment does not set explicitly.', + ]; + for (const [key, value] of Object.entries(secrets)) { + const escaped = value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n'); + lines.push(`${key}="${escaped}"`); + } + return lines.join('\n') + '\n'; +} + +/** + * Merges the store under the real environment: explicit env vars win, store + * values fill the gaps (and Zod defaults fill whatever remains at parse + * time). Empty strings count as unset on both sides — compose passes + * `${SMTP_HOST:-}` as `""` for variables the stage `.env` does not define, + * and those must not shadow wizard-written store values or schema defaults. + */ +export function overlayEnv( + env: Record, + secrets: Record, +): Record { + const merged: Record = {}; + for (const [key, value] of Object.entries(secrets)) { + if (value !== '') merged[key] = value; + } + for (const [key, value] of Object.entries(env)) { + if (value !== undefined && value !== '') merged[key] = value; + } + return merged; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3a17c8b..226cdf6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -166,6 +166,34 @@ importers: specifier: ^3.0.0 version: 3.2.6(@types/node@26.1.0)(jiti@2.7.0)(jsdom@26.1.0)(terser@5.48.0)(tsx@4.23.0) + apps/backup: + dependencies: + '@dorfteich/shared': + specifier: workspace:* + version: link:../../packages/shared + nodemailer: + specifier: ^9.0.3 + version: 9.0.3 + pino: + specifier: ^9.6.0 + version: 9.14.0 + zod: + specifier: ^3.25.76 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^26.1.0 + version: 26.1.0 + '@types/nodemailer': + specifier: ^8.0.1 + version: 8.0.1 + tsx: + specifier: ^4.19.0 + version: 4.23.0 + vitest: + specifier: ^3.0.0 + version: 3.2.6(@types/node@26.1.0)(jiti@2.7.0)(jsdom@26.1.0)(terser@5.48.0)(tsx@4.23.0) + apps/collab: dependencies: '@dorfteich/shared':