Add backup sidecar: nightly dump, volume archive, prune, status, restore (#83)
All checks were successful
CI / Lint, typecheck, test (push) Successful in 3m9s
CI / Build container images (push) Has been skipped
CD / Build and push images (push) Successful in 3m47s
CD / Deploy to Test (push) Successful in 8s
CD / Smoke tests against Test (push) Successful in 1m10s
CD / Promote to Int (push) Successful in 9s
CI / Auth e2e pack (push) Successful in 5m25s
CI / Import/export fidelity gate (push) Successful in 45s
All checks were successful
CI / Lint, typecheck, test (push) Successful in 3m9s
CI / Build container images (push) Has been skipped
CD / Build and push images (push) Successful in 3m47s
CD / Deploy to Test (push) Successful in 8s
CD / Smoke tests against Test (push) Successful in 1m10s
CD / Promote to Int (push) Successful in 9s
CI / Auth e2e pack (push) Successful in 5m25s
CI / Import/export fidelity gate (push) Successful in 45s
New apps/backup service (ADR 0015): nightly pg_dump -Fc plus one tar of the uploads/plugins volumes as a consistent restore set on a new backups volume, retention prune that never removes the newest complete set, atomic status.json for the readiness/admin consumers (#85/#86), and a failure mail sent directly via nodemailer (the api may be the broken part) with de/en texts in the shared mails catalog. BACKUP_RUN_ONCE=1 gives the on-demand path; deploy/backup/restore.sh automates the documented restore runbook. The pure secret-store helpers moved to @dorfteich/shared so the sidecar resolves the wizard-written SMTP relay exactly like the api. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1
This commit is contained in:
parent
fd2bdb3fb8
commit
8dbff86537
@ -47,6 +47,13 @@ jobs:
|
||||
docker push $IMAGE_BASE-collab:${{ github.sha }}
|
||||
docker push $IMAGE_BASE-collab:test
|
||||
|
||||
- name: Build and push backup image
|
||||
run: |
|
||||
docker build -f apps/backup/Dockerfile --build-arg APP_VERSION=${{ github.sha }} \
|
||||
-t $IMAGE_BASE-backup:${{ github.sha }} -t $IMAGE_BASE-backup:test .
|
||||
docker push $IMAGE_BASE-backup:${{ github.sha }}
|
||||
docker push $IMAGE_BASE-backup:test
|
||||
|
||||
deploy-test:
|
||||
name: Deploy to Test
|
||||
needs: build-push
|
||||
@ -120,6 +127,7 @@ jobs:
|
||||
docker buildx imagetools create -t $IMAGE_BASE-web:int $IMAGE_BASE-web:${{ github.sha }}
|
||||
docker buildx imagetools create -t $IMAGE_BASE-api:int $IMAGE_BASE-api:${{ github.sha }}
|
||||
docker buildx imagetools create -t $IMAGE_BASE-collab:int $IMAGE_BASE-collab:${{ github.sha }}
|
||||
docker buildx imagetools create -t $IMAGE_BASE-backup:int $IMAGE_BASE-backup:${{ github.sha }}
|
||||
|
||||
- name: Set up SSH
|
||||
run: |
|
||||
|
||||
@ -538,3 +538,6 @@ jobs:
|
||||
|
||||
- name: Build collab image
|
||||
run: docker build -f apps/collab/Dockerfile --build-arg APP_VERSION=${{ github.sha }} -t dorfteich-collab:ci .
|
||||
|
||||
- name: Build backup image
|
||||
run: docker build -f apps/backup/Dockerfile --build-arg APP_VERSION=${{ github.sha }} -t dorfteich-backup:ci .
|
||||
|
||||
@ -2,44 +2,16 @@ import { existsSync, readFileSync } from 'node:fs';
|
||||
import { chmod, mkdir, rename, writeFile } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
|
||||
import { parseSecretsFile, serializeSecrets } from '@dorfteich/shared';
|
||||
|
||||
/**
|
||||
* The env-backed secret store (security.md §Secrets, issue #80): secrets the
|
||||
* setup wizard collects in the browser (SMTP credentials) are persisted as a
|
||||
* mode-600 dotenv-style file on a volume — never as database rows. The file
|
||||
* extends the environment: `overlayEnv` fills only variables the process
|
||||
* environment does not set, so the stage `.env` always stays authoritative.
|
||||
* File I/O for the env-backed secret store (security.md §Secrets, issue #80).
|
||||
* The pure format/merge helpers (`parseSecretsFile`, `serializeSecrets`,
|
||||
* `overlayEnv`) moved to `@dorfteich/shared` in issue #83 so the backup
|
||||
* sidecar can read the same store; they are re-exported here to keep the
|
||||
* api-internal import paths stable.
|
||||
*/
|
||||
|
||||
/** Parses the dotenv-style store content. Ignores blank lines and comments. */
|
||||
export function parseSecretsFile(content: string): Record<string, string> {
|
||||
const secrets: Record<string, string> = {};
|
||||
for (const line of content.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const eq = trimmed.indexOf('=');
|
||||
if (eq <= 0) continue;
|
||||
const key = trimmed.slice(0, eq).trim();
|
||||
let value = trimmed.slice(eq + 1).trim();
|
||||
if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) {
|
||||
value = value.slice(1, -1).replace(/\\n/g, '\n').replace(/\\"/g, '"').replace(/\\\\/g, '\\');
|
||||
}
|
||||
secrets[key] = value;
|
||||
}
|
||||
return secrets;
|
||||
}
|
||||
|
||||
/** Serializes secrets with double-quoted, escaped values (dotenv-compatible). */
|
||||
export function serializeSecrets(secrets: Record<string, string>): string {
|
||||
const lines = [
|
||||
'# Managed by Dorfteich (setup wizard). Values here fill environment',
|
||||
'# variables that the container environment does not set explicitly.',
|
||||
];
|
||||
for (const [key, value] of Object.entries(secrets)) {
|
||||
const escaped = value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');
|
||||
lines.push(`${key}="${escaped}"`);
|
||||
}
|
||||
return lines.join('\n') + '\n';
|
||||
}
|
||||
export { overlayEnv, parseSecretsFile, serializeSecrets } from '@dorfteich/shared';
|
||||
|
||||
/** Reads the store file; a missing file is an empty store, not an error. */
|
||||
export function readSecretsFile(path: string): Record<string, string> {
|
||||
@ -47,27 +19,6 @@ export function readSecretsFile(path: string): Record<string, string> {
|
||||
return parseSecretsFile(readFileSync(path, 'utf8'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges the store under the real environment: explicit env vars win, store
|
||||
* values fill the gaps (and Zod defaults fill whatever remains at parse
|
||||
* time). Empty strings count as unset on both sides — compose passes
|
||||
* `${SMTP_HOST:-}` as `""` for variables the stage `.env` does not define,
|
||||
* and those must not shadow wizard-written store values or schema defaults.
|
||||
*/
|
||||
export function overlayEnv(
|
||||
env: Record<string, string | undefined>,
|
||||
secrets: Record<string, string>,
|
||||
): Record<string, string | undefined> {
|
||||
const merged: Record<string, string | undefined> = {};
|
||||
for (const [key, value] of Object.entries(secrets)) {
|
||||
if (value !== '') merged[key] = value;
|
||||
}
|
||||
for (const [key, value] of Object.entries(env)) {
|
||||
if (value !== undefined && value !== '') merged[key] = value;
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges entries into the store file atomically (staging file + rename, so a
|
||||
* crash mid-write never leaves a torn file) and keeps it owner-only readable.
|
||||
|
||||
31
apps/backup/Dockerfile
Normal file
31
apps/backup/Dockerfile
Normal file
@ -0,0 +1,31 @@
|
||||
# Build context is the repository root (workspace build):
|
||||
# docker build -f apps/backup/Dockerfile .
|
||||
|
||||
FROM node:22.15-alpine AS build
|
||||
WORKDIR /repo
|
||||
RUN npm install -g pnpm@11
|
||||
COPY pnpm-workspace.yaml pnpm-lock.yaml package.json tsconfig.base.json ./
|
||||
COPY packages/shared ./packages/shared
|
||||
COPY apps/backup ./apps/backup
|
||||
RUN pnpm install --frozen-lockfile --filter @dorfteich/backup... \
|
||||
&& pnpm --filter @dorfteich/shared build \
|
||||
&& pnpm --filter @dorfteich/backup build \
|
||||
# Self-contained production bundle (prod deps only) at /out.
|
||||
&& pnpm --filter @dorfteich/backup deploy --prod --legacy /out \
|
||||
&& cp -r apps/backup/dist /out/dist
|
||||
|
||||
FROM node:22.15-alpine
|
||||
ARG APP_VERSION=0.0.0-dev
|
||||
ENV NODE_ENV=production APP_VERSION=${APP_VERSION} \
|
||||
# Baked-in volume paths (self-sufficient without compose env, like the
|
||||
# api image's PLUGINS_DIR — issue #71's lesson).
|
||||
BACKUPS_DIR=/backups UPLOADS_DIR=/data/uploads PLUGINS_DIR=/data/plugins \
|
||||
SECRETS_FILE=/data/secrets/secrets.env
|
||||
# pg_dump/pg_restore matching the stack's postgres:17 server, GNU tar for the
|
||||
# volume archives, tzdata so BACKUP_TIME honors a configured TZ.
|
||||
RUN apk add --no-cache postgresql17-client tar tzdata \
|
||||
&& mkdir -p /backups && chown node:node /backups
|
||||
WORKDIR /app
|
||||
COPY --from=build --chown=node:node /out /app
|
||||
USER node
|
||||
CMD ["node", "dist/index.js"]
|
||||
28
apps/backup/package.json
Normal file
28
apps/backup/package.json
Normal file
@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "@dorfteich/backup",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"description": "Dorfteich backup sidecar — nightly pg_dump + volume archive with prune, status.json, and failure mail (ADR 0015)",
|
||||
"license": "MIT",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"start": "node dist/index.js",
|
||||
"start:dev": "tsx watch src/index.ts",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "vitest run --passWithNoTests"
|
||||
},
|
||||
"dependencies": {
|
||||
"@dorfteich/shared": "workspace:*",
|
||||
"nodemailer": "^9.0.3",
|
||||
"pino": "^9.6.0",
|
||||
"zod": "^3.25.76"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^26.1.0",
|
||||
"@types/nodemailer": "^8.0.1",
|
||||
"tsx": "^4.19.0",
|
||||
"vitest": "^3.0.0"
|
||||
}
|
||||
}
|
||||
55
apps/backup/src/archive.test.ts
Normal file
55
apps/backup/src/archive.test.ts
Normal file
@ -0,0 +1,55 @@
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import { archiveBase, createArchive, extractArchive } from './archive.js';
|
||||
|
||||
let dir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), 'dorfteich-archive-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('archiveBase', () => {
|
||||
it('requires a shared parent directory', () => {
|
||||
expect(() => archiveBase(['/data/uploads', '/other/plugins'])).toThrow(/share one parent/);
|
||||
expect(archiveBase(['/data/uploads', '/data/plugins'])).toEqual({
|
||||
base: '/data',
|
||||
names: ['uploads', 'plugins'],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('create/extract round-trip (real tar)', () => {
|
||||
it('restores the archived files byte-identically', async () => {
|
||||
const source = join(dir, 'source');
|
||||
await mkdir(join(source, 'uploads', 'ab'), { recursive: true });
|
||||
await mkdir(join(source, 'plugins'), { recursive: true });
|
||||
await writeFile(join(source, 'uploads', 'ab', 'file.png'), 'png-bytes');
|
||||
await writeFile(join(source, 'plugins', 'manifest.json'), '{"id":"toc"}');
|
||||
|
||||
const archive = join(dir, 'files.tar.gz');
|
||||
await createArchive(archive, [join(source, 'uploads'), join(source, 'plugins')]);
|
||||
|
||||
const target = join(dir, 'target');
|
||||
await mkdir(target, { recursive: true });
|
||||
await extractArchive(archive, [join(target, 'uploads'), join(target, 'plugins')]);
|
||||
|
||||
expect(await readFile(join(target, 'uploads', 'ab', 'file.png'), 'utf8')).toBe('png-bytes');
|
||||
expect(await readFile(join(target, 'plugins', 'manifest.json'), 'utf8')).toBe('{"id":"toc"}');
|
||||
});
|
||||
|
||||
it('produces a valid empty archive when no data directory exists yet', async () => {
|
||||
const archive = join(dir, 'empty.tar.gz');
|
||||
await createArchive(archive, [join(dir, 'none', 'uploads'), join(dir, 'none', 'plugins')]);
|
||||
const target = join(dir, 'extract');
|
||||
await mkdir(target, { recursive: true });
|
||||
await extractArchive(archive, [join(target, 'uploads'), join(target, 'plugins')]);
|
||||
});
|
||||
});
|
||||
50
apps/backup/src/archive.ts
Normal file
50
apps/backup/src/archive.ts
Normal file
@ -0,0 +1,50 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { basename, dirname } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/**
|
||||
* The uploads and plugins directories travel in one tar archive per restore
|
||||
* set (ADR 0015). Both must share a parent directory (in the compose stack
|
||||
* that is `/data`): entries are stored relative to it (`uploads/…`,
|
||||
* `plugins/…`), so extraction lands exactly where the api reads.
|
||||
*/
|
||||
export function archiveBase(dataDirs: string[]): { base: string; names: string[] } {
|
||||
const bases = new Set(dataDirs.map((dir) => dirname(dir)));
|
||||
if (bases.size !== 1) {
|
||||
throw new Error(
|
||||
`data directories must share one parent to form a single archive, got: ${dataDirs.join(', ')}`,
|
||||
);
|
||||
}
|
||||
return { base: [...bases][0]!, names: dataDirs.map((dir) => basename(dir)) };
|
||||
}
|
||||
|
||||
async function runTar(args: string[]): Promise<void> {
|
||||
try {
|
||||
await execFileAsync('tar', args, { maxBuffer: 16 * 1024 * 1024 });
|
||||
} catch (error) {
|
||||
const stderr = (error as { stderr?: string }).stderr?.trim();
|
||||
throw new Error(`tar failed: ${stderr || (error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates the gzip'd volume archive; directories that do not exist yet
|
||||
* (fresh instance without uploads) are skipped, an empty set still produces
|
||||
* a valid empty archive. */
|
||||
export async function createArchive(outFile: string, dataDirs: string[]): Promise<void> {
|
||||
const { base, names } = archiveBase(dataDirs);
|
||||
const existing = names.filter((name) => existsSync(`${base}/${name}`));
|
||||
if (existing.length === 0) {
|
||||
await runTar(['-czf', outFile, '--files-from', '/dev/null']);
|
||||
return;
|
||||
}
|
||||
await runTar(['-czf', outFile, '-C', base, ...existing]);
|
||||
}
|
||||
|
||||
/** Unpacks a volume archive back over the data directories (restore path). */
|
||||
export async function extractArchive(archiveFile: string, dataDirs: string[]): Promise<void> {
|
||||
const { base } = archiveBase(dataDirs);
|
||||
await runTar(['-xzf', archiveFile, '-C', base]);
|
||||
}
|
||||
73
apps/backup/src/backup-set.test.ts
Normal file
73
apps/backup/src/backup-set.test.ts
Normal file
@ -0,0 +1,73 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
archiveFileName,
|
||||
backupIdTime,
|
||||
dumpFileName,
|
||||
expiredSets,
|
||||
listSets,
|
||||
newBackupId,
|
||||
} from './backup-set.js';
|
||||
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
|
||||
function setFiles(id: string): string[] {
|
||||
return [dumpFileName(id), archiveFileName(id)];
|
||||
}
|
||||
|
||||
describe('backup ids', () => {
|
||||
it('round-trips through the id format', () => {
|
||||
const now = new Date('2026-07-11T03:00:00Z');
|
||||
const id = newBackupId(now);
|
||||
expect(id).toBe('20260711-030000');
|
||||
expect(backupIdTime(id)?.toISOString()).toBe(now.toISOString());
|
||||
});
|
||||
|
||||
it('rejects malformed ids', () => {
|
||||
expect(backupIdTime('not-a-backup')).toBeNull();
|
||||
expect(backupIdTime('20260711')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('listSets', () => {
|
||||
it('groups files by id and flags completeness', () => {
|
||||
const sets = listSets([
|
||||
'status.json',
|
||||
'db-20260701-030000.dump.partial',
|
||||
...setFiles('20260710-030000'),
|
||||
'db-20260711-030000.dump',
|
||||
]);
|
||||
expect(sets.map((set) => set.id)).toEqual(['20260710-030000', '20260711-030000']);
|
||||
expect(sets[0]?.complete).toBe(true);
|
||||
expect(sets[1]?.complete).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('expiredSets', () => {
|
||||
const now = new Date('2026-07-31T04:00:00Z');
|
||||
|
||||
it('removes only sets past retention', () => {
|
||||
const fresh = newBackupId(new Date(now.getTime() - 2 * DAY));
|
||||
const old = newBackupId(new Date(now.getTime() - 10 * DAY));
|
||||
const newest = newBackupId(new Date(now.getTime() - 1 * DAY));
|
||||
const sets = listSets([...setFiles(old), ...setFiles(fresh), ...setFiles(newest)]);
|
||||
|
||||
const expired = expiredSets(sets, now, 7);
|
||||
expect(expired.map((set) => set.id)).toEqual([old]);
|
||||
});
|
||||
|
||||
it('never removes the newest complete set, even when expired', () => {
|
||||
const older = newBackupId(new Date(now.getTime() - 40 * DAY));
|
||||
const newestComplete = newBackupId(new Date(now.getTime() - 20 * DAY));
|
||||
// A newer but incomplete set (crashed run) must not shield the complete one.
|
||||
const newerIncomplete = newBackupId(new Date(now.getTime() - 9 * DAY));
|
||||
const sets = listSets([
|
||||
...setFiles(older),
|
||||
...setFiles(newestComplete),
|
||||
dumpFileName(newerIncomplete),
|
||||
]);
|
||||
|
||||
const expired = expiredSets(sets, now, 7);
|
||||
expect(expired.map((set) => set.id)).toEqual([older, newerIncomplete]);
|
||||
});
|
||||
});
|
||||
89
apps/backup/src/backup-set.ts
Normal file
89
apps/backup/src/backup-set.ts
Normal file
@ -0,0 +1,89 @@
|
||||
/**
|
||||
* A restore set (ADR 0015) is one nightly `pg_dump` plus the matching
|
||||
* uploads/plugins archive, tied together by a shared backup id derived from
|
||||
* the run's UTC start time. This module owns the naming scheme and the pure
|
||||
* prune decision; the runner applies it to the filesystem.
|
||||
*/
|
||||
|
||||
export interface BackupSet {
|
||||
id: string;
|
||||
/** File names (not paths) present in the backups directory. */
|
||||
files: string[];
|
||||
/** Complete = both the dump and the volume archive exist. */
|
||||
complete: boolean;
|
||||
}
|
||||
|
||||
const ID_PATTERN = /^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})$/;
|
||||
const SET_FILE_PATTERN = /^(?:db-|files-)(\d{8}-\d{6})\.(?:dump|tar\.gz)$/;
|
||||
|
||||
/** Backup id for a run starting now: UTC timestamp, filesystem-safe. */
|
||||
export function newBackupId(now: Date): string {
|
||||
const pad = (value: number): string => String(value).padStart(2, '0');
|
||||
return (
|
||||
`${now.getUTCFullYear()}${pad(now.getUTCMonth() + 1)}${pad(now.getUTCDate())}` +
|
||||
`-${pad(now.getUTCHours())}${pad(now.getUTCMinutes())}${pad(now.getUTCSeconds())}`
|
||||
);
|
||||
}
|
||||
|
||||
/** The UTC time encoded in a backup id, or null for a malformed id. */
|
||||
export function backupIdTime(id: string): Date | null {
|
||||
const match = ID_PATTERN.exec(id);
|
||||
if (!match) return null;
|
||||
const [, year, month, day, hour, minute, second] = match;
|
||||
return new Date(
|
||||
Date.UTC(
|
||||
Number(year),
|
||||
Number(month) - 1,
|
||||
Number(day),
|
||||
Number(hour),
|
||||
Number(minute),
|
||||
Number(second),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function dumpFileName(id: string): string {
|
||||
return `db-${id}.dump`;
|
||||
}
|
||||
|
||||
export function archiveFileName(id: string): string {
|
||||
return `files-${id}.tar.gz`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Groups the backup directory's file names into sets, oldest first. Files
|
||||
* that do not belong to the naming scheme (status.json, `.partial` staging
|
||||
* files of a running or crashed run) are ignored — prune never touches them.
|
||||
*/
|
||||
export function listSets(fileNames: string[]): BackupSet[] {
|
||||
const byId = new Map<string, string[]>();
|
||||
for (const name of fileNames) {
|
||||
const match = SET_FILE_PATTERN.exec(name);
|
||||
if (!match || !backupIdTime(match[1]!)) continue;
|
||||
const files = byId.get(match[1]!) ?? [];
|
||||
files.push(name);
|
||||
byId.set(match[1]!, files);
|
||||
}
|
||||
return [...byId.entries()]
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([id, files]) => ({
|
||||
id,
|
||||
files: files.sort(),
|
||||
complete: files.includes(dumpFileName(id)) && files.includes(archiveFileName(id)),
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* The sets prune may delete: older than the retention cutoff — but never
|
||||
* the newest complete set, even when it is expired. A stalled instance must
|
||||
* always keep one restorable set (issue #83 acceptance criteria).
|
||||
*/
|
||||
export function expiredSets(sets: BackupSet[], now: Date, retentionDays: number): BackupSet[] {
|
||||
const cutoff = now.getTime() - retentionDays * 24 * 60 * 60 * 1000;
|
||||
const newestComplete = [...sets].reverse().find((set) => set.complete);
|
||||
return sets.filter((set) => {
|
||||
if (set === newestComplete) return false;
|
||||
const time = backupIdTime(set.id);
|
||||
return time !== null && time.getTime() < cutoff;
|
||||
});
|
||||
}
|
||||
56
apps/backup/src/config.test.ts
Normal file
56
apps/backup/src/config.test.ts
Normal file
@ -0,0 +1,56 @@
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import { loadBackupEnv } from './config.js';
|
||||
|
||||
const DATABASE_URL = 'postgresql://dorfteich:pw@db:5432/dorfteich';
|
||||
|
||||
let dir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), 'dorfteich-backup-env-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('loadBackupEnv', () => {
|
||||
it('applies defaults and drops compose empty strings', () => {
|
||||
// Compose passes optional variables as "" — they must fall through to
|
||||
// the schema defaults, not fail enum/number parsing (issue #80 lesson).
|
||||
const env = loadBackupEnv({
|
||||
DATABASE_URL,
|
||||
BACKUP_TIME: '',
|
||||
BACKUP_RETENTION_DAYS: '',
|
||||
BACKUP_MAIL_LOCALE: '',
|
||||
SECRETS_FILE: join(dir, 'missing.env'),
|
||||
});
|
||||
expect(env.BACKUP_TIME).toBe('03:00');
|
||||
expect(env.BACKUP_RETENTION_DAYS).toBe(30);
|
||||
expect(env.BACKUP_MAIL_LOCALE).toBe('en');
|
||||
});
|
||||
|
||||
it('fills SMTP settings from the wizard-written secret store', () => {
|
||||
const secretsFile = join(dir, 'secrets.env');
|
||||
return writeFile(secretsFile, 'SMTP_HOST="relay.example.com"\nSMTP_PORT="2525"\n').then(() => {
|
||||
const env = loadBackupEnv({
|
||||
DATABASE_URL,
|
||||
SECRETS_FILE: secretsFile,
|
||||
// Explicit container env must win over the store.
|
||||
SMTP_PORT: '465',
|
||||
});
|
||||
expect(env.SMTP_HOST).toBe('relay.example.com');
|
||||
expect(env.SMTP_PORT).toBe(465);
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects a malformed BACKUP_TIME', () => {
|
||||
expect(() => loadBackupEnv({ DATABASE_URL, BACKUP_TIME: '25:99' })).toThrow(
|
||||
/BACKUP_TIME.*HH:MM/s,
|
||||
);
|
||||
});
|
||||
});
|
||||
24
apps/backup/src/config.ts
Normal file
24
apps/backup/src/config.ts
Normal file
@ -0,0 +1,24 @@
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
|
||||
import {
|
||||
backupEnvSchema,
|
||||
overlayEnv,
|
||||
parseEnv,
|
||||
parseSecretsFile,
|
||||
type BackupEnv,
|
||||
} from '@dorfteich/shared';
|
||||
|
||||
/**
|
||||
* Loads the sidecar configuration the same way the api does (issue #80):
|
||||
* the wizard-written secret store fills SMTP variables the container env
|
||||
* does not set, and empty strings count as unset — compose passes optional
|
||||
* variables as `""`. Without this, a wizard-configured relay would never
|
||||
* reach the failure mail.
|
||||
*/
|
||||
export function loadBackupEnv(env: Record<string, string | undefined> = process.env): BackupEnv {
|
||||
const secretsFile = backupEnvSchema.shape.SECRETS_FILE.parse(env.SECRETS_FILE || undefined);
|
||||
const secrets = existsSync(secretsFile)
|
||||
? parseSecretsFile(readFileSync(secretsFile, 'utf8'))
|
||||
: {};
|
||||
return parseEnv(backupEnvSchema, overlayEnv(env, secrets));
|
||||
}
|
||||
49
apps/backup/src/index.ts
Normal file
49
apps/backup/src/index.ts
Normal file
@ -0,0 +1,49 @@
|
||||
import { pino } from 'pino';
|
||||
|
||||
import { createArchive } from './archive.js';
|
||||
import { loadBackupEnv } from './config.js';
|
||||
import { sendFailureMail } from './mail.js';
|
||||
import { pgDump } from './pg.js';
|
||||
import { runBackup } from './runner.js';
|
||||
import { scheduleDaily } from './scheduler.js';
|
||||
import type { RunnerDeps } from './runner.js';
|
||||
|
||||
/**
|
||||
* Sidecar entrypoint: runs the nightly backup at BACKUP_TIME (ADR 0015).
|
||||
* `BACKUP_RUN_ONCE=1` runs a single backup and exits with the outcome as
|
||||
* the exit code — the on-demand path (`docker compose run backup`) and what
|
||||
* the admin panel's manual trigger (#86) will call.
|
||||
*/
|
||||
const env = loadBackupEnv();
|
||||
const log = pino({ level: env.LOG_LEVEL, base: { service: 'backup' } });
|
||||
|
||||
const deps: RunnerDeps = {
|
||||
backupsDir: env.BACKUPS_DIR,
|
||||
retentionDays: env.BACKUP_RETENTION_DAYS,
|
||||
now: () => new Date(),
|
||||
dump: (outFile) => pgDump(env.DATABASE_URL, outFile),
|
||||
archive: (outFile) => createArchive(outFile, [env.UPLOADS_DIR, env.PLUGINS_DIR]),
|
||||
onFailure: async (run) => {
|
||||
const sent = await sendFailureMail(env, run);
|
||||
if (!sent) log.warn({ backupId: run.backupId }, 'no BACKUP_MAIL_TO configured, alert not sent');
|
||||
},
|
||||
log,
|
||||
};
|
||||
|
||||
if (process.env.BACKUP_RUN_ONCE === '1') {
|
||||
const status = await runBackup(deps);
|
||||
process.exit(status.lastRun.outcome === 'succeeded' ? 0 : 1);
|
||||
}
|
||||
|
||||
log.info(
|
||||
{ time: env.BACKUP_TIME, retentionDays: env.BACKUP_RETENTION_DAYS, dir: env.BACKUPS_DIR },
|
||||
'backup sidecar started',
|
||||
);
|
||||
const schedule = scheduleDaily(env.BACKUP_TIME, async () => void (await runBackup(deps)), log);
|
||||
|
||||
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
|
||||
process.on(signal, () => {
|
||||
schedule.stop();
|
||||
process.exit(0);
|
||||
});
|
||||
}
|
||||
30
apps/backup/src/mail.test.ts
Normal file
30
apps/backup/src/mail.test.ts
Normal file
@ -0,0 +1,30 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { renderFailureMail } from './mail.js';
|
||||
|
||||
const input = {
|
||||
backupId: '20260711-030000',
|
||||
error: 'pg_dump failed: connection refused',
|
||||
lastSuccessAt: '2026-07-10T03:00:12.000Z',
|
||||
};
|
||||
|
||||
describe('renderFailureMail', () => {
|
||||
it('renders the English alert with all details interpolated', () => {
|
||||
const mail = renderFailureMail(input, 'en', 'dorfteich-test');
|
||||
expect(mail.subject).toBe('[dorfteich-test] Backup failed (20260711-030000)');
|
||||
expect(mail.text).toContain('Backup id: 20260711-030000');
|
||||
expect(mail.text).toContain('Error: pg_dump failed: connection refused');
|
||||
expect(mail.text).toContain('Last successful backup: 2026-07-10T03:00:12.000Z');
|
||||
});
|
||||
|
||||
it('renders the German alert', () => {
|
||||
const mail = renderFailureMail(input, 'de', 'dorfteich-test');
|
||||
expect(mail.subject).toBe('[dorfteich-test] Backup fehlgeschlagen (20260711-030000)');
|
||||
expect(mail.text).toContain('Fehler: pg_dump failed: connection refused');
|
||||
});
|
||||
|
||||
it('states when no backup ever succeeded', () => {
|
||||
const mail = renderFailureMail({ ...input, lastSuccessAt: null }, 'en', 'Dorfteich');
|
||||
expect(mail.text).toContain('Last successful backup: none yet');
|
||||
});
|
||||
});
|
||||
93
apps/backup/src/mail.ts
Normal file
93
apps/backup/src/mail.ts
Normal file
@ -0,0 +1,93 @@
|
||||
import { createTransport } from 'nodemailer';
|
||||
|
||||
import type { BackupEnv } from '@dorfteich/shared';
|
||||
|
||||
import deMails from '@dorfteich/shared/i18n/de/mails.json' with { type: 'json' };
|
||||
import enMails from '@dorfteich/shared/i18n/en/mails.json' with { type: 'json' };
|
||||
|
||||
/**
|
||||
* The failure alert goes out through nodemailer directly instead of the
|
||||
* api's mail outbox — when backups fail, the api may be the broken part
|
||||
* (issue #83). Texts live in the shared `mails` i18n namespace (ADR 0012,
|
||||
* de + en); the operator picks the language via BACKUP_MAIL_LOCALE. The
|
||||
* catalog uses i18next's `{{var}}` syntax, interpolated here without
|
||||
* pulling i18next into the sidecar.
|
||||
*/
|
||||
|
||||
export interface FailureMailInput {
|
||||
backupId: string;
|
||||
error: string;
|
||||
lastSuccessAt: string | null;
|
||||
}
|
||||
|
||||
export interface RenderedFailureMail {
|
||||
subject: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
const CATALOGS = { de: deMails, en: enMails } as const;
|
||||
|
||||
function t(
|
||||
locale: 'de' | 'en',
|
||||
key: keyof (typeof CATALOGS)['en']['backupFailed'],
|
||||
params: Record<string, string> = {},
|
||||
): string {
|
||||
let text: string = CATALOGS[locale].backupFailed[key];
|
||||
for (const [name, value] of Object.entries(params)) {
|
||||
text = text.replaceAll(`{{${name}}}`, value);
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
export function renderFailureMail(
|
||||
input: FailureMailInput,
|
||||
locale: 'de' | 'en',
|
||||
instanceLabel: string,
|
||||
): RenderedFailureMail {
|
||||
const lastSuccess = input.lastSuccessAt
|
||||
? t(locale, 'lastSuccess', { finishedAt: input.lastSuccessAt })
|
||||
: t(locale, 'lastSuccessNever');
|
||||
return {
|
||||
subject: t(locale, 'subject', { instance: instanceLabel, backupId: input.backupId }),
|
||||
text: [
|
||||
t(locale, 'intro', { instance: instanceLabel }),
|
||||
'',
|
||||
t(locale, 'backupId', { backupId: input.backupId }),
|
||||
t(locale, 'error', { error: input.error }),
|
||||
lastSuccess,
|
||||
'',
|
||||
t(locale, 'hint'),
|
||||
].join('\n'),
|
||||
};
|
||||
}
|
||||
|
||||
/** Sends the alert; returns false (after logging upstream) when no relay or
|
||||
* recipient is configured — a missing mail must never fail the run. */
|
||||
export async function sendFailureMail(env: BackupEnv, input: FailureMailInput): Promise<boolean> {
|
||||
if (!env.BACKUP_MAIL_TO) return false;
|
||||
const mail = renderFailureMail(
|
||||
input,
|
||||
env.BACKUP_MAIL_LOCALE,
|
||||
env.BACKUP_INSTANCE_LABEL || 'Dorfteich',
|
||||
);
|
||||
const transport = createTransport({
|
||||
host: env.SMTP_HOST,
|
||||
port: env.SMTP_PORT,
|
||||
secure: env.SMTP_SECURE,
|
||||
auth: env.SMTP_USER ? { user: env.SMTP_USER, pass: env.SMTP_PASS } : undefined,
|
||||
connectionTimeout: 10_000,
|
||||
greetingTimeout: 10_000,
|
||||
socketTimeout: 20_000,
|
||||
});
|
||||
try {
|
||||
await transport.sendMail({
|
||||
from: env.SMTP_FROM,
|
||||
to: env.BACKUP_MAIL_TO,
|
||||
subject: mail.subject,
|
||||
text: mail.text,
|
||||
});
|
||||
return true;
|
||||
} finally {
|
||||
transport.close();
|
||||
}
|
||||
}
|
||||
54
apps/backup/src/pg.ts
Normal file
54
apps/backup/src/pg.ts
Normal file
@ -0,0 +1,54 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/**
|
||||
* Connection settings for the libpq CLI tools as environment variables.
|
||||
* Deliberately not `--dbname=<url>`: the URL carries the password, and argv
|
||||
* is world-readable inside the container (`/proc/<pid>/cmdline`) — the
|
||||
* credential-handling rule is env/file only, never argv.
|
||||
*/
|
||||
export function pgEnvFromUrl(databaseUrl: string): Record<string, string> {
|
||||
const url = new URL(databaseUrl);
|
||||
const env: Record<string, string> = {
|
||||
PGHOST: url.hostname,
|
||||
PGDATABASE: decodeURIComponent(url.pathname.replace(/^\//, '')),
|
||||
};
|
||||
if (url.port) env.PGPORT = url.port;
|
||||
if (url.username) env.PGUSER = decodeURIComponent(url.username);
|
||||
if (url.password) env.PGPASSWORD = decodeURIComponent(url.password);
|
||||
return env;
|
||||
}
|
||||
|
||||
async function runPgTool(tool: string, args: string[], databaseUrl: string): Promise<void> {
|
||||
try {
|
||||
await execFileAsync(tool, args, {
|
||||
env: { ...process.env, ...pgEnvFromUrl(databaseUrl) },
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
});
|
||||
} catch (error) {
|
||||
const stderr = (error as { stderr?: string }).stderr?.trim();
|
||||
throw new Error(`${tool} failed: ${stderr || (error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** `pg_dump -Fc` of the whole database into `outFile` (ADR 0015). */
|
||||
export async function pgDump(databaseUrl: string, outFile: string): Promise<void> {
|
||||
await runPgTool('pg_dump', ['--format=custom', '--file', outFile], databaseUrl);
|
||||
}
|
||||
|
||||
/**
|
||||
* Restores a custom-format dump into the live database. `--clean
|
||||
* --if-exists` drops recreated objects first, so the restore lands on a
|
||||
* database that may still hold newer state (the runbook stops the app
|
||||
* services, not the db container).
|
||||
*/
|
||||
export async function pgRestore(databaseUrl: string, dumpFile: string): Promise<void> {
|
||||
const database = pgEnvFromUrl(databaseUrl).PGDATABASE ?? '';
|
||||
await runPgTool(
|
||||
'pg_restore',
|
||||
['--clean', '--if-exists', '--no-owner', '--dbname', database, dumpFile],
|
||||
databaseUrl,
|
||||
);
|
||||
}
|
||||
40
apps/backup/src/restore.ts
Normal file
40
apps/backup/src/restore.ts
Normal file
@ -0,0 +1,40 @@
|
||||
import { existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { pino } from 'pino';
|
||||
|
||||
import { extractArchive } from './archive.js';
|
||||
import { archiveFileName, backupIdTime, dumpFileName } from './backup-set.js';
|
||||
import { loadBackupEnv } from './config.js';
|
||||
import { pgRestore } from './pg.js';
|
||||
|
||||
/**
|
||||
* In-container half of the restore runbook (operations.md §Backup & restore):
|
||||
* `pg_restore --clean --if-exists` of the set's dump, then the volume archive
|
||||
* back over the uploads/plugins mounts. The host-side `restore.sh` wraps this
|
||||
* with stopping/starting the app services — run through that, not directly,
|
||||
* unless you know the api and collab are down.
|
||||
*/
|
||||
const env = loadBackupEnv();
|
||||
const log = pino({ level: env.LOG_LEVEL, base: { service: 'backup-restore' } });
|
||||
|
||||
const backupId = process.argv[2];
|
||||
if (!backupId || !backupIdTime(backupId)) {
|
||||
log.error({ backupId }, 'usage: node dist/restore.js <backup-id> (e.g. 20260711-030000)');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const dumpFile = join(env.BACKUPS_DIR, dumpFileName(backupId));
|
||||
const archiveFile = join(env.BACKUPS_DIR, archiveFileName(backupId));
|
||||
for (const file of [dumpFile, archiveFile]) {
|
||||
if (!existsSync(file)) {
|
||||
log.error({ file }, 'restore set is incomplete — file not found');
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
log.info({ backupId }, 'restoring database dump');
|
||||
await pgRestore(env.DATABASE_URL, dumpFile);
|
||||
log.info({ backupId }, 'restoring uploads/plugins archive');
|
||||
await extractArchive(archiveFile, [env.UPLOADS_DIR, env.PLUGINS_DIR]);
|
||||
log.info({ backupId }, 'restore complete — start the stack and verify /readyz');
|
||||
139
apps/backup/src/runner.test.ts
Normal file
139
apps/backup/src/runner.test.ts
Normal file
@ -0,0 +1,139 @@
|
||||
import { mkdtemp, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import { archiveFileName, dumpFileName } from './backup-set.js';
|
||||
import { runBackup, type RunnerDeps } from './runner.js';
|
||||
import { readStatus } from './status.js';
|
||||
|
||||
/**
|
||||
* Time-accelerated nightly runs (issue #83 acceptance criteria): the clock
|
||||
* is injected, so "a month of nights" is a loop, with real files in a temp
|
||||
* backups directory and dump/archive doubles standing in for pg_dump/tar
|
||||
* (their real counterparts are covered by pg.ts/archive.ts on the stage).
|
||||
*/
|
||||
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
const START = new Date('2026-07-01T03:00:00Z').getTime();
|
||||
|
||||
let dir: string;
|
||||
let clock: Date;
|
||||
let failures: Array<{ backupId: string; error: string; lastSuccessAt: string | null }>;
|
||||
|
||||
function makeDeps(overrides: Partial<RunnerDeps> = {}): RunnerDeps {
|
||||
return {
|
||||
backupsDir: dir,
|
||||
retentionDays: 7,
|
||||
now: () => new Date(clock),
|
||||
dump: (outFile) => writeFile(outFile, 'dump-bytes'),
|
||||
archive: (outFile) => writeFile(outFile, 'archive-bytes!'),
|
||||
onFailure: async (run) => {
|
||||
failures.push(run);
|
||||
},
|
||||
log: { info: () => {}, error: () => {} },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), 'dorfteich-backup-'));
|
||||
clock = new Date(START);
|
||||
failures = [];
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('runBackup', () => {
|
||||
it('produces a dump, a matching archive, and status.json', async () => {
|
||||
const status = await runBackup(makeDeps());
|
||||
|
||||
const names = await readdir(dir);
|
||||
expect(names).toContain(dumpFileName('20260701-030000'));
|
||||
expect(names).toContain(archiveFileName('20260701-030000'));
|
||||
expect(status.lastRun.outcome).toBe('succeeded');
|
||||
expect(status.lastRun.sizes).toEqual({ dumpBytes: 10, archiveBytes: 14 });
|
||||
expect(status.lastSuccess?.backupId).toBe('20260701-030000');
|
||||
expect(readStatus(dir)).toEqual(status);
|
||||
});
|
||||
|
||||
it('keeps exactly the retention window across simulated nights', async () => {
|
||||
for (let night = 0; night < 30; night += 1) {
|
||||
clock = new Date(START + night * DAY);
|
||||
await runBackup(makeDeps());
|
||||
}
|
||||
|
||||
const names = (await readdir(dir)).filter((name) => name.startsWith('db-'));
|
||||
// Retention 7 days: the last 7 nights survive plus the current night's set.
|
||||
expect(names.length).toBe(8);
|
||||
expect(names).toContain(dumpFileName('20260730-030000'));
|
||||
expect(names).not.toContain(dumpFileName('20260722-030000'));
|
||||
expect(names).toContain(dumpFileName('20260723-030000'));
|
||||
});
|
||||
|
||||
it('records a failed run, keeps lastSuccess, alerts, and cleans partials', async () => {
|
||||
await runBackup(makeDeps());
|
||||
|
||||
clock = new Date(START + DAY);
|
||||
const status = await runBackup(
|
||||
makeDeps({
|
||||
dump: async () => {
|
||||
throw new Error('connection refused');
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(status.lastRun.outcome).toBe('failed');
|
||||
expect(status.lastRun.error).toContain('connection refused');
|
||||
expect(status.lastSuccess?.backupId).toBe('20260701-030000');
|
||||
expect(failures).toHaveLength(1);
|
||||
expect(failures[0]?.lastSuccessAt).toBe(new Date(START).toISOString());
|
||||
|
||||
const names = await readdir(dir);
|
||||
expect(names.filter((name) => name.endsWith('.partial'))).toHaveLength(0);
|
||||
// The last good set survives a failed night.
|
||||
expect(names).toContain(dumpFileName('20260701-030000'));
|
||||
});
|
||||
|
||||
it('a failure alert that itself fails never breaks the run', async () => {
|
||||
const status = await runBackup(
|
||||
makeDeps({
|
||||
dump: async () => {
|
||||
throw new Error('disk full');
|
||||
},
|
||||
onFailure: async () => {
|
||||
throw new Error('smtp down');
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(status.lastRun.outcome).toBe('failed');
|
||||
expect(readStatus(dir)?.lastRun.error).toContain('disk full');
|
||||
});
|
||||
|
||||
it('after long downtime the newest complete set survives pruning', async () => {
|
||||
await runBackup(makeDeps());
|
||||
|
||||
// 60 days later (retention 7): the old set is expired but must survive
|
||||
// as the newest complete one until a new success replaces it.
|
||||
clock = new Date(START + 60 * DAY);
|
||||
const status = await runBackup(
|
||||
makeDeps({
|
||||
dump: async () => {
|
||||
throw new Error('db gone');
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(status.lastRun.outcome).toBe('failed');
|
||||
expect(await readdir(dir)).toContain(dumpFileName('20260701-030000'));
|
||||
|
||||
// The next success prunes it.
|
||||
clock = new Date(START + 61 * DAY);
|
||||
await runBackup(makeDeps());
|
||||
const names = await readdir(dir);
|
||||
expect(names).not.toContain(dumpFileName('20260701-030000'));
|
||||
expect(names).toContain(dumpFileName('20260831-030000'));
|
||||
});
|
||||
});
|
||||
132
apps/backup/src/runner.ts
Normal file
132
apps/backup/src/runner.ts
Normal file
@ -0,0 +1,132 @@
|
||||
import { mkdir, readdir, rename, rm, stat } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { archiveFileName, dumpFileName, expiredSets, listSets, newBackupId } from './backup-set.js';
|
||||
import {
|
||||
readStatus,
|
||||
writeStatus,
|
||||
type BackupRun,
|
||||
type BackupSizes,
|
||||
type BackupStatus,
|
||||
} from './status.js';
|
||||
|
||||
/**
|
||||
* One nightly run (ADR 0015): dump first, then the volume archive (files may
|
||||
* be minutes newer than the dump — documented consistency model), then prune,
|
||||
* then `status.json`. Artifacts are written under a `.partial` suffix and
|
||||
* renamed on completion, so a crash never leaves a file that looks like a
|
||||
* restorable artifact, and prune (which ignores `.partial`) never counts a
|
||||
* torn set as the "newest complete" one.
|
||||
*/
|
||||
|
||||
export interface RunnerDeps {
|
||||
backupsDir: string;
|
||||
retentionDays: number;
|
||||
now(): Date;
|
||||
/** Real implementation: pg_dump -Fc (pg.ts). */
|
||||
dump(outFile: string): Promise<void>;
|
||||
/** Real implementation: tar of the uploads/plugins mounts (archive.ts). */
|
||||
archive(outFile: string): Promise<void>;
|
||||
/** Failure alert (mail.ts); errors here are logged, never rethrown. */
|
||||
onFailure(run: { backupId: string; error: string; lastSuccessAt: string | null }): Promise<void>;
|
||||
log: {
|
||||
info(details: object, message: string): void;
|
||||
error(details: object, message: string): void;
|
||||
};
|
||||
}
|
||||
|
||||
export async function runBackup(deps: RunnerDeps): Promise<BackupStatus> {
|
||||
const startedAt = deps.now();
|
||||
const backupId = newBackupId(startedAt);
|
||||
const previous = readStatus(deps.backupsDir);
|
||||
const lastSuccess = previous?.lastSuccess ?? null;
|
||||
await mkdir(deps.backupsDir, { recursive: true });
|
||||
|
||||
const dumpFile = join(deps.backupsDir, dumpFileName(backupId));
|
||||
const archiveFile = join(deps.backupsDir, archiveFileName(backupId));
|
||||
|
||||
let run: BackupRun;
|
||||
let success: BackupStatus['lastSuccess'] = lastSuccess;
|
||||
try {
|
||||
const sizes = await produceArtifacts(deps, dumpFile, archiveFile);
|
||||
const finishedAt = deps.now();
|
||||
run = {
|
||||
backupId,
|
||||
startedAt: startedAt.toISOString(),
|
||||
finishedAt: finishedAt.toISOString(),
|
||||
durationMs: finishedAt.getTime() - startedAt.getTime(),
|
||||
outcome: 'succeeded',
|
||||
sizes,
|
||||
};
|
||||
success = { backupId, finishedAt: run.finishedAt, sizes };
|
||||
deps.log.info({ backupId, sizes }, 'backup run succeeded');
|
||||
} catch (error) {
|
||||
await removePartials(deps.backupsDir);
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const finishedAt = deps.now();
|
||||
run = {
|
||||
backupId,
|
||||
startedAt: startedAt.toISOString(),
|
||||
finishedAt: finishedAt.toISOString(),
|
||||
durationMs: finishedAt.getTime() - startedAt.getTime(),
|
||||
outcome: 'failed',
|
||||
error: message,
|
||||
};
|
||||
deps.log.error({ backupId, error: message }, 'backup run failed');
|
||||
try {
|
||||
await deps.onFailure({
|
||||
backupId,
|
||||
error: message,
|
||||
lastSuccessAt: lastSuccess?.finishedAt ?? null,
|
||||
});
|
||||
} catch (mailError) {
|
||||
deps.log.error({ backupId, error: String(mailError) }, 'failure alert could not be sent');
|
||||
}
|
||||
}
|
||||
|
||||
await prune(deps);
|
||||
|
||||
const status: BackupStatus = {
|
||||
schemaVersion: 1,
|
||||
updatedAt: deps.now().toISOString(),
|
||||
retentionDays: deps.retentionDays,
|
||||
lastRun: run,
|
||||
lastSuccess: success,
|
||||
};
|
||||
await writeStatus(deps.backupsDir, status);
|
||||
return status;
|
||||
}
|
||||
|
||||
/** Dump, then archive — each staged as `.partial` and renamed only when done. */
|
||||
async function produceArtifacts(
|
||||
deps: RunnerDeps,
|
||||
dumpFile: string,
|
||||
archiveFile: string,
|
||||
): Promise<BackupSizes> {
|
||||
await deps.dump(`${dumpFile}.partial`);
|
||||
await rename(`${dumpFile}.partial`, dumpFile);
|
||||
await deps.archive(`${archiveFile}.partial`);
|
||||
await rename(`${archiveFile}.partial`, archiveFile);
|
||||
return {
|
||||
dumpBytes: (await stat(dumpFile)).size,
|
||||
archiveBytes: (await stat(archiveFile)).size,
|
||||
};
|
||||
}
|
||||
|
||||
/** Leftover staging files: from this failed run or an earlier crash — no
|
||||
* resumption path exists, so they are dead weight either way. */
|
||||
async function removePartials(backupsDir: string): Promise<void> {
|
||||
for (const name of await readdir(backupsDir)) {
|
||||
if (name.endsWith('.partial')) await rm(join(backupsDir, name), { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function prune(deps: RunnerDeps): Promise<void> {
|
||||
const names = await readdir(deps.backupsDir);
|
||||
for (const set of expiredSets(listSets(names), deps.now(), deps.retentionDays)) {
|
||||
for (const file of set.files) {
|
||||
await rm(join(deps.backupsDir, file), { force: true });
|
||||
}
|
||||
deps.log.info({ backupId: set.id }, 'pruned expired backup set');
|
||||
}
|
||||
}
|
||||
22
apps/backup/src/scheduler.test.ts
Normal file
22
apps/backup/src/scheduler.test.ts
Normal file
@ -0,0 +1,22 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { msUntilNext } from './scheduler.js';
|
||||
|
||||
// msUntilNext works in local time (the container's TZ); the tests build
|
||||
// their expectations with local-time Dates, so they hold in any zone.
|
||||
describe('msUntilNext', () => {
|
||||
it('targets today when the time is still ahead', () => {
|
||||
const now = new Date(2026, 6, 11, 1, 30, 0);
|
||||
expect(msUntilNext(now, '03:00')).toBe(90 * 60 * 1000);
|
||||
});
|
||||
|
||||
it('targets tomorrow when the time has passed', () => {
|
||||
const now = new Date(2026, 6, 11, 3, 0, 1);
|
||||
expect(msUntilNext(now, '03:00')).toBe(24 * 60 * 60 * 1000 - 1000);
|
||||
});
|
||||
|
||||
it('an exact hit schedules a full day ahead (never a zero delay)', () => {
|
||||
const now = new Date(2026, 6, 11, 3, 0, 0);
|
||||
expect(msUntilNext(now, '03:00')).toBe(24 * 60 * 60 * 1000);
|
||||
});
|
||||
});
|
||||
32
apps/backup/src/scheduler.ts
Normal file
32
apps/backup/src/scheduler.ts
Normal file
@ -0,0 +1,32 @@
|
||||
/**
|
||||
* Daily scheduling without a cron dependency: one `setTimeout` to the next
|
||||
* HH:MM occurrence (container-local time via TZ), re-armed after every run.
|
||||
* A run that crosses midnight simply shifts the next one — backups are
|
||||
* hours apart, drift by seconds is irrelevant.
|
||||
*/
|
||||
|
||||
/** Milliseconds from `now` to the next local-time occurrence of `time` (HH:MM). */
|
||||
export function msUntilNext(now: Date, time: string): number {
|
||||
const [hours = 0, minutes = 0] = time.split(':').map(Number);
|
||||
const next = new Date(now);
|
||||
next.setHours(hours, minutes, 0, 0);
|
||||
if (next.getTime() <= now.getTime()) next.setDate(next.getDate() + 1);
|
||||
return next.getTime() - now.getTime();
|
||||
}
|
||||
|
||||
export function scheduleDaily(
|
||||
time: string,
|
||||
task: () => Promise<void>,
|
||||
log: { info(details: object, message: string): void },
|
||||
): { stop(): void } {
|
||||
let timer: NodeJS.Timeout;
|
||||
const arm = (): void => {
|
||||
const delay = msUntilNext(new Date(), time);
|
||||
log.info({ nextRunInMs: delay }, 'next backup run scheduled');
|
||||
timer = setTimeout(() => {
|
||||
void task().finally(arm);
|
||||
}, delay);
|
||||
};
|
||||
arm();
|
||||
return { stop: () => clearTimeout(timer) };
|
||||
}
|
||||
57
apps/backup/src/status.ts
Normal file
57
apps/backup/src/status.ts
Normal file
@ -0,0 +1,57 @@
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import { rename, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
|
||||
/**
|
||||
* `status.json` on the backups volume is the machine-readable outcome of the
|
||||
* last run (ADR 0015): the api's backup-freshness readiness check (#85) and
|
||||
* the admin panel's backup card (#86) consume it. Keep the shape additive —
|
||||
* bump `schemaVersion` on breaking changes.
|
||||
*/
|
||||
|
||||
export const STATUS_FILE = 'status.json';
|
||||
|
||||
export interface BackupSizes {
|
||||
dumpBytes: number;
|
||||
archiveBytes: number;
|
||||
}
|
||||
|
||||
export interface BackupRun {
|
||||
backupId: string;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
durationMs: number;
|
||||
outcome: 'succeeded' | 'failed';
|
||||
/** Present on failure: the first error the run hit, as a plain string. */
|
||||
error?: string;
|
||||
/** Present on success. */
|
||||
sizes?: BackupSizes;
|
||||
}
|
||||
|
||||
export interface BackupStatus {
|
||||
schemaVersion: 1;
|
||||
updatedAt: string;
|
||||
retentionDays: number;
|
||||
lastRun: BackupRun;
|
||||
/** Carried across failed runs so freshness checks see the real gap. */
|
||||
lastSuccess: { backupId: string; finishedAt: string; sizes: BackupSizes } | null;
|
||||
}
|
||||
|
||||
/** Reads the previous status; a missing or torn file is simply "no status". */
|
||||
export function readStatus(backupsDir: string): BackupStatus | null {
|
||||
const path = join(backupsDir, STATUS_FILE);
|
||||
if (!existsSync(path)) return null;
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, 'utf8')) as BackupStatus;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Atomic write (staging + rename) so readers never see a torn file. */
|
||||
export async function writeStatus(backupsDir: string, status: BackupStatus): Promise<void> {
|
||||
const path = join(backupsDir, STATUS_FILE);
|
||||
const staging = `${path}.tmp-${process.pid}`;
|
||||
await writeFile(staging, JSON.stringify(status, null, 2) + '\n');
|
||||
await rename(staging, path);
|
||||
}
|
||||
7
apps/backup/tsconfig.json
Normal file
7
apps/backup/tsconfig.json
Normal file
@ -0,0 +1,7 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
7
apps/backup/vitest.config.ts
Normal file
7
apps/backup/vitest.config.ts
Normal file
@ -0,0 +1,7 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
environment: 'node',
|
||||
},
|
||||
});
|
||||
40
deploy/backup/restore.sh
Executable file
40
deploy/backup/restore.sh
Executable file
@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env sh
|
||||
# Restore a nightly backup set (ADR 0015, operations.md §Backup & restore).
|
||||
#
|
||||
# Run from the stage directory (where docker-compose.yml and .env live):
|
||||
# deploy/backup/restore.sh <backup-id> # e.g. 20260711-030000
|
||||
#
|
||||
# Steps (the documented runbook, automated): stop the app services (the db
|
||||
# stays up — pg_restore needs it), replay the dump and the uploads/plugins
|
||||
# archive through the backup sidecar image, start the stack, check /readyz.
|
||||
set -eu
|
||||
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "usage: $0 <backup-id> (list sets: docker compose exec backup ls /backups)" >&2
|
||||
exit 2
|
||||
fi
|
||||
backup_id="$1"
|
||||
|
||||
echo "Stopping app services (db keeps running) ..."
|
||||
docker compose stop web api collab
|
||||
|
||||
echo "Restoring set ${backup_id} ..."
|
||||
docker compose run --rm --no-deps backup node dist/restore.js "${backup_id}"
|
||||
|
||||
echo "Starting the stack ..."
|
||||
docker compose up -d
|
||||
|
||||
api_port="$(docker compose port api 3000 2>/dev/null | head -n1)"
|
||||
if [ -n "${api_port}" ]; then
|
||||
echo "Waiting for readiness on ${api_port} ..."
|
||||
for _ in $(seq 1 30); do
|
||||
if curl -sf "http://${api_port}/api/v1/readyz" >/dev/null 2>&1; then
|
||||
echo "Restore complete — /readyz is green. Spot-check a page and a file."
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "Restore applied, but /readyz did not turn green within 60 s — check 'docker compose logs api'." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Restore applied. Verify /readyz through your reverse proxy."
|
||||
@ -48,6 +48,23 @@ SMTP_USER=wiki@example.com
|
||||
SMTP_PASS=change-me
|
||||
SMTP_FROM=Dorfteich <wiki@example.com>
|
||||
|
||||
# --- backups (ADR 0015, issue #83) --------------------------------------------
|
||||
# The backup sidecar dumps the database and archives the uploads/plugins
|
||||
# volumes nightly onto the `backups` volume; restore via
|
||||
# deploy/backup/restore.sh <backup-id>. All values optional.
|
||||
# Daily run time HH:MM in TZ (default 03:00; set TZ for stage-local time,
|
||||
# e.g. TZ=Europe/Berlin — unset means UTC).
|
||||
#TZ=Europe/Berlin
|
||||
#BACKUP_TIME=03:00
|
||||
# Local retention in days: 30 (default) for Prod, 7 for Test/Int (ADR 0015).
|
||||
#BACKUP_RETENTION_DAYS=30
|
||||
# Failure alert: recipient (unset = no mail, failures only in the logs and
|
||||
# status.json), mail language (de|en), and the label used in the subject
|
||||
# (defaults to the compose project name).
|
||||
#BACKUP_MAIL_TO=ops@example.com
|
||||
#BACKUP_MAIL_LOCALE=en
|
||||
#BACKUP_INSTANCE_LABEL=dorfteich-test
|
||||
|
||||
# --- first-run setup (optional pre-seeding, issue #80) ------------------------
|
||||
# A fresh (empty) database makes the instance require the browser setup
|
||||
# wizard. Automated deploys can skip it entirely by pre-seeding the Site
|
||||
|
||||
@ -127,6 +127,52 @@ services:
|
||||
condition: service_healthy
|
||||
<<: *logging
|
||||
|
||||
# Backup sidecar (ADR 0015, issue #83): nightly `pg_dump -Fc` + one tar of
|
||||
# the uploads/plugins volumes as a consistent restore set on the `backups`
|
||||
# volume, prune by retention, `status.json`, failure mail directly via SMTP
|
||||
# (the api may be the broken part). Restore runs through
|
||||
# deploy/backup/restore.sh, which drives this same image.
|
||||
backup:
|
||||
image: ${IMAGE_PREFIX:-dorfteich}-backup:${TAG:-latest}
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: apps/backup/Dockerfile
|
||||
args:
|
||||
APP_VERSION: ${TAG:-latest}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
NODE_ENV: production
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
DATABASE_URL: postgresql://dorfteich:${POSTGRES_PASSWORD:?set in .env}@db:5432/dorfteich
|
||||
# Daily run time (HH:MM) in TZ; retention 30 d default, 7 d on Test/Int.
|
||||
TZ: ${TZ:-}
|
||||
BACKUP_TIME: ${BACKUP_TIME:-}
|
||||
BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-}
|
||||
# Failure-alert recipient; empty disables the mail (logged instead).
|
||||
BACKUP_MAIL_TO: ${BACKUP_MAIL_TO:-}
|
||||
BACKUP_MAIL_LOCALE: ${BACKUP_MAIL_LOCALE:-}
|
||||
BACKUP_INSTANCE_LABEL: ${BACKUP_INSTANCE_LABEL:-${COMPOSE_PROJECT_NAME:-dorfteich}}
|
||||
# Same SMTP resolution as the api: explicit env wins, the wizard-written
|
||||
# secret store fills the gaps (issue #80).
|
||||
SMTP_HOST: ${SMTP_HOST:-}
|
||||
SMTP_PORT: ${SMTP_PORT:-}
|
||||
SMTP_SECURE: ${SMTP_SECURE:-}
|
||||
SMTP_USER: ${SMTP_USER:-}
|
||||
SMTP_PASS: ${SMTP_PASS:-}
|
||||
SMTP_FROM: ${SMTP_FROM:-}
|
||||
networks: [internal]
|
||||
volumes:
|
||||
# Write access to uploads/plugins is for the restore path only; the
|
||||
# nightly run just reads them into the archive.
|
||||
- uploads:/data/uploads
|
||||
- plugins:/data/plugins
|
||||
- secrets:/data/secrets:ro
|
||||
- backups:/backups
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
<<: *logging
|
||||
|
||||
db:
|
||||
image: postgres:17.5-alpine
|
||||
restart: unless-stopped
|
||||
@ -182,3 +228,4 @@ volumes:
|
||||
uploads:
|
||||
plugins:
|
||||
secrets:
|
||||
backups:
|
||||
|
||||
@ -23,7 +23,9 @@ Every stage (and every self-hosted instance) runs the same services:
|
||||
| `gotenberg` | `gotenberg/gotenberg:<pinned>` | internal only |
|
||||
| `backup` | `dorfteich-backup` | cron sidecar: pg_dump, volume archive, prune, mirror (ADR 0015) |
|
||||
|
||||
Volumes: `db-data`, `uploads` (uploads + installed plugins), `backups`.
|
||||
Volumes: `db-data`, `uploads`, `plugins` (installed plugin bundles),
|
||||
`secrets` (wizard-written secret store), `backups` (restore sets +
|
||||
`status.json`).
|
||||
Networks: `frontend` (reverse proxy ↔ web/api/collab) and `internal`
|
||||
(api/collab ↔ db/pandoc/gotenberg); db and converters are never exposed.
|
||||
|
||||
|
||||
@ -36,14 +36,19 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack.
|
||||
|
||||
## Backup & restore (operational view of ADR 0015)
|
||||
|
||||
- Nightly at 03:00 stage-local time: `pg_dump -Fc` → uploads/plugins volume
|
||||
archive → prune (> 30 days Prod, > 7 days Test/Int) → rsync mirror to
|
||||
BASEL `/home/RAID/BACKUPS/dorfteich-prod/` (Prod only, dedicated
|
||||
`dorfteich-backup` user).
|
||||
- **Restore runbook** (also the Prod-relocation procedure):
|
||||
1. `docker compose down` (keep volumes),
|
||||
- Nightly at 03:00 stage-local time (sidecar `backup`, issue #83; env
|
||||
`BACKUP_TIME`/`TZ`): `pg_dump -Fc` → uploads/plugins volume archive (one
|
||||
tar, same backup id `YYYYMMDD-HHMMSS`) → prune (`BACKUP_RETENTION_DAYS`,
|
||||
30 default / 7 Test+Int; the newest complete set always survives) →
|
||||
`status.json` on the `backups` volume → on failure a mail directly via
|
||||
the instance SMTP to `BACKUP_MAIL_TO`. Mirror to BASEL is issue #84.
|
||||
- **On-demand run**: `docker compose run --rm -e BACKUP_RUN_ONCE=1 backup`
|
||||
(exit code = outcome); list sets with `docker compose exec backup ls /backups`.
|
||||
- **Restore runbook** (also the Prod-relocation procedure) — automated by
|
||||
`deploy/backup/restore.sh <backup-id>`, run from the stage directory:
|
||||
1. stop the app services (`web`, `api`, `collab`; the db stays up),
|
||||
2. restore DB: `pg_restore --clean --if-exists` into the `db` container,
|
||||
3. restore volume: unpack the matching uploads archive,
|
||||
3. restore volume: unpack the matching uploads/plugins archive,
|
||||
4. `docker compose up -d`, verify `/readyz`, spot-check a page + a file.
|
||||
- **Drills**: monthly automated restore of the latest Prod dump into a
|
||||
scratch database on Test with a row-count sanity report; quarterly manual
|
||||
|
||||
@ -21,5 +21,14 @@
|
||||
"body": "diese Testnachricht bestätigt, dass dein Dorfteich E-Mails über den konfigurierten SMTP-Server versenden kann. Deine Instanz erreichst du hier:",
|
||||
"action": "Dorfteich öffnen",
|
||||
"expiry": "Du kannst diese E-Mail einfach löschen."
|
||||
},
|
||||
"backupFailed": {
|
||||
"subject": "[{{instance}}] Backup fehlgeschlagen ({{backupId}})",
|
||||
"intro": "Der nächtliche Backup-Lauf auf {{instance}} ist fehlgeschlagen.",
|
||||
"backupId": "Backup-ID: {{backupId}}",
|
||||
"error": "Fehler: {{error}}",
|
||||
"lastSuccess": "Letztes erfolgreiches Backup: {{finishedAt}}",
|
||||
"lastSuccessNever": "Letztes erfolgreiches Backup: noch keines",
|
||||
"hint": "Prüfe die Sidecar-Logs (docker compose logs backup) und die status.json auf dem Backups-Volume."
|
||||
}
|
||||
}
|
||||
|
||||
@ -21,5 +21,14 @@
|
||||
"body": "this test message confirms that your Dorfteich can send e-mail through the configured SMTP server. You can reach your instance here:",
|
||||
"action": "Open Dorfteich",
|
||||
"expiry": "You can simply delete this e-mail."
|
||||
},
|
||||
"backupFailed": {
|
||||
"subject": "[{{instance}}] Backup failed ({{backupId}})",
|
||||
"intro": "The nightly backup run on {{instance}} failed.",
|
||||
"backupId": "Backup id: {{backupId}}",
|
||||
"error": "Error: {{error}}",
|
||||
"lastSuccess": "Last successful backup: {{finishedAt}}",
|
||||
"lastSuccessNever": "Last successful backup: none yet",
|
||||
"hint": "Check the sidecar logs (docker compose logs backup) and status.json on the backups volume."
|
||||
}
|
||||
}
|
||||
|
||||
@ -27,6 +27,25 @@ const databaseUrl = z
|
||||
*/
|
||||
const collabTokenSecret = z.string().min(16).default('dev-insecure-collab-token-secret-change-me');
|
||||
|
||||
/**
|
||||
* SMTP delivery fields, shared by the api (transactional mail) and the
|
||||
* backup sidecar (failure alert mail, issue #83). Defaults match the
|
||||
* Mailpit container from the dev overlay; production instances configure
|
||||
* their real relay in the stage `.env` or through the M8 setup wizard,
|
||||
* whose secret store both services overlay the same way.
|
||||
*/
|
||||
const smtpFields = {
|
||||
SMTP_HOST: z.string().default('localhost'),
|
||||
SMTP_PORT: z.coerce.number().int().default(1025),
|
||||
SMTP_SECURE: z
|
||||
.enum(['true', 'false'])
|
||||
.default('false')
|
||||
.transform((value) => value === 'true'),
|
||||
SMTP_USER: z.string().optional(),
|
||||
SMTP_PASS: z.string().optional(),
|
||||
SMTP_FROM: z.string().default('Dorfteich <no-reply@localhost>'),
|
||||
};
|
||||
|
||||
export const apiEnvSchema = z.object({
|
||||
NODE_ENV: nodeEnv,
|
||||
PORT: z.coerce.number().int().min(1).max(65535).default(3000),
|
||||
@ -41,20 +60,7 @@ export const apiEnvSchema = z.object({
|
||||
.transform((value) => value === 'true'),
|
||||
/** Public base URL of this instance — used in e-mail links. */
|
||||
APP_BASE_URL: z.string().url().default('http://localhost:5173'),
|
||||
/**
|
||||
* SMTP delivery. Defaults match the Mailpit container from the dev
|
||||
* overlay; production instances configure their real relay here (the
|
||||
* M8 setup wizard writes these).
|
||||
*/
|
||||
SMTP_HOST: z.string().default('localhost'),
|
||||
SMTP_PORT: z.coerce.number().int().default(1025),
|
||||
SMTP_SECURE: z
|
||||
.enum(['true', 'false'])
|
||||
.default('false')
|
||||
.transform((value) => value === 'true'),
|
||||
SMTP_USER: z.string().optional(),
|
||||
SMTP_PASS: z.string().optional(),
|
||||
SMTP_FROM: z.string().default('Dorfteich <no-reply@localhost>'),
|
||||
...smtpFields,
|
||||
/**
|
||||
* Filesystem root for uploaded files (ADR 0011). The compose stack
|
||||
* mounts the `uploads` volume at `/data/uploads` and sets this
|
||||
@ -132,6 +138,41 @@ export const collabEnvSchema = z.object({
|
||||
|
||||
export type CollabEnv = z.infer<typeof collabEnvSchema>;
|
||||
|
||||
/**
|
||||
* Configuration for the backup sidecar (ADR 0015, issue #83). It talks to
|
||||
* the same database and mounts the same data volumes as the api, plus its
|
||||
* own `backups` volume for the nightly restore sets and `status.json`.
|
||||
*/
|
||||
export const backupEnvSchema = z.object({
|
||||
NODE_ENV: nodeEnv,
|
||||
LOG_LEVEL: logLevel,
|
||||
APP_VERSION: appVersion,
|
||||
DATABASE_URL: databaseUrl,
|
||||
/** Where restore sets and `status.json` are written (the `backups` volume). */
|
||||
BACKUPS_DIR: z.string().min(1).default('./data/backups'),
|
||||
/** Same mounts as the api — archived together as one restore set. */
|
||||
UPLOADS_DIR: z.string().min(1).default('./data/uploads'),
|
||||
PLUGINS_DIR: z.string().min(1).default('./data/plugins'),
|
||||
/** Daily run time as HH:MM, interpreted in the container's TZ. */
|
||||
BACKUP_TIME: z
|
||||
.string()
|
||||
.regex(/^([01]\d|2[0-3]):[0-5]\d$/, 'must be HH:MM (24h)')
|
||||
.default('03:00'),
|
||||
/** Local retention in days: 30 for Prod, 7 for Test/Int (ADR 0015). */
|
||||
BACKUP_RETENTION_DAYS: z.coerce.number().int().min(1).default(30),
|
||||
/** Failure-alert recipient; unset disables the mail (logged instead). */
|
||||
BACKUP_MAIL_TO: z.string().optional(),
|
||||
/** Language of the failure mail (ADR 0012 — both exist, operator picks). */
|
||||
BACKUP_MAIL_LOCALE: z.enum(['de', 'en']).default('en'),
|
||||
/** Instance label in the mail subject, e.g. "dorfteich-test". */
|
||||
BACKUP_INSTANCE_LABEL: z.string().optional(),
|
||||
...smtpFields,
|
||||
/** Read-only view of the wizard-written secret store (issue #80). */
|
||||
SECRETS_FILE: z.string().min(1).default('./data/secrets.env'),
|
||||
});
|
||||
|
||||
export type BackupEnv = z.infer<typeof backupEnvSchema>;
|
||||
|
||||
export function parseEnv<Schema extends z.ZodTypeAny>(
|
||||
schema: Schema,
|
||||
env: Record<string, string | undefined>,
|
||||
|
||||
@ -17,6 +17,7 @@ export * from './pages';
|
||||
export * from './permissions';
|
||||
export * from './plugins';
|
||||
export * from './search';
|
||||
export * from './secret-store';
|
||||
export * from './setup';
|
||||
export * from './ponds';
|
||||
export * from './quotas';
|
||||
|
||||
62
packages/shared/src/secret-store.ts
Normal file
62
packages/shared/src/secret-store.ts
Normal file
@ -0,0 +1,62 @@
|
||||
/**
|
||||
* The env-backed secret store (security.md §Secrets, issue #80): secrets the
|
||||
* setup wizard collects in the browser (SMTP credentials) are persisted as a
|
||||
* mode-600 dotenv-style file on a volume — never as database rows. The file
|
||||
* extends the environment: `overlayEnv` fills only variables the process
|
||||
* environment does not set. These pure format/merge helpers live in shared
|
||||
* because two services read the store the api writes: the api itself and the
|
||||
* backup sidecar (issue #83), which needs the wizard's SMTP relay for its
|
||||
* failure mail. File I/O stays with each service.
|
||||
*/
|
||||
|
||||
/** Parses the dotenv-style store content. Ignores blank lines and comments. */
|
||||
export function parseSecretsFile(content: string): Record<string, string> {
|
||||
const secrets: Record<string, string> = {};
|
||||
for (const line of content.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const eq = trimmed.indexOf('=');
|
||||
if (eq <= 0) continue;
|
||||
const key = trimmed.slice(0, eq).trim();
|
||||
let value = trimmed.slice(eq + 1).trim();
|
||||
if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) {
|
||||
value = value.slice(1, -1).replace(/\\n/g, '\n').replace(/\\"/g, '"').replace(/\\\\/g, '\\');
|
||||
}
|
||||
secrets[key] = value;
|
||||
}
|
||||
return secrets;
|
||||
}
|
||||
|
||||
/** Serializes secrets with double-quoted, escaped values (dotenv-compatible). */
|
||||
export function serializeSecrets(secrets: Record<string, string>): string {
|
||||
const lines = [
|
||||
'# Managed by Dorfteich (setup wizard). Values here fill environment',
|
||||
'# variables that the container environment does not set explicitly.',
|
||||
];
|
||||
for (const [key, value] of Object.entries(secrets)) {
|
||||
const escaped = value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');
|
||||
lines.push(`${key}="${escaped}"`);
|
||||
}
|
||||
return lines.join('\n') + '\n';
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges the store under the real environment: explicit env vars win, store
|
||||
* values fill the gaps (and Zod defaults fill whatever remains at parse
|
||||
* time). Empty strings count as unset on both sides — compose passes
|
||||
* `${SMTP_HOST:-}` as `""` for variables the stage `.env` does not define,
|
||||
* and those must not shadow wizard-written store values or schema defaults.
|
||||
*/
|
||||
export function overlayEnv(
|
||||
env: Record<string, string | undefined>,
|
||||
secrets: Record<string, string>,
|
||||
): Record<string, string | undefined> {
|
||||
const merged: Record<string, string | undefined> = {};
|
||||
for (const [key, value] of Object.entries(secrets)) {
|
||||
if (value !== '') merged[key] = value;
|
||||
}
|
||||
for (const [key, value] of Object.entries(env)) {
|
||||
if (value !== undefined && value !== '') merged[key] = value;
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
28
pnpm-lock.yaml
generated
28
pnpm-lock.yaml
generated
@ -166,6 +166,34 @@ importers:
|
||||
specifier: ^3.0.0
|
||||
version: 3.2.6(@types/node@26.1.0)(jiti@2.7.0)(jsdom@26.1.0)(terser@5.48.0)(tsx@4.23.0)
|
||||
|
||||
apps/backup:
|
||||
dependencies:
|
||||
'@dorfteich/shared':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/shared
|
||||
nodemailer:
|
||||
specifier: ^9.0.3
|
||||
version: 9.0.3
|
||||
pino:
|
||||
specifier: ^9.6.0
|
||||
version: 9.14.0
|
||||
zod:
|
||||
specifier: ^3.25.76
|
||||
version: 3.25.76
|
||||
devDependencies:
|
||||
'@types/node':
|
||||
specifier: ^26.1.0
|
||||
version: 26.1.0
|
||||
'@types/nodemailer':
|
||||
specifier: ^8.0.1
|
||||
version: 8.0.1
|
||||
tsx:
|
||||
specifier: ^4.19.0
|
||||
version: 4.23.0
|
||||
vitest:
|
||||
specifier: ^3.0.0
|
||||
version: 3.2.6(@types/node@26.1.0)(jiti@2.7.0)(jsdom@26.1.0)(terser@5.48.0)(tsx@4.23.0)
|
||||
|
||||
apps/collab:
|
||||
dependencies:
|
||||
'@dorfteich/shared':
|
||||
|
||||
Loading…
Reference in New Issue
Block a user