dorfteich/apps/backup/src/index.ts
Claude Fable 5 52192eb05f
All checks were successful
CD / Build and push images (push) Successful in 3m51s
CI / Lint, typecheck, test (push) Successful in 4m5s
CD / Deploy to Test (push) Successful in 11s
CI / Build container images (push) Has been skipped
CD / Smoke tests against Test (push) Successful in 1m11s
CD / Promote to Int (push) Successful in 12s
CI / Auth e2e pack (push) Successful in 5m52s
CI / Import/export fidelity gate (push) Successful in 47s
Backup mirror to BASEL: rsync of the sets after every successful run (#84)
The operator-level extra beside the admin-configured Nextcloud target
(#103), unblocked now that the ONE→BASEL tunnel is stable again.

- sidecar: optional mirror step (mirror.ts) driven purely by env —
  BACKUP_MIRROR_TARGET (rsync-over-ssh), BACKUP_MIRROR_SSH_KEY (private
  key on the secrets volume, never in image or repo),
  BACKUP_MIRROR_SSH_PORT. Runs after the prune of every successful run,
  so --delete aligns the remote retention with the local one (the
  newest-complete-set guarantee carries over). Only set files travel
  (db-*.dump, files-*.tar.gz); status files and bundles stay local.
  Host key pinned via accept-new into .mirror_known_hosts on the backups
  volume; fixed remote modes (dirs 750, files 640, symbolic --chmod —
  octal needs rsync ≥ 3, macOS dev machines ship 2.6.9). rsync +
  openssh-client added to the sidecar image.
- status: additive `mirror` block in status.json (outcome, transferred
  count, lastSuccessAt carried across failures) — shown on the admin
  backup card; failures alert via a new backupMirrorFailed mail (de+en)
  while the local run still counts as succeeded.
- deploy/backup-basel.md: complete BASEL-side walkthrough — dedicated
  user dorfteich-backup with a /home/ home and a bash login shell,
  explicitly avoiding the Debian backup-user (UID 34) pitfalls
  (nologin shell rejects rsync sessions, /var/backups home), key
  placement through the api container onto the secrets volume, .env
  values, on-demand verification.
- tests: rsync-arg/stats-parsing units plus an integration suite against
  the real rsync binary (local target; skips where rsync is absent) —
  transfer, idempotent re-run (0 files), retention alignment, failure
  path carrying lastSuccessAt.

Verified live against the real BASEL host from a native sidecar run:
initial transfer, host-key pinning, retention alignment after a local
prune, idempotency, and the failure path (surfaced in status.json while
the local run stayed green). BASEL side provisioned per the doc.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1
2026-07-12 12:20:32 +02:00

212 lines
7.3 KiB
TypeScript

import { existsSync, readFileSync } from 'node:fs';
import { BACKUP_MAINTENANCE_CHANNEL, parseSecretsFile } from '@dorfteich/shared';
import { Client } from 'pg';
import { pino } from 'pino';
import { createArchive } from './archive.js';
import { createCommandListener } from './commands.js';
import { loadBackupEnv } from './config.js';
import { sendFailureMail } from './mail.js';
import { mirrorSets, resolveMirrorConfig } from './mirror.js';
import { performRestore } from './perform-restore.js';
import { pgDump } from './pg.js';
import { fetchRemoteSet, resolveRemoteTarget, uploadDue, uploadSet } from './remote.js';
import { orchestrateRestore } from './restore-orchestrator.js';
import { runBackup } from './runner.js';
import { scheduleDaily } from './scheduler.js';
import { readBackupDbSettings } from './settings.js';
import type { RunnerDeps } from './runner.js';
import type { BackupCommand } from '@dorfteich/shared';
/**
* Sidecar entrypoint: runs the nightly backup at BACKUP_TIME (ADR 0015) and
* listens for api commands ("back up now", "restore set X", issue #103).
* `BACKUP_RUN_ONCE=1` runs a single backup and exits with the outcome as
* the exit code — the on-demand compose path.
*/
const env = loadBackupEnv();
const log = pino({ level: env.LOG_LEVEL, base: { service: 'backup' } });
function readSecrets(): Record<string, string> {
return existsSync(env.SECRETS_FILE)
? parseSecretsFile(readFileSync(env.SECRETS_FILE, 'utf8'))
: {};
}
/**
* Composes the runner dependencies for one run. Settings come fresh from
* the database (admin changes apply without a restart); the DB row wins
* over the env for local retention, the env stays authoritative until an
* admin saves the setting once. A manual trigger uploads whenever a target
* is configured; scheduled runs honor the upload schedule.
*/
async function buildRunnerDeps(trigger: 'scheduled' | 'manual'): Promise<RunnerDeps> {
const settings = await readBackupDbSettings(env.DATABASE_URL);
const target = resolveRemoteTarget(settings, readSecrets());
const mirrorConfig = resolveMirrorConfig(env);
return {
backupsDir: env.BACKUPS_DIR,
retentionDays: settings.localRetentionDays ?? env.BACKUP_RETENTION_DAYS,
now: () => new Date(),
dump: (outFile) => pgDump(env.DATABASE_URL, outFile),
archive: (outFile) => createArchive(outFile, [env.UPLOADS_DIR, env.PLUGINS_DIR]),
onFailure: async (run) => {
const sent = await sendFailureMail(env, run);
if (!sent)
log.warn({ backupId: run.backupId }, 'no BACKUP_MAIL_TO configured, alert not sent');
},
upload: target
? async ({ backupId, previous }) => {
const due =
trigger === 'manual' ||
uploadDue(
settings.nextcloud.uploadSchedule,
previous?.lastSuccessfulUpload?.finishedAt ?? null,
new Date(),
);
if (!due) return undefined;
return uploadSet({
env,
backupsDir: env.BACKUPS_DIR,
target,
remoteRetentionDays: settings.remoteRetentionDays,
backupId,
previous,
now: () => new Date(),
log,
});
}
: undefined,
mirror: mirrorConfig
? ({ previous }) =>
mirrorSets({
env,
config: mirrorConfig,
backupsDir: env.BACKUPS_DIR,
previous,
now: () => new Date(),
log,
})
: undefined,
log,
};
}
/**
* runBackup handles run errors itself (failed status + alert); this guard
* catches what it cannot — e.g. an unwritable backups volume, where even
* status.json fails. Logging instead of crashing keeps the container out
* of a restart loop; the missed run shows up through the freshness check.
*/
async function guardedRun(trigger: 'scheduled' | 'manual'): Promise<'succeeded' | 'failed'> {
try {
return (await runBackup(await buildRunnerDeps(trigger))).lastRun.outcome;
} catch (error) {
log.error({ error: String(error) }, 'backup run could not even record its status');
return 'failed';
}
}
if (process.env.BACKUP_RUN_ONCE === '1') {
process.exit((await guardedRun('manual')) === 'succeeded' ? 0 : 1);
}
/**
* All work — nightly runs, manual runs, restores — flows through one
* serial queue: a restore must never race a running backup, and command
* bursts must not overlap. Tasks never reject (each wraps its own errors).
*/
let queue: Promise<unknown> = Promise.resolve();
function enqueue(name: string, task: () => Promise<unknown>): void {
queue = queue.then(async () => {
try {
await task();
} catch (error) {
log.error({ task: name, error: String(error) }, 'queued task failed unexpectedly');
}
});
}
async function notifyMaintenance(event: { phase: 'enter' | 'exit' }): Promise<void> {
const client = new Client({ connectionString: env.DATABASE_URL });
try {
await client.connect();
await client.query('SELECT pg_notify($1, $2)', [
BACKUP_MAINTENANCE_CHANNEL,
JSON.stringify(event),
]);
} finally {
await client.end().catch(() => undefined);
}
}
/**
* Kills every other connection to the database right before pg_restore —
* the in-app equivalent of restore.sh stopping the app services. The api
* is already holding requests in maintenance mode and restarts afterwards;
* pools and listeners (including our own command listener) reconnect.
*/
async function terminateOtherConnections(): Promise<void> {
const client = new Client({ connectionString: env.DATABASE_URL });
try {
await client.connect();
await client.query(
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
WHERE datname = current_database() AND pid <> pg_backend_pid()`,
);
} finally {
await client.end().catch(() => undefined);
}
}
async function handleRestore(command: Extract<BackupCommand, { kind: 'restore' }>): Promise<void> {
const settings = await readBackupDbSettings(env.DATABASE_URL);
const target = resolveRemoteTarget(settings, readSecrets());
await orchestrateRestore(
{
backupsDir: env.BACKUPS_DIR,
now: () => new Date(),
notifyMaintenance,
terminateOtherConnections,
fetchRemoteSet: async (backupId) => {
if (!target) throw new Error('no Nextcloud target configured');
await fetchRemoteSet({ backupsDir: env.BACKUPS_DIR, target, backupId, log });
},
restoreSet: (backupId) => performRestore(env, backupId, log),
log,
},
command,
);
}
log.info(
{ time: env.BACKUP_TIME, retentionDays: env.BACKUP_RETENTION_DAYS, dir: env.BACKUPS_DIR },
'backup sidecar started',
);
const schedule = scheduleDaily(
env.BACKUP_TIME,
async () => enqueue('scheduled backup', () => guardedRun('scheduled')),
log,
);
const commands = createCommandListener({
createClient: () => new Client({ connectionString: env.DATABASE_URL }),
onCommand: (command) => {
if (command.kind === 'run') {
log.info({ requestedBy: command.requestedBy }, 'manual backup requested');
enqueue('manual backup', () => guardedRun('manual'));
} else {
enqueue('restore', () => handleRestore(command));
}
},
log,
});
await commands.start();
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
process.on(signal, () => {
schedule.stop();
void commands.stop().finally(() => process.exit(0));
});
}