From 5cef359b8fc64b4c4c300ae1f1cdb4e85030a7ae Mon Sep 17 00:00:00 2001 From: Claude Fable 5 Date: Sun, 12 Jul 2026 10:39:18 +0200 Subject: [PATCH] Nextcloud backup target: admin-configured, manual + scheduled uploads, in-app restore (#103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Off-host backups for every self-hoster, configured entirely in the admin UI — supersedes the host-specific mirror plan behind #84. shared: - webdav.ts (new package entry like token-crypto): minimal WebDAV client with basic auth — PROPFIND (tolerant multistatus parser), MKCOL, PUT (streamed), GET, DELETE; Nextcloud DAV path derived from the plain server URL, explicit DAV bases pass through - backup-status.ts: additive remote-upload status in status.json, the restore-status.json contract (running/succeeded/failed + staleness bound), the backup_command/backup_maintenance NOTIFY channels, and the one-bundle-per-set naming (dorfteich-backup-.tar.gz) - backup-set.ts moved here from apps/backup (api lists local sets) backup sidecar: - reads the backup.* instance settings directly from the database (admin changes apply next run; local retention row overrides the env) and the app password from the secret store - after each successful set: bundle dump + files archive + manifest into ONE self-contained tar.gz, upload via WebDAV per schedule (off/daily/weekly; manual runs always upload), prune remote bundles — never the newest — and record the outcome in status.json; upload failures alert via a new backupUploadFailed mail (de+en) - command listener on backup_command (run / restore) with a serial queue against the nightly timer - restore orchestrator: restore-status.json → maintenance NOTIFY → grace → (remote: download + manifest-verify bundle) → terminate other DB connections → shared perform-restore path (same code as restore.sh) → final status + maintenance exit api: - MaintenanceGuard (global, registered before the setup gate): 503 maintenance_mode while restore-status says running; health endpoints and the new public GET /backup/restore-status stay exempt; a stale running state (crashed sidecar) unblocks after 30 min - MaintenanceStateService watches the file and restarts the api after a successful restore (fresh caches, migrate-on-start for older dumps); main.ts refuses to touch the database while a restore runs — a container restarting mid-restore must not race pg_restore with migrate deploy - worker sweeps (conversion, mail outbox, scheduler) catch transient database failures instead of dying on an unhandled rejection — the restore's connection termination crashed the api in verification - backup admin endpoints under /admin/system/backup: settings (live connection test before save, password write-only into the secret store), nextcloud/test, sets (local via the ro backups mount + remote via WebDAV), run + restore (type-to-confirm backstop, source validation) — commands travel as NOTIFY payloads; audit actions backup.settings_changed/run_triggered/restore_requested - readyz: new warning-level backup_remote check while a target is configured (26 h daily / 170 h weekly bound) collab: - maintenance listener: on enter, persist + close every live session and refuse new connections until exit (failsafe timeout 30 min) — no in-memory document may write pre-restore content back afterwards web: - Admin → System backup section: status card with remote facts and a "Back up now" button, the Nextcloud settings form with test button, and the restore picker (local + remote sets, type-to-confirm) - global maintenance screen: any 503 maintenance_mode flips the SPA to a status page polling the exempt endpoint, reloading when the instance returns Verified end-to-end against a live stack (fresh DB, native api + sidecar, fake WebDAV server): configure → test → manual backup → bundle upload → readyz/sets/status surfaces → remote restore with maintenance gate, marker rollback and api restart; suites: shared 21, backup 9, collab 11, api 58 files green, lint + i18n:check + typecheck clean. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1 --- apps/api/src/admin/admin.module.ts | 14 +- apps/api/src/admin/backup-admin.controller.ts | 73 +++ .../api/src/admin/backup-admin.e2e.db.test.ts | 405 ++++++++++++++ apps/api/src/admin/backup-admin.service.ts | 205 +++++++ apps/api/src/admin/system-admin.controller.ts | 2 +- apps/api/src/admin/system-admin.service.ts | 11 +- apps/api/src/app.module.ts | 9 +- .../src/backup/backup-status.controller.ts | 27 + apps/api/src/backup/backup-target.service.ts | 83 +++ apps/api/src/backup/backup.module.ts | 27 + .../src/backup/maintenance-state.service.ts | 116 ++++ apps/api/src/backup/maintenance.guard.ts | 46 ++ apps/api/src/health/backup-freshness.test.ts | 118 +++- apps/api/src/health/backup-freshness.ts | 61 +++ apps/api/src/health/health.controller.ts | 2 + apps/api/src/health/health.module.ts | 2 + apps/api/src/health/readiness.service.ts | 20 +- .../conversion-worker.service.ts | 14 +- apps/api/src/mail/mail-worker.service.ts | 11 +- apps/api/src/main.ts | 36 ++ apps/api/src/scheduler/scheduler.service.ts | 11 +- .../src/settings/instance-settings.service.ts | 19 + apps/backup/package.json | 2 + apps/backup/src/archive.ts | 19 +- apps/backup/src/backup-set.ts | 100 +--- apps/backup/src/commands.ts | 125 +++++ apps/backup/src/index.ts | 175 +++++- apps/backup/src/mail.ts | 78 ++- apps/backup/src/perform-restore.ts | 34 ++ apps/backup/src/remote.test.ts | 273 ++++++++++ apps/backup/src/remote.ts | 252 +++++++++ apps/backup/src/restore-orchestrator.test.ts | 108 ++++ apps/backup/src/restore-orchestrator.ts | 110 ++++ apps/backup/src/restore.ts | 33 +- apps/backup/src/runner.ts | 26 + apps/backup/src/settings.test.ts | 92 ++++ apps/backup/src/settings.ts | 78 +++ apps/backup/src/status.ts | 30 +- apps/collab/src/index.ts | 20 + apps/collab/src/maintenance-listener.test.ts | 76 +++ apps/collab/src/maintenance-listener.ts | 148 +++++ apps/collab/src/server.ts | 13 + apps/web/src/App.tsx | 15 + apps/web/src/lib/api.ts | 14 + apps/web/src/pages/AdminBackupSection.tsx | 504 ++++++++++++++++++ apps/web/src/pages/AdminSystemPage.tsx | 65 +-- apps/web/src/pages/MaintenancePage.tsx | 68 +++ apps/web/src/styles/base.css | 92 ++++ deploy/compose/.env.example | 4 + deploy/monitoring.md | 11 +- docs/architecture/operations.md | 27 +- docs/operations/restore-runbook.md | 25 + docs/self-hosting/README.md | 38 +- packages/shared/i18n/de/errors.json | 8 +- packages/shared/i18n/de/mails.json | 9 + packages/shared/i18n/de/system.json | 69 ++- packages/shared/i18n/en/errors.json | 8 +- packages/shared/i18n/en/mails.json | 9 + packages/shared/i18n/en/system.json | 69 ++- packages/shared/package.json | 12 +- packages/shared/src/backup-set.ts | 91 ++++ packages/shared/src/backup-status.ts | 99 ++++ packages/shared/src/index.ts | 1 + packages/shared/src/system.ts | 87 ++- packages/shared/src/webdav.test.ts | 162 ++++++ packages/shared/src/webdav.ts | 273 ++++++++++ pnpm-lock.yaml | 6 + 67 files changed, 4622 insertions(+), 248 deletions(-) create mode 100644 apps/api/src/admin/backup-admin.controller.ts create mode 100644 apps/api/src/admin/backup-admin.e2e.db.test.ts create mode 100644 apps/api/src/admin/backup-admin.service.ts create mode 100644 apps/api/src/backup/backup-status.controller.ts create mode 100644 apps/api/src/backup/backup-target.service.ts create mode 100644 apps/api/src/backup/backup.module.ts create mode 100644 apps/api/src/backup/maintenance-state.service.ts create mode 100644 apps/api/src/backup/maintenance.guard.ts create mode 100644 apps/backup/src/commands.ts create mode 100644 apps/backup/src/perform-restore.ts create mode 100644 apps/backup/src/remote.test.ts create mode 100644 apps/backup/src/remote.ts create mode 100644 apps/backup/src/restore-orchestrator.test.ts create mode 100644 apps/backup/src/restore-orchestrator.ts create mode 100644 apps/backup/src/settings.test.ts create mode 100644 apps/backup/src/settings.ts create mode 100644 apps/collab/src/maintenance-listener.test.ts create mode 100644 apps/collab/src/maintenance-listener.ts create mode 100644 apps/web/src/pages/AdminBackupSection.tsx create mode 100644 apps/web/src/pages/MaintenancePage.tsx create mode 100644 packages/shared/src/backup-set.ts create mode 100644 packages/shared/src/webdav.test.ts create mode 100644 packages/shared/src/webdav.ts diff --git a/apps/api/src/admin/admin.module.ts b/apps/api/src/admin/admin.module.ts index bcb0115..b82aece 100644 --- a/apps/api/src/admin/admin.module.ts +++ b/apps/api/src/admin/admin.module.ts @@ -1,11 +1,14 @@ import { Module } from '@nestjs/common'; import { AuthModule } from '../auth/auth.module'; +import { BackupModule } from '../backup/backup.module'; import { QuotasModule } from '../quotas/quotas.module'; import { SchedulerModule } from '../scheduler/scheduler.module'; import { UsersModule } from '../users/users.module'; import { AdminSettingsController } from './admin.controller'; +import { BackupAdminController } from './backup-admin.controller'; +import { BackupAdminService } from './backup-admin.service'; import { PseudonymizationService } from './pseudonymization.service'; import { QuotaAdminController } from './quota-admin.controller'; import { SystemAdminController } from './system-admin.controller'; @@ -15,13 +18,20 @@ import { UserAdminController } from './user-admin.controller'; import { UserAdminService } from './user-admin.service'; @Module({ - imports: [QuotasModule, UsersModule, AuthModule, SchedulerModule], + imports: [QuotasModule, UsersModule, AuthModule, SchedulerModule, BackupModule], controllers: [ AdminSettingsController, + BackupAdminController, QuotaAdminController, SystemAdminController, UserAdminController, ], - providers: [QuotaAdminService, SystemAdminService, UserAdminService, PseudonymizationService], + providers: [ + BackupAdminService, + QuotaAdminService, + SystemAdminService, + UserAdminService, + PseudonymizationService, + ], }) export class AdminModule {} diff --git a/apps/api/src/admin/backup-admin.controller.ts b/apps/api/src/admin/backup-admin.controller.ts new file mode 100644 index 0000000..d919110 --- /dev/null +++ b/apps/api/src/admin/backup-admin.controller.ts @@ -0,0 +1,73 @@ +import { Body, Controller, Get, HttpCode, Post, Put, Req, UseGuards } from '@nestjs/common'; +import { + backupConnectionTestInputSchema, + backupRestoreInputSchema, + backupSettingsInputSchema, + type BackupConnectionTestInput, + type BackupConnectionTestResult, + type BackupRestoreInput, + type BackupSetsView, + type BackupSettingsInput, + type BackupSettingsView, +} from '@dorfteich/shared'; + +import { AuthedRequest } from '../auth/auth.guard'; +import { ZodValidationPipe } from '../common/zod-validation.pipe'; +import { SiteAdminGuard } from './site-admin.guard'; +import { BackupAdminService } from './backup-admin.service'; + +/** + * Site-Admin backup management (issue #103): Nextcloud target settings with + * a live connection test, the manual backup trigger, and the in-app restore + * (both answer 202 — the sidecar executes, progress arrives through the + * status files surfaced on GET /admin/system/backup). + */ +@Controller('admin/system/backup') +@UseGuards(SiteAdminGuard) +export class BackupAdminController { + constructor(private readonly backup: BackupAdminService) {} + + @Get('settings') + settings(): Promise { + return this.backup.settingsView(); + } + + @Put('settings') + saveSettings( + @Body(new ZodValidationPipe(backupSettingsInputSchema)) input: BackupSettingsInput, + @Req() request: AuthedRequest, + ): Promise { + return this.backup.saveSettings(input, request.user!); + } + + @Post('nextcloud/test') + @HttpCode(200) + testConnection( + @Body(new ZodValidationPipe(backupConnectionTestInputSchema)) + input: BackupConnectionTestInput, + ): Promise { + return this.backup.testConnection(input); + } + + @Get('sets') + sets(): Promise { + return this.backup.sets(); + } + + @Post('run') + @HttpCode(202) + async run(@Req() request: AuthedRequest): Promise<{ requested: true }> { + await this.backup.requestRun(request.user!); + return { requested: true }; + } + + @Post('restore') + @HttpCode(202) + async restore( + @Body(new ZodValidationPipe(backupRestoreInputSchema)) input: BackupRestoreInput, + @Req() request: AuthedRequest, + ): Promise<{ requested: true }> { + await this.backup.requestRestore(input, request.user!); + return { requested: true }; + } +} diff --git a/apps/api/src/admin/backup-admin.e2e.db.test.ts b/apps/api/src/admin/backup-admin.e2e.db.test.ts new file mode 100644 index 0000000..0e0fa05 --- /dev/null +++ b/apps/api/src/admin/backup-admin.e2e.db.test.ts @@ -0,0 +1,405 @@ +import { createServer, type Server } from 'node:http'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; + +import { INestApplication } from '@nestjs/common'; +import { + BACKUP_COMMAND_CHANNEL, + RESTORE_STATUS_FILE, + archiveFileName, + dumpFileName, + type BackupCommand, + type BackupSetsView, + type BackupSettingsView, + type RestoreStatus, + type SystemBackupView, +} from '@dorfteich/shared'; +import { PrismaClient } from '@prisma/client'; +import { Client } from 'pg'; +import request from 'supertest'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +import { createTestApp, sessionCookieOf } from '../testing/test-app'; +import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db'; +import { UsersService } from '../users/users.service'; + +/** + * A fake Nextcloud endpoint covering the WebDAV subset the admin endpoints + * use (PROPFIND/MKCOL) plus a remote bundle listing — the connection test + * and the restore picker run against real HTTP. + */ +function createDavServer(): { + server: Server; + start(): Promise; + stop(): Promise; + setAuthOk(ok: boolean): void; +} { + let authOk = true; + const files = ['dorfteich-backup-20260710-030000.tar.gz']; + const server = createServer((req, res) => { + if (!authOk) { + res.statusCode = 401; + return res.end(); + } + if (req.method === 'PROPFIND') { + const depth = req.headers.depth; + res.statusCode = 207; + res.setHeader('Content-Type', 'application/xml'); + const children = + depth === '1' + ? files + .map( + (name) => `${req.url}/${name} + 2048 + `, + ) + .join('') + : ''; + return res.end( + ` + ${req.url}/ + + ${children}`, + ); + } + if (req.method === 'MKCOL') { + res.statusCode = 201; + return res.end(); + } + res.statusCode = 405; + res.end(); + }); + return { + server, + setAuthOk: (ok) => { + authOk = ok; + }, + start: () => + new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => { + resolve(`http://127.0.0.1:${(server.address() as { port: number }).port}`); + }); + }), + stop: () => new Promise((resolve) => server.close(() => resolve())), + }; +} + +/** + * Backup administration end to end (issue #103): settings roundtrip with + * the app password landing in the secret store (never the database), the + * live connection test, the restore picker's set listing, command NOTIFYs + * for run/restore, the public restore-status endpoint, and the maintenance + * gate's 503 semantics including staleness. + */ +describe.skipIf(!hasTestDb)('backup admin (e2e, issue #103)', () => { + let app: INestApplication; + let prisma: PrismaClient; + let backupsDir: string; + let secretsFile: string; + let davUrl: string; + const dav = createDavServer(); + const suffix = uniqueSuffix(); + const password = 'backupadmin ist vorsichtig 1'; + const ids: Record = {}; + const cookies: Record = {}; + const baseSecretsFile = process.env.SECRETS_FILE; + + const commands: BackupCommand[] = []; + let listenClient: Client; + + const api = () => request(app.getHttpServer()); + + async function makeUser(handle: string, siteAdmin: boolean): Promise { + const users = app.get(UsersService); + const username = `bak-${handle}-${suffix}`; + const user = await users.createUser({ + username, + email: `${username}@example.org`, + displayName: `Bak ${handle}`, + password, + locale: 'en', + }); + await users.markEmailVerified(user.id); + if (siteAdmin) + await prisma.user.update({ where: { id: user.id }, data: { isSiteAdmin: true } }); + ids[handle] = user.id; + cookies[handle] = sessionCookieOf( + await api() + .post('/api/v1/auth/login') + .send({ usernameOrEmail: username, password }) + .expect(200), + ); + } + + function settingsInput(overrides: Record = {}): Record { + return { + localRetentionDays: 14, + remoteRetentionDays: 60, + nextcloud: { + enabled: true, + baseUrl: davUrl, + username: 'clouduser', + folder: `dorfteich-e2e-${suffix}`, + uploadSchedule: 'weekly', + password: 'app-password-123', + ...((overrides.nextcloud as object) ?? {}), + }, + ...Object.fromEntries(Object.entries(overrides).filter(([k]) => k !== 'nextcloud')), + }; + } + + beforeAll(async () => { + backupsDir = mkdtempSync(join(tmpdir(), 'dorfteich-backup-admin-')); + secretsFile = join(mkdtempSync(join(tmpdir(), 'dorfteich-backup-secrets-')), 'secrets.env'); + process.env.BACKUPS_DIR = backupsDir; + process.env.SECRETS_FILE = secretsFile; + davUrl = await dav.start(); + prisma = createTestPrisma(); + await prisma.rateLimit.deleteMany({}); + // Clean leftovers of earlier runs — the settings keys are singletons. + await prisma.instanceSetting.deleteMany({ where: { key: { startsWith: 'backup.' } } }); + app = await createTestApp(); + await makeUser('admin', true); + await makeUser('user', false); + + listenClient = new Client({ connectionString: process.env.TEST_DATABASE_URL }); + await listenClient.connect(); + listenClient.on('notification', (message) => { + if (message.channel === BACKUP_COMMAND_CHANNEL && message.payload) { + commands.push(JSON.parse(message.payload) as BackupCommand); + } + }); + await listenClient.query(`LISTEN ${BACKUP_COMMAND_CHANNEL}`); + }); + + afterAll(async () => { + delete process.env.BACKUPS_DIR; + if (baseSecretsFile === undefined) delete process.env.SECRETS_FILE; + else process.env.SECRETS_FILE = baseSecretsFile; + await dav.stop(); + await listenClient.end().catch(() => undefined); + const all = Object.values(ids); + await prisma.instanceSetting.deleteMany({ where: { key: { startsWith: 'backup.' } } }); + await prisma.auditEntry.deleteMany({ where: { actorId: { in: all } } }); + await prisma.session.deleteMany({ where: { userId: { in: all } } }); + await prisma.userIdentity.deleteMany({ where: { userId: { in: all } } }); + await prisma.user.deleteMany({ where: { id: { in: all } } }); + await prisma.$disconnect(); + await app.close(); + }); + + it('rejects non-admins on every backup admin route', async () => { + for (const [method, path] of [ + ['get', '/api/v1/admin/system/backup/settings'], + ['put', '/api/v1/admin/system/backup/settings'], + ['post', '/api/v1/admin/system/backup/nextcloud/test'], + ['get', '/api/v1/admin/system/backup/sets'], + ['post', '/api/v1/admin/system/backup/run'], + ['post', '/api/v1/admin/system/backup/restore'], + ] as const) { + await api()[method](path).set('Cookie', cookies.user!).expect(403); + } + }); + + it('tests the connection against a live WebDAV endpoint', async () => { + const ok = await api() + .post('/api/v1/admin/system/backup/nextcloud/test') + .set('Cookie', cookies.admin!) + .send({ + baseUrl: davUrl, + username: 'clouduser', + folder: 'dorfteich-e2e', + password: 'app-password-123', + }) + .expect(200); + expect(ok.body).toEqual({ ok: true }); + + dav.setAuthOk(false); + const bad = await api() + .post('/api/v1/admin/system/backup/nextcloud/test') + .set('Cookie', cookies.admin!) + .send({ + baseUrl: davUrl, + username: 'clouduser', + folder: 'dorfteich-e2e', + password: 'wrong', + }) + .expect(200); + expect(bad.body.ok).toBe(false); + expect(String(bad.body.error)).toContain('authentication failed'); + dav.setAuthOk(true); + }); + + it('refuses to save an enabled target that fails the connection test', async () => { + dav.setAuthOk(false); + const res = await api() + .put('/api/v1/admin/system/backup/settings') + .set('Cookie', cookies.admin!) + .send(settingsInput()) + .expect(400); + expect(res.body.code).toBe('backup_connection_failed'); + dav.setAuthOk(true); + // Nothing was persisted. + const view = await api() + .get('/api/v1/admin/system/backup/settings') + .set('Cookie', cookies.admin!) + .expect(200); + expect((view.body as BackupSettingsView).nextcloud.enabled).toBe(false); + }); + + it('saves settings, keeping the app password out of the database', async () => { + const res = await api() + .put('/api/v1/admin/system/backup/settings') + .set('Cookie', cookies.admin!) + .send(settingsInput()) + .expect(200); + const view = res.body as BackupSettingsView; + expect(view).toMatchObject({ + localRetentionDays: 14, + remoteRetentionDays: 60, + nextcloud: { + enabled: true, + baseUrl: davUrl, + username: 'clouduser', + uploadSchedule: 'weekly', + passwordSet: true, + }, + }); + + // Secret store holds the password; instance_settings never does. + expect(readFileSync(secretsFile, 'utf8')).toMatch( + /BACKUP_NEXTCLOUD_PASSWORD="?app-password-123"?/, + ); + const rows = await prisma.instanceSetting.findMany({ + where: { key: { startsWith: 'backup.' } }, + }); + expect(JSON.stringify(rows.map((row) => row.value))).not.toContain('app-password-123'); + + // Re-saving without a password keeps the stored one (write-only field). + await api() + .put('/api/v1/admin/system/backup/settings') + .set('Cookie', cookies.admin!) + .send(settingsInput({ nextcloud: { password: undefined } })) + .expect(200); + expect(readFileSync(secretsFile, 'utf8')).toMatch( + /BACKUP_NEXTCLOUD_PASSWORD="?app-password-123"?/, + ); + }); + + it('lists local and remote restore sets, and reflects the target on the card', async () => { + writeFileSync(join(backupsDir, dumpFileName('20260712-030000')), 'dump'); + writeFileSync(join(backupsDir, archiveFileName('20260712-030000')), 'archive'); + // An incomplete set never shows up as restorable. + writeFileSync(join(backupsDir, dumpFileName('20260712-040000')), 'dump-only'); + + const res = await api() + .get('/api/v1/admin/system/backup/sets') + .set('Cookie', cookies.admin!) + .expect(200); + const sets = res.body as BackupSetsView; + expect(sets.remoteConfigured).toBe(true); + expect(sets.local.map((s) => s.backupId)).toEqual(['20260712-030000']); + expect(sets.remote.map((s) => s.backupId)).toEqual(['20260710-030000']); + expect(sets.remote[0]!.sizeBytes).toBe(2048); + + const card = await api() + .get('/api/v1/admin/system/backup') + .set('Cookie', cookies.admin!) + .expect(200); + expect((card.body as SystemBackupView).remoteConfigured).toBe(true); + }); + + it('sends run and restore commands over NOTIFY, audit-logged', async () => { + commands.length = 0; + await api().post('/api/v1/admin/system/backup/run').set('Cookie', cookies.admin!).expect(202); + + await api() + .post('/api/v1/admin/system/backup/restore') + .set('Cookie', cookies.admin!) + .send({ source: 'local', backupId: '20260712-030000', confirm: '20260712-030000' }) + .expect(202); + + await sleep(300); + expect(commands).toEqual([ + { kind: 'run', requestedBy: `bak-admin-${suffix}` }, + { + kind: 'restore', + source: 'local', + backupId: '20260712-030000', + requestedBy: `bak-admin-${suffix}`, + }, + ]); + + const audit = await prisma.auditEntry.findMany({ + where: { actorId: ids.admin!, action: { startsWith: 'backup.' } }, + }); + const actions = audit.map((entry) => entry.action); + expect(actions).toContain('backup.run_triggered'); + expect(actions).toContain('backup.restore_requested'); + }); + + it('guards the restore trigger: confirm mismatch, unknown sets, bad sources', async () => { + await api() + .post('/api/v1/admin/system/backup/restore') + .set('Cookie', cookies.admin!) + .send({ source: 'local', backupId: '20260712-030000', confirm: 'nope' }) + .expect(400); + + await api() + .post('/api/v1/admin/system/backup/restore') + .set('Cookie', cookies.admin!) + .send({ source: 'local', backupId: '20260712-040000', confirm: '20260712-040000' }) + .expect(404); + + await api() + .post('/api/v1/admin/system/backup/restore') + .set('Cookie', cookies.admin!) + .send({ source: 'remote', backupId: '20260712-050000', confirm: '20260712-050000' }) + .expect(404); + }); + + it('serves the public restore status and gates the api during a restore', async () => { + // No restore yet → idle, and the api serves normally. + const idle = await api().get('/api/v1/backup/restore-status').expect(200); + expect(idle.body).toEqual({ state: 'idle' }); + + const running: RestoreStatus = { + schemaVersion: 1, + state: 'running', + backupId: '20260712-030000', + source: 'local', + requestedBy: 'admin', + startedAt: new Date().toISOString(), + finishedAt: null, + }; + writeFileSync(join(backupsDir, RESTORE_STATUS_FILE), JSON.stringify(running)); + await sleep(1600); // maintenance state cache TTL + + // Anonymous status endpoint keeps answering; everything else 503s. + const status = await api().get('/api/v1/backup/restore-status').expect(200); + expect((status.body as RestoreStatus).state).toBe('running'); + const gated = await api().get('/api/v1/ponds').set('Cookie', cookies.admin!).expect(503); + expect(gated.body.code).toBe('maintenance_mode'); + await api().get('/api/v1/healthz').expect(200); + + // A crashed restore (stale running state) must not brick the instance. + writeFileSync( + join(backupsDir, RESTORE_STATUS_FILE), + JSON.stringify({ ...running, startedAt: new Date(Date.now() - 31 * 60_000).toISOString() }), + ); + await sleep(1600); + await api().get('/api/v1/ponds').set('Cookie', cookies.admin!).expect(200); + + // A finished restore leaves the gate open and reports its result. + writeFileSync( + join(backupsDir, RESTORE_STATUS_FILE), + JSON.stringify({ ...running, state: 'succeeded', finishedAt: new Date().toISOString() }), + ); + await sleep(1600); + const done = await api().get('/api/v1/backup/restore-status').expect(200); + expect((done.body as RestoreStatus).state).toBe('succeeded'); + await api().get('/api/v1/ponds').set('Cookie', cookies.admin!).expect(200); + }); +}); diff --git a/apps/api/src/admin/backup-admin.service.ts b/apps/api/src/admin/backup-admin.service.ts new file mode 100644 index 0000000..cb4b062 --- /dev/null +++ b/apps/api/src/admin/backup-admin.service.ts @@ -0,0 +1,205 @@ +import { readdirSync } from 'node:fs'; +import { statSync } from 'node:fs'; +import { join } from 'node:path'; + +import { + BadRequestException, + ConflictException, + Injectable, + NotFoundException, +} from '@nestjs/common'; +import { + BACKUP_COMMAND_CHANNEL, + archiveFileName, + backupIdTime, + dumpFileName, + listSets, + remoteBundleId, + type BackupCommand, + type BackupConnectionTestInput, + type BackupConnectionTestResult, + type BackupRestoreInput, + type BackupSetView, + type BackupSetsView, + type BackupSettingsInput, + type BackupSettingsView, +} from '@dorfteich/shared'; +import { webdavList } from '@dorfteich/shared/webdav'; +import { User } from '@prisma/client'; + +import { AuditService } from '../audit/audit.service'; +import { BackupTargetService } from '../backup/backup-target.service'; +import { MaintenanceStateService } from '../backup/maintenance-state.service'; +import { AppConfig } from '../config/app-config.service'; +import { PrismaService } from '../prisma/prisma.service'; +import { InstanceSettingsService } from '../settings/instance-settings.service'; + +/** + * Site-Admin backup management (issue #103): the Nextcloud target + * configuration, manual "back up now", the restore picker (local sets from + * the read-only backups mount, remote sets via WebDAV), and the restore + * trigger. The sidecar does the actual work — commands travel over the + * {@link BACKUP_COMMAND_CHANNEL} NOTIFY bus, progress comes back through + * `status.json`/`restore-status.json`. + */ +@Injectable() +export class BackupAdminService { + constructor( + private readonly target: BackupTargetService, + private readonly maintenance: MaintenanceStateService, + private readonly settings: InstanceSettingsService, + private readonly prisma: PrismaService, + private readonly audit: AuditService, + private readonly config: AppConfig, + ) {} + + settingsView(): Promise { + return this.target.settingsView(); + } + + /** + * Persists the backup settings. When the Nextcloud side is enabled, the + * connection is live-tested first (with the new password if provided, + * else the stored one) — like the setup wizard's SMTP step, nothing is + * saved on failure. + */ + async saveSettings(input: BackupSettingsInput, actor: User): Promise { + if (input.nextcloud.enabled) { + const test = await this.target.testConnection({ + baseUrl: input.nextcloud.baseUrl, + username: input.nextcloud.username, + folder: input.nextcloud.folder, + password: input.nextcloud.password, + }); + if (!test.ok) { + throw new BadRequestException({ + code: 'backup_connection_failed', + details: { nextcloud: [test.error ?? 'connection failed'] }, + }); + } + } + await this.target.storePassword(input.nextcloud.password ?? ''); + await this.settings.set('backup.localRetentionDays', input.localRetentionDays, actor.id); + await this.settings.set('backup.remoteRetentionDays', input.remoteRetentionDays, actor.id); + await this.settings.set('backup.nextcloud.enabled', input.nextcloud.enabled, actor.id); + await this.settings.set('backup.nextcloud.baseUrl', input.nextcloud.baseUrl, actor.id); + await this.settings.set('backup.nextcloud.username', input.nextcloud.username, actor.id); + await this.settings.set('backup.nextcloud.folder', input.nextcloud.folder, actor.id); + await this.settings.set( + 'backup.nextcloud.uploadSchedule', + input.nextcloud.uploadSchedule, + actor.id, + ); + // settings.set audits each key; one summary entry names the intent. + await this.audit.record({ + action: 'backup.settings_changed', + actorId: actor.id, + details: { nextcloudEnabled: input.nextcloud.enabled }, + }); + return this.settingsView(); + } + + testConnection(input: BackupConnectionTestInput): Promise { + return this.target.testConnection(input); + } + + /** Both restore sources for the picker: newest first. */ + async sets(): Promise { + const local = this.localSets(); + const target = await this.target.resolveTarget(); + if (!target) return { local, remoteConfigured: false, remote: [] }; + + const listed = await webdavList(target); + if (!listed.ok) { + return { local, remoteConfigured: true, remote: [], remoteError: listed.error }; + } + const remote = listed.value + .filter((entry) => !entry.isCollection) + .map((entry) => ({ id: remoteBundleId(entry.name), sizeBytes: entry.sizeBytes })) + .filter((entry): entry is { id: string; sizeBytes: number | null } => entry.id !== null) + .map((entry) => this.setView(entry.id, entry.sizeBytes)) + .sort((a, b) => b.backupId.localeCompare(a.backupId)); + return { local, remoteConfigured: true, remote }; + } + + /** "Back up now": dump + upload, executed by the sidecar (202-style). */ + async requestRun(actor: User): Promise { + await this.notify({ kind: 'run', requestedBy: actor.username }); + await this.audit.record({ action: 'backup.run_triggered', actorId: actor.id }); + } + + /** + * Requests an in-app restore. The type-to-confirm value must repeat the + * backup id — the UI enforces it too, this is the server-side backstop + * for the most destructive action the instance has. + */ + async requestRestore(input: BackupRestoreInput, actor: User): Promise { + if (input.confirm !== input.backupId) { + throw new BadRequestException({ code: 'backup_restore_confirm_mismatch' }); + } + if (this.maintenance.current()?.state === 'running' && this.maintenance.isActive()) { + throw new ConflictException({ code: 'backup_restore_running' }); + } + if (input.source === 'remote') { + const target = await this.target.resolveTarget(); + if (!target) throw new BadRequestException({ code: 'backup_remote_not_configured' }); + const listed = await webdavList(target); + const exists = + listed.ok && listed.value.some((entry) => remoteBundleId(entry.name) === input.backupId); + if (!exists) throw new NotFoundException({ code: 'backup_set_not_found' }); + } else { + const complete = this.localSets().some((set) => set.backupId === input.backupId); + if (!complete) throw new NotFoundException({ code: 'backup_set_not_found' }); + } + await this.notify({ + kind: 'restore', + source: input.source, + backupId: input.backupId, + requestedBy: actor.username, + }); + await this.audit.record({ + action: 'backup.restore_requested', + actorId: actor.id, + targetType: 'backup', + targetId: input.backupId, + details: { source: input.source }, + }); + } + + private async notify(command: BackupCommand): Promise { + await this.prisma + .$executeRaw`SELECT pg_notify(${BACKUP_COMMAND_CHANNEL}, ${JSON.stringify(command)})`; + } + + /** Complete sets on the read-only backups mount, newest first. */ + private localSets(): BackupSetView[] { + let names: string[]; + try { + names = readdirSync(this.config.env.BACKUPS_DIR); + } catch { + return []; + } + return listSets(names) + .filter((set) => set.complete) + .map((set) => { + let sizeBytes: number | null = 0; + for (const file of [dumpFileName(set.id), archiveFileName(set.id)]) { + try { + sizeBytes = (sizeBytes ?? 0) + statSync(join(this.config.env.BACKUPS_DIR, file)).size; + } catch { + sizeBytes = null; + } + } + return this.setView(set.id, sizeBytes); + }) + .sort((a, b) => b.backupId.localeCompare(a.backupId)); + } + + private setView(backupId: string, sizeBytes: number | null): BackupSetView { + return { + backupId, + startedAt: backupIdTime(backupId)?.toISOString() ?? new Date(0).toISOString(), + sizeBytes, + }; + } +} diff --git a/apps/api/src/admin/system-admin.controller.ts b/apps/api/src/admin/system-admin.controller.ts index c31c88d..09e40ce 100644 --- a/apps/api/src/admin/system-admin.controller.ts +++ b/apps/api/src/admin/system-admin.controller.ts @@ -34,7 +34,7 @@ export class SystemAdminController { } @Get('backup') - backup(): SystemBackupView { + backup(): Promise { return this.system.backup(); } diff --git a/apps/api/src/admin/system-admin.service.ts b/apps/api/src/admin/system-admin.service.ts index d7032aa..95b55b8 100644 --- a/apps/api/src/admin/system-admin.service.ts +++ b/apps/api/src/admin/system-admin.service.ts @@ -17,6 +17,8 @@ import { } from '@dorfteich/shared'; import { AuditService } from '../audit/audit.service'; +import { BackupTargetService } from '../backup/backup-target.service'; +import { MaintenanceStateService } from '../backup/maintenance-state.service'; import { AppConfig } from '../config/app-config.service'; import { PrismaService } from '../prisma/prisma.service'; import { SchedulerService } from '../scheduler/scheduler.service'; @@ -35,6 +37,8 @@ export class SystemAdminService { private readonly scheduler: SchedulerService, private readonly audit: AuditService, private readonly config: AppConfig, + private readonly backupTarget: BackupTargetService, + private readonly maintenance: MaintenanceStateService, ) {} /** @@ -89,8 +93,9 @@ export class SystemAdminService { return { outcome, job }; } - /** The backup card mirrors status.json including the freshness verdict. */ - backup(): SystemBackupView { + /** The backup card mirrors status.json including the freshness verdict, + * plus the off-host target state and restore progress (issue #103). */ + async backup(): Promise { const path = join(this.config.env.BACKUPS_DIR, BACKUP_STATUS_FILE); let status: BackupStatus | null = null; if (existsSync(path)) { @@ -109,6 +114,8 @@ export class SystemAdminService { fresh: Number.isFinite(ageHours) && ageHours <= BACKUP_FRESH_MAX_AGE_HOURS, status, maxAgeHours: BACKUP_FRESH_MAX_AGE_HOURS, + remoteConfigured: (await this.backupTarget.resolveTarget()) !== null, + restore: this.maintenance.current(), }; } diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 2a9be17..c6b0b1a 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -5,6 +5,7 @@ import { LoggerModule } from 'nestjs-pino'; import { AdminModule } from './admin/admin.module'; import { AuditModule } from './audit/audit.module'; import { AuthModule } from './auth/auth.module'; +import { BackupModule } from './backup/backup.module'; import { ApiExceptionFilter } from './common/api-exception.filter'; import { CommentsModule } from './comments/comments.module'; import { CompactionModule } from './compaction/compaction.module'; @@ -43,8 +44,12 @@ import { VersionsModule } from './versions/versions.module'; RateLimitModule, MailModule, SettingsModule, - // Before AuthModule: global guards run in registration order, and the - // setup gate must win over AuthGuard's 401 while setup is pending. + // Before SetupModule and AuthModule: global guards run in registration + // order, and the maintenance gate (in-app restore, issue #103) must + // answer before anything touches the mid-restore database. + BackupModule, + // Before AuthModule: the setup gate must win over AuthGuard's 401 while + // setup is pending. SetupModule, UsersModule, PermissionsModule, diff --git a/apps/api/src/backup/backup-status.controller.ts b/apps/api/src/backup/backup-status.controller.ts new file mode 100644 index 0000000..118b634 --- /dev/null +++ b/apps/api/src/backup/backup-status.controller.ts @@ -0,0 +1,27 @@ +import { Controller, Get } from '@nestjs/common'; +import type { RestoreStatusResponse } from '@dorfteich/shared'; + +import { Public } from '../auth/auth.guard'; +import { SetupExempt } from '../setup/setup.guard'; +import { MaintenanceExempt } from './maintenance.guard'; +import { MaintenanceStateService } from './maintenance-state.service'; + +/** + * The status page behind maintenance mode (issue #103): while an in-app + * restore runs, this is the one application endpoint that keeps answering — + * the SPA's maintenance screen polls it to show progress and to know when + * to reload. Public: everyone hitting the instance mid-restore deserves the + * honest answer, and the status carries nothing sensitive. + */ +@Controller('backup') +export class BackupStatusController { + constructor(private readonly state: MaintenanceStateService) {} + + @Get('restore-status') + @Public() + @SetupExempt() + @MaintenanceExempt() + restoreStatus(): RestoreStatusResponse { + return this.state.current() ?? { state: 'idle' }; + } +} diff --git a/apps/api/src/backup/backup-target.service.ts b/apps/api/src/backup/backup-target.service.ts new file mode 100644 index 0000000..8d5f3fb --- /dev/null +++ b/apps/api/src/backup/backup-target.service.ts @@ -0,0 +1,83 @@ +import { Injectable } from '@nestjs/common'; +import type { BackupConnectionTestResult, BackupSettingsView } from '@dorfteich/shared'; +import { webdavCheck, type WebDavTarget } from '@dorfteich/shared/webdav'; + +import { SecretStoreService } from '../config/secret-store.service'; +import { InstanceSettingsService } from '../settings/instance-settings.service'; + +/** + * The api's view of the Nextcloud backup target (issue #103): instance + * settings hold everything non-secret, the app password lives in the + * wizard-written secret store (security.md §Secrets) under the key the + * backup sidecar reads. This service resolves, tests, and persists the + * combination — it never returns the password. + */ +export const NEXTCLOUD_PASSWORD_SECRET_KEY = 'BACKUP_NEXTCLOUD_PASSWORD'; + +@Injectable() +export class BackupTargetService { + constructor( + private readonly settings: InstanceSettingsService, + private readonly secretStore: SecretStoreService, + ) {} + + async settingsView(): Promise { + return { + localRetentionDays: await this.settings.get('backup.localRetentionDays'), + remoteRetentionDays: await this.settings.get('backup.remoteRetentionDays'), + nextcloud: { + enabled: await this.settings.get('backup.nextcloud.enabled'), + baseUrl: await this.settings.get('backup.nextcloud.baseUrl'), + username: await this.settings.get('backup.nextcloud.username'), + folder: await this.settings.get('backup.nextcloud.folder'), + uploadSchedule: await this.settings.get('backup.nextcloud.uploadSchedule'), + passwordSet: Boolean(this.storedPassword()), + }, + }; + } + + /** + * The effective WebDAV target, or null when disabled or not fully + * configured — the exact resolution the sidecar applies on its side. + */ + async resolveTarget(): Promise { + const view = await this.settingsView(); + const password = this.storedPassword(); + const { enabled, baseUrl, username, folder } = view.nextcloud; + if (!enabled || !baseUrl || !username || !password) return null; + return { baseUrl, username, password, folder }; + } + + /** + * Live connection test (credentials + folder, creating missing folder + * segments) for the admin "test connection" button. An empty password + * falls back to the stored one, so a saved configuration can be re-tested + * without re-entering the secret. + */ + async testConnection(candidate: { + baseUrl: string; + username: string; + folder: string; + password?: string; + }): Promise { + const password = candidate.password || this.storedPassword(); + if (!password) return { ok: false, error: 'no app password provided or stored' }; + const result = await webdavCheck({ + baseUrl: candidate.baseUrl, + username: candidate.username, + folder: candidate.folder, + password, + }); + return result.ok ? { ok: true } : { ok: false, error: result.error }; + } + + /** Stores a new app password; empty input keeps the current one. */ + async storePassword(password: string): Promise { + if (!password) return; + await this.secretStore.set({ [NEXTCLOUD_PASSWORD_SECRET_KEY]: password }); + } + + storedPassword(): string { + return this.secretStore.read()[NEXTCLOUD_PASSWORD_SECRET_KEY] ?? ''; + } +} diff --git a/apps/api/src/backup/backup.module.ts b/apps/api/src/backup/backup.module.ts new file mode 100644 index 0000000..7113eaa --- /dev/null +++ b/apps/api/src/backup/backup.module.ts @@ -0,0 +1,27 @@ +import { Module } from '@nestjs/common'; +import { APP_GUARD } from '@nestjs/core'; + +import { SettingsModule } from '../settings/settings.module'; +import { BackupStatusController } from './backup-status.controller'; +import { BackupTargetService } from './backup-target.service'; +import { MaintenanceGuard } from './maintenance.guard'; +import { MaintenanceStateService } from './maintenance-state.service'; + +/** + * Backup/restore integration of the api (issue #103): the maintenance gate + * around in-app restores plus the Nextcloud target resolution. Imported in + * AppModule BEFORE SetupModule on purpose — global guards run in + * registration order, and mid-restore nothing (not even the setup gate's + * database read) should touch the database. + */ +@Module({ + imports: [SettingsModule], + controllers: [BackupStatusController], + providers: [ + BackupTargetService, + MaintenanceStateService, + { provide: APP_GUARD, useClass: MaintenanceGuard }, + ], + exports: [BackupTargetService, MaintenanceStateService], +}) +export class BackupModule {} diff --git a/apps/api/src/backup/maintenance-state.service.ts b/apps/api/src/backup/maintenance-state.service.ts new file mode 100644 index 0000000..b4d95af --- /dev/null +++ b/apps/api/src/backup/maintenance-state.service.ts @@ -0,0 +1,116 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { + RESTORE_STALE_MAX_AGE_MINUTES, + RESTORE_STATUS_FILE, + type RestoreStatus, +} from '@dorfteich/shared'; +import { PinoLogger } from 'nestjs-pino'; + +import { AppConfig } from '../config/app-config.service'; + +const CACHE_TTL_MS = 1500; +const WATCH_INTERVAL_MS = 2000; + +/** + * Mirrors the sidecar's `restore-status.json` (issue #103): while a restore + * is `running` the maintenance guard answers 503, and once it flips to + * `succeeded` this service restarts the api — the restored database + * invalidates every in-process cache (settings, permissions, setup state), + * and a fresh boot also runs `migrate deploy` for sets from older versions. + * Docker's `unless-stopped` policy brings the container back up. + * + * A `running` state older than {@link RESTORE_STALE_MAX_AGE_MINUTES} counts + * as crashed (the sidecar died before writing a final state), so the + * instance never stays bricked behind the gate. + */ +@Injectable() +export class MaintenanceStateService implements OnModuleInit, OnModuleDestroy { + private cached: { status: RestoreStatus | null; readAt: number } | null = null; + private watcher: NodeJS.Timeout | null = null; + private sawRunning = false; + private restartScheduled = false; + + constructor( + private readonly config: AppConfig, + private readonly logger: PinoLogger, + ) { + this.logger.setContext(MaintenanceStateService.name); + } + + onModuleInit(): void { + if (this.config.env.NODE_ENV === 'test') return; + // Poll independently of traffic: the restart must also happen when no + // request arrives while the instance sits in maintenance. + this.watcher = setInterval(() => this.observe(), WATCH_INTERVAL_MS); + this.watcher.unref?.(); + } + + onModuleDestroy(): void { + if (this.watcher) clearInterval(this.watcher); + } + + /** The current restore status (short cache — the guard reads per request). */ + current(): RestoreStatus | null { + const now = Date.now(); + if (this.cached && now - this.cached.readAt < CACHE_TTL_MS) return this.cached.status; + const status = this.read(); + this.cached = { status, readAt: now }; + return status; + } + + /** Whether the instance is in maintenance (a fresh, running restore). */ + isActive(): boolean { + const status = this.current(); + if (!status || status.state !== 'running') return false; + const ageMinutes = (Date.now() - new Date(status.startedAt).getTime()) / 60_000; + if (!Number.isFinite(ageMinutes) || ageMinutes > RESTORE_STALE_MAX_AGE_MINUTES) { + return false; + } + return true; + } + + private observe(): void { + this.cached = null; + if (this.isActive()) { + if (!this.sawRunning) { + this.sawRunning = true; + this.logger.warn({}, 'restore running — maintenance mode active'); + } + return; + } + if (!this.sawRunning) return; + const status = this.current(); + this.sawRunning = false; + if (status?.state === 'succeeded' && !this.restartScheduled) { + this.restartScheduled = true; + this.logger.warn( + { backupId: status.backupId }, + 'restore succeeded — restarting the api for a clean boot on the restored database', + ); + if (this.config.env.NODE_ENV === 'production') { + // A short delay lets the log line flush; docker restarts the container. + setTimeout(() => process.exit(0), 1000).unref?.(); + } else { + this.logger.warn({}, 'non-production api: restart the api process manually now'); + } + } else if (status?.state === 'failed') { + this.logger.error( + { backupId: status.backupId, error: status.error }, + 'restore failed — instance left maintenance mode without restoring', + ); + } + } + + private read(): RestoreStatus | null { + const path = join(this.config.env.BACKUPS_DIR, RESTORE_STATUS_FILE); + if (!existsSync(path)) return null; + try { + return JSON.parse(readFileSync(path, 'utf8')) as RestoreStatus; + } catch { + return null; + } + } +} diff --git a/apps/api/src/backup/maintenance.guard.ts b/apps/api/src/backup/maintenance.guard.ts new file mode 100644 index 0000000..07d86ef --- /dev/null +++ b/apps/api/src/backup/maintenance.guard.ts @@ -0,0 +1,46 @@ +import { + CanActivate, + ExecutionContext, + Injectable, + ServiceUnavailableException, + SetMetadata, +} from '@nestjs/common'; +import { Reflector } from '@nestjs/core'; + +import { MaintenanceStateService } from './maintenance-state.service'; + +const MAINTENANCE_EXEMPT_KEY = 'maintenanceExempt'; + +/** + * Marks routes that stay reachable while an in-app restore runs: the health + * probes (monitors must keep seeing the instance) and the restore status + * endpoint the maintenance screen polls. + */ +export const MaintenanceExempt = (): MethodDecorator & ClassDecorator => + SetMetadata(MAINTENANCE_EXEMPT_KEY, true); + +/** + * Global first-line guard (registered before SetupModule and AuthModule via + * module order): while the backup sidecar restores the database, every + * non-exempt route answers 503 `maintenance_mode` (issue #103) — nothing + * may read or write mid-restore state. + */ +@Injectable() +export class MaintenanceGuard implements CanActivate { + constructor( + private readonly reflector: Reflector, + private readonly state: MaintenanceStateService, + ) {} + + canActivate(context: ExecutionContext): boolean { + const exempt = this.reflector.getAllAndOverride(MAINTENANCE_EXEMPT_KEY, [ + context.getHandler(), + context.getClass(), + ]); + if (exempt) return true; + if (this.state.isActive()) { + throw new ServiceUnavailableException({ code: 'maintenance_mode' }); + } + return true; + } +} diff --git a/apps/api/src/health/backup-freshness.test.ts b/apps/api/src/health/backup-freshness.test.ts index 8b1b806..9539364 100644 --- a/apps/api/src/health/backup-freshness.test.ts +++ b/apps/api/src/health/backup-freshness.test.ts @@ -5,11 +5,13 @@ import { join } from 'node:path'; import { BACKUP_STATUS_FILE, type BackupStatus } from '@dorfteich/shared'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { backupFreshnessCheck } from './backup-freshness'; +import { backupFreshnessCheck, backupRemoteFreshnessCheck } from './backup-freshness'; import { ReadinessService } from './readiness.service'; +import type { BackupTargetService } from '../backup/backup-target.service'; import type { AppConfig } from '../config/app-config.service'; import type { PrismaService } from '../prisma/prisma.service'; +import type { InstanceSettingsService } from '../settings/instance-settings.service'; const NOW = new Date('2026-07-12T12:00:00Z'); const HOUR = 3_600_000; @@ -122,6 +124,17 @@ describe('ReadinessService report (issue #85 degraded semantics)', () => { }, } as unknown as PrismaService; + const noTarget = { resolveTarget: async () => null } as unknown as BackupTargetService; + const configuredTarget = { + resolveTarget: async () => ({ + baseUrl: 'https://cloud.example.com', + username: 'u', + password: 'p', + folder: 'f', + }), + } as unknown as BackupTargetService; + const dailySettings = { get: async () => 'daily' } as unknown as InstanceSettingsService; + function configWith(backupsDir: string): AppConfig { return { env: { @@ -143,7 +156,12 @@ describe('ReadinessService report (issue #85 degraded semantics)', () => { sizes: { dumpBytes: 1, archiveBytes: 1 }, }, }); - const report = await new ReadinessService(prismaUp, configWith(dir)).report(); + const report = await new ReadinessService( + prismaUp, + configWith(dir), + noTarget, + dailySettings, + ).report(); expect(report.status).toBe('degraded'); const byName = Object.fromEntries(report.checks.map((c) => [c.name, c.status])); @@ -151,8 +169,102 @@ describe('ReadinessService report (issue #85 degraded semantics)', () => { }); it('reports unready only on hard failures', async () => { - const report = await new ReadinessService(prismaDown, configWith(dir)).report(); + const report = await new ReadinessService( + prismaDown, + configWith(dir), + noTarget, + dailySettings, + ).report(); expect(report.status).toBe('unready'); expect(report.checks.find((c) => c.name === 'database')?.status).toBe('failed'); }); + + it('adds the off-host check only while a target is configured', async () => { + writeStatus({}); + const withoutTarget = await new ReadinessService( + prismaUp, + configWith(dir), + noTarget, + dailySettings, + ).report(); + expect(withoutTarget.checks.some((c) => c.name === 'backup_remote')).toBe(false); + + const withTarget = await new ReadinessService( + prismaUp, + configWith(dir), + configuredTarget, + dailySettings, + ).report(); + const remote = withTarget.checks.find((c) => c.name === 'backup_remote'); + // status.json has no remote section yet → the copy is missing → warn. + expect(remote).toMatchObject({ status: 'warn' }); + expect(withTarget.status).toBe('degraded'); + }); +}); + +describe('backupRemoteFreshnessCheck (issue #103)', () => { + const fresh = new Date(NOW.getTime() - 3 * HOUR).toISOString(); + + it('is ok (with detail) when the schedule is manual-only', () => { + expect(backupRemoteFreshnessCheck(dir, NOW, 'off')).toMatchObject({ + name: 'backup_remote', + status: 'ok', + }); + }); + + it('warns when no upload ever succeeded, with the last error', () => { + writeStatus({ + remote: { + lastUpload: { + backupId: '20260712-090000', + finishedAt: fresh, + outcome: 'failed', + error: 'authentication failed', + }, + lastSuccessfulUpload: null, + }, + }); + const check = backupRemoteFreshnessCheck(dir, NOW, 'daily'); + expect(check.status).toBe('warn'); + expect(check.detail).toContain('authentication failed'); + }); + + it('applies the weekly bound to weekly schedules', () => { + const sixDaysOld = new Date(NOW.getTime() - 6 * 24 * HOUR).toISOString(); + writeStatus({ + remote: { + lastUpload: { + backupId: '20260706-090000', + finishedAt: sixDaysOld, + outcome: 'succeeded', + sizeBytes: 5, + }, + lastSuccessfulUpload: { + backupId: '20260706-090000', + finishedAt: sixDaysOld, + sizeBytes: 5, + }, + }, + }); + expect(backupRemoteFreshnessCheck(dir, NOW, 'weekly').status).toBe('ok'); + expect(backupRemoteFreshnessCheck(dir, NOW, 'daily').status).toBe('warn'); + }); + + it('is ok on a fresh upload', () => { + writeStatus({ + remote: { + lastUpload: { + backupId: '20260712-090000', + finishedAt: fresh, + outcome: 'succeeded', + sizeBytes: 5, + }, + lastSuccessfulUpload: { backupId: '20260712-090000', finishedAt: fresh, sizeBytes: 5 }, + }, + }); + expect(backupRemoteFreshnessCheck(dir, NOW, 'daily')).toEqual({ + name: 'backup_remote', + status: 'ok', + }); + }); }); diff --git a/apps/api/src/health/backup-freshness.ts b/apps/api/src/health/backup-freshness.ts index 1cc4cd0..fc54d7c 100644 --- a/apps/api/src/health/backup-freshness.ts +++ b/apps/api/src/health/backup-freshness.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import { BACKUP_FRESH_MAX_AGE_HOURS, + BACKUP_REMOTE_WEEKLY_MAX_AGE_HOURS, BACKUP_STATUS_FILE, type BackupStatus, } from '@dorfteich/shared'; @@ -58,3 +59,63 @@ export function backupFreshnessCheck(backupsDir: string, now: Date): ReadinessCh } return { name: 'backup', status: 'ok' }; } + +/** + * Off-host copy freshness (issue #103) — evaluated only while a Nextcloud + * target is configured. Like the local check it is always warning-level. + * With schedule `off` (manual uploads only) there is no cadence to hold the + * instance to, so the check reports ok with a detail. + */ +export function backupRemoteFreshnessCheck( + backupsDir: string, + now: Date, + schedule: 'off' | 'daily' | 'weekly', +): ReadinessCheck { + const name = 'backup_remote'; + if (schedule === 'off') { + return { name, status: 'ok', detail: 'manual uploads only (schedule off)' }; + } + const maxAgeHours = + schedule === 'weekly' ? BACKUP_REMOTE_WEEKLY_MAX_AGE_HOURS : BACKUP_FRESH_MAX_AGE_HOURS; + + const path = join(backupsDir, BACKUP_STATUS_FILE); + let status: BackupStatus | null = null; + if (existsSync(path)) { + try { + status = JSON.parse(readFileSync(path, 'utf8')) as BackupStatus; + } catch { + status = null; + } + } + if (!status) { + return { name, status: 'warn', detail: 'no backup status recorded yet' }; + } + if (!status.remote?.lastSuccessfulUpload) { + const error = status.remote?.lastUpload?.error; + return { + name, + status: 'warn', + detail: error + ? `no successful off-host upload yet — last attempt: ${error}` + : 'no successful off-host upload yet', + }; + } + + const upload = status.remote.lastSuccessfulUpload; + const ageHours = (now.getTime() - new Date(upload.finishedAt).getTime()) / 3_600_000; + if (!Number.isFinite(ageHours) || ageHours > maxAgeHours) { + return { + name, + status: 'warn', + detail: `last off-host copy ${upload.backupId} is ${Math.round(ageHours)} h old (max ${maxAgeHours} h)`, + }; + } + if (status.remote.lastUpload.outcome === 'failed') { + return { + name, + status: 'ok', + detail: `fresh, but the last upload failed: ${status.remote.lastUpload.error ?? 'unknown error'}`, + }; + } + return { name, status: 'ok' }; +} diff --git a/apps/api/src/health/health.controller.ts b/apps/api/src/health/health.controller.ts index 045c9dc..f5619ef 100644 --- a/apps/api/src/health/health.controller.ts +++ b/apps/api/src/health/health.controller.ts @@ -3,12 +3,14 @@ import { HealthResponse, healthResponse } from '@dorfteich/shared'; import type { Response } from 'express'; import { Public } from '../auth/auth.guard'; +import { MaintenanceExempt } from '../backup/maintenance.guard'; import { AppConfig } from '../config/app-config.service'; import { SetupExempt } from '../setup/setup.guard'; import { ReadinessService } from './readiness.service'; @Public() @SetupExempt() // deploys and monitors must see health during first-run setup +@MaintenanceExempt() // …and during an in-app restore (issue #103) @Controller() export class HealthController { constructor( diff --git a/apps/api/src/health/health.module.ts b/apps/api/src/health/health.module.ts index c878991..e1880a7 100644 --- a/apps/api/src/health/health.module.ts +++ b/apps/api/src/health/health.module.ts @@ -1,9 +1,11 @@ import { Module } from '@nestjs/common'; +import { BackupModule } from '../backup/backup.module'; import { HealthController } from './health.controller'; import { ReadinessService } from './readiness.service'; @Module({ + imports: [BackupModule], controllers: [HealthController], providers: [ReadinessService], }) diff --git a/apps/api/src/health/readiness.service.ts b/apps/api/src/health/readiness.service.ts index 6d8939f..0b174a4 100644 --- a/apps/api/src/health/readiness.service.ts +++ b/apps/api/src/health/readiness.service.ts @@ -1,8 +1,10 @@ import { Injectable } from '@nestjs/common'; +import { BackupTargetService } from '../backup/backup-target.service'; import { AppConfig } from '../config/app-config.service'; import { PrismaService } from '../prisma/prisma.service'; -import { backupFreshnessCheck } from './backup-freshness'; +import { InstanceSettingsService } from '../settings/instance-settings.service'; +import { backupFreshnessCheck, backupRemoteFreshnessCheck } from './backup-freshness'; export interface ReadinessCheck { /** `warn` reports a degraded-but-serving dependency: the instance still @@ -33,6 +35,8 @@ export class ReadinessService { constructor( private readonly prisma: PrismaService, private readonly config: AppConfig, + private readonly backupTarget: BackupTargetService, + private readonly settings: InstanceSettingsService, ) {} /** @@ -49,6 +53,7 @@ export class ReadinessService { await this.converterReachable(), await this.rendererReachable(), backupFreshnessCheck(this.config.env.BACKUPS_DIR, new Date()), + ...(await this.remoteBackupCheck()), ]; const status = checks.some((c) => c.status === 'failed') ? 'unready' @@ -58,6 +63,19 @@ export class ReadinessService { return { status, checks }; } + /** Off-host copy freshness (issue #103) — only while a target is + * configured; the check itself never talks to the network, it reads the + * sidecar's status.json. A database hiccup here must not break readyz. */ + private async remoteBackupCheck(): Promise { + try { + if (!(await this.backupTarget.resolveTarget())) return []; + const schedule = await this.settings.get('backup.nextcloud.uploadSchedule'); + return [backupRemoteFreshnessCheck(this.config.env.BACKUPS_DIR, new Date(), schedule)]; + } catch { + return []; + } + } + private async converterReachable(): Promise { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), CONVERTER_PROBE_TIMEOUT_MS); diff --git a/apps/api/src/import-export/conversion-worker.service.ts b/apps/api/src/import-export/conversion-worker.service.ts index 28114bc..17e32a2 100644 --- a/apps/api/src/import-export/conversion-worker.service.ts +++ b/apps/api/src/import-export/conversion-worker.service.ts @@ -67,10 +67,20 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy { onModuleInit(): void { if (this.config.env.NODE_ENV === 'test') return; // tests drive drain() directly - this.timer = setInterval(() => void this.drain(), SWEEP_MS); + this.timer = setInterval(() => this.drainSafely(), SWEEP_MS); this.timer.unref(); } + /** A sweep hitting a transient database failure (outage, or the backup + * sidecar terminating connections mid-restore, #103) must degrade and + * retry on the next tick — an unhandled rejection here killed the whole + * api once. */ + private drainSafely(): void { + this.drain().catch((error: unknown) => { + this.logger.warn({ err: error }, 'conversion sweep failed; retrying on the next tick'); + }); + } + onModuleDestroy(): void { if (this.timer) clearInterval(this.timer); } @@ -80,7 +90,7 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy { * No-op under test, where tests drive {@link drain} deterministically. */ wake(): void { if (this.config.env.NODE_ENV === 'test') return; - void this.drain(); + this.drainSafely(); } /** Process every claimable job, then stop. Re-entrant-safe: a second call diff --git a/apps/api/src/mail/mail-worker.service.ts b/apps/api/src/mail/mail-worker.service.ts index 94289f6..fbecb6f 100644 --- a/apps/api/src/mail/mail-worker.service.ts +++ b/apps/api/src/mail/mail-worker.service.ts @@ -45,7 +45,16 @@ export class MailWorker implements OnModuleInit, OnModuleDestroy { onModuleInit(): void { if (this.config.env.NODE_ENV === 'test') return; - this.timer = setInterval(() => void this.deliverDueMail(), POLL_INTERVAL_MS); + // A pass hitting a transient database failure (outage, or the backup + // sidecar terminating connections mid-restore, #103) must retry on the + // next tick, never crash the api via an unhandled rejection. + this.timer = setInterval( + () => + void this.deliverDueMail().catch((error: unknown) => { + this.logger.warn({ err: error }, 'mail delivery pass failed; retrying on the next tick'); + }), + POLL_INTERVAL_MS, + ); this.timer.unref(); } diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts index 552f6cc..396a517 100644 --- a/apps/api/src/main.ts +++ b/apps/api/src/main.ts @@ -1,6 +1,14 @@ import { execFileSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; import cookieParser from 'cookie-parser'; +import { + RESTORE_STALE_MAX_AGE_MINUTES, + RESTORE_STATUS_FILE, + type RestoreStatus, +} from '@dorfteich/shared'; import { NestFactory } from '@nestjs/core'; import type { NestExpressApplication } from '@nestjs/platform-express'; import { Logger } from 'nestjs-pino'; @@ -18,10 +26,38 @@ function runMigrations(): void { execFileSync(process.execPath, [prismaCli, 'migrate', 'deploy'], { stdio: 'inherit' }); } +/** + * An api starting while the backup sidecar restores the database (issue + * #103) must not touch it: `migrate deploy` racing `pg_restore --clean` + * would corrupt the restore. This happens in practice — the restore + * terminates every connection, which can crash a worker pass and have + * Docker restart the container mid-restore. Wait for the sidecar's status + * file to leave `running` (bounded by the same staleness rule as the + * maintenance gate) before doing anything with the database. + */ +async function waitWhileRestoreRuns(backupsDir: string): Promise { + const path = join(backupsDir, RESTORE_STATUS_FILE); + for (;;) { + let status: RestoreStatus | null = null; + try { + status = existsSync(path) ? (JSON.parse(readFileSync(path, 'utf8')) as RestoreStatus) : null; + } catch { + status = null; + } + if (!status || status.state !== 'running') return; + const ageMinutes = (Date.now() - new Date(status.startedAt).getTime()) / 60_000; + if (!Number.isFinite(ageMinutes) || ageMinutes > RESTORE_STALE_MAX_AGE_MINUTES) return; + // eslint-disable-next-line no-console -- the pino logger does not exist yet + console.log(`restore of ${status.backupId} is running — waiting before touching the database`); + await sleep(2000); + } +} + async function bootstrap(): Promise { // Validate the environment before doing anything with it; this throws a // readable list of problems and prevents a half-started process. const env = loadApiEnv(); + await waitWhileRestoreRuns(env.BACKUPS_DIR); if (env.MIGRATE_ON_START) { runMigrations(); } diff --git a/apps/api/src/scheduler/scheduler.service.ts b/apps/api/src/scheduler/scheduler.service.ts index 2a9d42d..1d3d8bd 100644 --- a/apps/api/src/scheduler/scheduler.service.ts +++ b/apps/api/src/scheduler/scheduler.service.ts @@ -69,7 +69,16 @@ export class SchedulerService implements OnModuleInit, OnModuleDestroy { onModuleInit(): void { if (this.config.env.NODE_ENV === 'test') return; // tests drive jobs directly - this.timer = setInterval(() => void this.tick(), TICK_MS); + // A tick hitting a transient database failure (outage, or the backup + // sidecar terminating connections mid-restore, #103) must retry on the + // next tick, never crash the api via an unhandled rejection. + this.timer = setInterval( + () => + void this.tick().catch((error: unknown) => { + this.logger.warn({ err: error }, 'scheduler tick failed; retrying on the next tick'); + }), + TICK_MS, + ); this.timer.unref(); } diff --git a/apps/api/src/settings/instance-settings.service.ts b/apps/api/src/settings/instance-settings.service.ts index 041d2b2..5c4f7b4 100644 --- a/apps/api/src/settings/instance-settings.service.ts +++ b/apps/api/src/settings/instance-settings.service.ts @@ -49,6 +49,25 @@ export const INSTANCE_SETTINGS = { // SVG upload handling (security.md §Uploads): sanitize strips scripts and // event handlers with a maintained library; reject refuses SVG outright. 'upload.svgPolicy': z.enum(['reject', 'sanitize']).default('sanitize'), + // Backup targets (ADR 0015, issue #103). The backup sidecar reads these + // rows directly (apps/backup settings.ts — keep the schemas in sync); the + // Nextcloud app password is NOT here, it lives in the secret store + // (security.md §Secrets). localRetentionDays `null` = no admin override, + // the sidecar's BACKUP_RETENTION_DAYS env stays authoritative. + 'backup.localRetentionDays': z.number().int().min(1).nullable().default(null), + 'backup.remoteRetentionDays': z.number().int().min(1).default(30), + 'backup.nextcloud.enabled': z.boolean().default(false), + 'backup.nextcloud.baseUrl': z.string().trim().url().or(z.literal('')).default(''), + 'backup.nextcloud.username': z.string().trim().max(200).default(''), + 'backup.nextcloud.folder': z + .string() + .trim() + .max(500) + .refine((folder) => !folder.split('/').some((s) => s === '.' || s === '..'), { + message: 'validation.invalid', + }) + .default('dorfteich-backups'), + 'backup.nextcloud.uploadSchedule': z.enum(['off', 'daily', 'weekly']).default('daily'), // Instance legal pages (issue #82, security.md §Privacy): Markdown texts // for imprint and privacy policy, rendered publicly at /legal/. // Empty = not configured yet (the legal pages then show a notice and diff --git a/apps/backup/package.json b/apps/backup/package.json index a9b648b..7aa1d82 100644 --- a/apps/backup/package.json +++ b/apps/backup/package.json @@ -16,12 +16,14 @@ "dependencies": { "@dorfteich/shared": "workspace:*", "nodemailer": "^9.0.3", + "pg": "^8.16.0", "pino": "^9.6.0", "zod": "^3.25.76" }, "devDependencies": { "@types/node": "^26.1.0", "@types/nodemailer": "^8.0.1", + "@types/pg": "^8.11.0", "tsx": "^4.19.0", "vitest": "^3.0.0" } diff --git a/apps/backup/src/archive.ts b/apps/backup/src/archive.ts index d762dc6..81af325 100644 --- a/apps/backup/src/archive.ts +++ b/apps/backup/src/archive.ts @@ -46,5 +46,22 @@ export async function createArchive(outFile: string, dataDirs: string[]): Promis /** Unpacks a volume archive back over the data directories (restore path). */ export async function extractArchive(archiveFile: string, dataDirs: string[]): Promise { const { base } = archiveBase(dataDirs); - await runTar(['-xzf', archiveFile, '-C', base]); + await extractArchiveTo(archiveFile, base); +} + +/** Unpacks any of our tar.gz archives into a destination directory. */ +export async function extractArchiveTo(archiveFile: string, destDir: string): Promise { + await runTar(['-xzf', archiveFile, '-C', destDir]); +} + +/** + * Packs individual files (relative to `baseDir`, stored flat) into a gzip'd + * tar — the remote bundle format of issue #103. + */ +export async function createArchiveOfFiles( + outFile: string, + baseDir: string, + fileNames: string[], +): Promise { + await runTar(['-czf', outFile, '-C', baseDir, ...fileNames]); } diff --git a/apps/backup/src/backup-set.ts b/apps/backup/src/backup-set.ts index 6254134..a6c60d1 100644 --- a/apps/backup/src/backup-set.ts +++ b/apps/backup/src/backup-set.ts @@ -1,89 +1,15 @@ /** - * A restore set (ADR 0015) is one nightly `pg_dump` plus the matching - * uploads/plugins archive, tied together by a shared backup id derived from - * the run's UTC start time. This module owns the naming scheme and the pure - * prune decision; the runner applies it to the filesystem. + * The restore-set naming scheme and prune decision moved to + * `@dorfteich/shared` (backup-set.ts) in issue #103, because the api lists + * local sets for the in-app restore picker. Re-exported here to keep the + * sidecar-internal import paths stable. */ - -export interface BackupSet { - id: string; - /** File names (not paths) present in the backups directory. */ - files: string[]; - /** Complete = both the dump and the volume archive exist. */ - complete: boolean; -} - -const ID_PATTERN = /^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})$/; -const SET_FILE_PATTERN = /^(?:db-|files-)(\d{8}-\d{6})\.(?:dump|tar\.gz)$/; - -/** Backup id for a run starting now: UTC timestamp, filesystem-safe. */ -export function newBackupId(now: Date): string { - const pad = (value: number): string => String(value).padStart(2, '0'); - return ( - `${now.getUTCFullYear()}${pad(now.getUTCMonth() + 1)}${pad(now.getUTCDate())}` + - `-${pad(now.getUTCHours())}${pad(now.getUTCMinutes())}${pad(now.getUTCSeconds())}` - ); -} - -/** The UTC time encoded in a backup id, or null for a malformed id. */ -export function backupIdTime(id: string): Date | null { - const match = ID_PATTERN.exec(id); - if (!match) return null; - const [, year, month, day, hour, minute, second] = match; - return new Date( - Date.UTC( - Number(year), - Number(month) - 1, - Number(day), - Number(hour), - Number(minute), - Number(second), - ), - ); -} - -export function dumpFileName(id: string): string { - return `db-${id}.dump`; -} - -export function archiveFileName(id: string): string { - return `files-${id}.tar.gz`; -} - -/** - * Groups the backup directory's file names into sets, oldest first. Files - * that do not belong to the naming scheme (status.json, `.partial` staging - * files of a running or crashed run) are ignored — prune never touches them. - */ -export function listSets(fileNames: string[]): BackupSet[] { - const byId = new Map(); - for (const name of fileNames) { - const match = SET_FILE_PATTERN.exec(name); - if (!match || !backupIdTime(match[1]!)) continue; - const files = byId.get(match[1]!) ?? []; - files.push(name); - byId.set(match[1]!, files); - } - return [...byId.entries()] - .sort(([a], [b]) => a.localeCompare(b)) - .map(([id, files]) => ({ - id, - files: files.sort(), - complete: files.includes(dumpFileName(id)) && files.includes(archiveFileName(id)), - })); -} - -/** - * The sets prune may delete: older than the retention cutoff — but never - * the newest complete set, even when it is expired. A stalled instance must - * always keep one restorable set (issue #83 acceptance criteria). - */ -export function expiredSets(sets: BackupSet[], now: Date, retentionDays: number): BackupSet[] { - const cutoff = now.getTime() - retentionDays * 24 * 60 * 60 * 1000; - const newestComplete = [...sets].reverse().find((set) => set.complete); - return sets.filter((set) => { - if (set === newestComplete) return false; - const time = backupIdTime(set.id); - return time !== null && time.getTime() < cutoff; - }); -} +export { + archiveFileName, + backupIdTime, + dumpFileName, + expiredSets, + listSets, + newBackupId, + type BackupSet, +} from '@dorfteich/shared'; diff --git a/apps/backup/src/commands.ts b/apps/backup/src/commands.ts new file mode 100644 index 0000000..26e2104 --- /dev/null +++ b/apps/backup/src/commands.ts @@ -0,0 +1,125 @@ +import { BACKUP_COMMAND_CHANNEL, type BackupCommand } from '@dorfteich/shared'; +import { Client } from 'pg'; + +import type { RemoteLogger } from './remote.js'; + +/** + * Listens for api-issued backup commands ("back up now", "restore set X", + * issue #103) on the {@link BACKUP_COMMAND_CHANNEL} — the same dedicated- + * connection LISTEN/NOTIFY pattern as the collab server's listeners. The + * handler is invoked fire-and-forget; serialization against the nightly + * schedule happens in the caller's queue (index.ts). + */ + +export interface CommandListenerDeps { + createClient(): Client; + onCommand(command: BackupCommand): void; + log: RemoteLogger; + reconnectDelayMs?: number; +} + +export interface CommandListener { + start(): Promise; + stop(): Promise; +} + +const DEFAULT_RECONNECT_DELAY_MS = 1000; + +export function parseCommand(payload: string): BackupCommand | null { + let parsed: unknown; + try { + parsed = JSON.parse(payload); + } catch { + return null; + } + const command = parsed as Partial; + if (command.kind === 'run') { + return { kind: 'run', requestedBy: command.requestedBy ?? null }; + } + if ( + command.kind === 'restore' && + (command.source === 'local' || command.source === 'remote') && + typeof command.backupId === 'string' && + /^\d{8}-\d{6}$/.test(command.backupId) + ) { + return { + kind: 'restore', + source: command.source, + backupId: command.backupId, + requestedBy: command.requestedBy ?? null, + }; + } + return null; +} + +export function createCommandListener(deps: CommandListenerDeps): CommandListener { + const reconnectDelayMs = deps.reconnectDelayMs ?? DEFAULT_RECONNECT_DELAY_MS; + let client: Client | null = null; + let stopped = false; + let reconnectTimer: NodeJS.Timeout | null = null; + + function scheduleReconnect(): void { + if (stopped || reconnectTimer) return; + reconnectTimer = setTimeout(() => { + reconnectTimer = null; + void connect(); + }, reconnectDelayMs); + reconnectTimer.unref?.(); + } + + async function connect(): Promise { + if (stopped) return; + const next = deps.createClient(); + next.on('error', (error) => { + deps.log.warn({ error: error.message }, 'command listener connection error; will reconnect'); + if (client === next) client = null; + scheduleReconnect(); + }); + next.on('end', () => { + // The restore path terminates every other connection, including this + // one — reconnect quietly so the next command still arrives. + if (client === next) client = null; + scheduleReconnect(); + }); + next.on('notification', (message) => { + if (message.channel !== BACKUP_COMMAND_CHANNEL || !message.payload) return; + const command = parseCommand(message.payload); + if (!command) { + deps.log.warn({ payload: message.payload }, 'ignoring malformed backup command'); + return; + } + deps.onCommand(command); + }); + + try { + await next.connect(); + await next.query(`LISTEN ${BACKUP_COMMAND_CHANNEL}`); + client = next; + deps.log.info({ channel: BACKUP_COMMAND_CHANNEL }, 'listening for backup commands'); + } catch (error) { + deps.log.warn( + { error: (error as Error).message }, + 'could not start command listener; will retry', + ); + await next.end().catch(() => undefined); + scheduleReconnect(); + } + } + + return { + async start(): Promise { + stopped = false; + await connect(); + }, + async stop(): Promise { + stopped = true; + if (reconnectTimer) { + clearTimeout(reconnectTimer); + reconnectTimer = null; + } + const current = client; + client = null; + if (current) await current.end().catch(() => undefined); + }, + }; +} diff --git a/apps/backup/src/index.ts b/apps/backup/src/index.ts index be8a5f1..e19c9f9 100644 --- a/apps/backup/src/index.ts +++ b/apps/backup/src/index.ts @@ -1,34 +1,84 @@ +import { existsSync, readFileSync } from 'node:fs'; + +import { BACKUP_MAINTENANCE_CHANNEL, parseSecretsFile } from '@dorfteich/shared'; +import { Client } from 'pg'; import { pino } from 'pino'; import { createArchive } from './archive.js'; +import { createCommandListener } from './commands.js'; import { loadBackupEnv } from './config.js'; import { sendFailureMail } from './mail.js'; +import { performRestore } from './perform-restore.js'; import { pgDump } from './pg.js'; +import { fetchRemoteSet, resolveRemoteTarget, uploadDue, uploadSet } from './remote.js'; +import { orchestrateRestore } from './restore-orchestrator.js'; import { runBackup } from './runner.js'; import { scheduleDaily } from './scheduler.js'; +import { readBackupDbSettings } from './settings.js'; import type { RunnerDeps } from './runner.js'; +import type { BackupCommand } from '@dorfteich/shared'; /** - * Sidecar entrypoint: runs the nightly backup at BACKUP_TIME (ADR 0015). + * Sidecar entrypoint: runs the nightly backup at BACKUP_TIME (ADR 0015) and + * listens for api commands ("back up now", "restore set X", issue #103). * `BACKUP_RUN_ONCE=1` runs a single backup and exits with the outcome as - * the exit code — the on-demand path (`docker compose run backup`) and what - * the admin panel's manual trigger (#86) will call. + * the exit code — the on-demand compose path. */ const env = loadBackupEnv(); const log = pino({ level: env.LOG_LEVEL, base: { service: 'backup' } }); -const deps: RunnerDeps = { - backupsDir: env.BACKUPS_DIR, - retentionDays: env.BACKUP_RETENTION_DAYS, - now: () => new Date(), - dump: (outFile) => pgDump(env.DATABASE_URL, outFile), - archive: (outFile) => createArchive(outFile, [env.UPLOADS_DIR, env.PLUGINS_DIR]), - onFailure: async (run) => { - const sent = await sendFailureMail(env, run); - if (!sent) log.warn({ backupId: run.backupId }, 'no BACKUP_MAIL_TO configured, alert not sent'); - }, - log, -}; +function readSecrets(): Record { + return existsSync(env.SECRETS_FILE) + ? parseSecretsFile(readFileSync(env.SECRETS_FILE, 'utf8')) + : {}; +} + +/** + * Composes the runner dependencies for one run. Settings come fresh from + * the database (admin changes apply without a restart); the DB row wins + * over the env for local retention, the env stays authoritative until an + * admin saves the setting once. A manual trigger uploads whenever a target + * is configured; scheduled runs honor the upload schedule. + */ +async function buildRunnerDeps(trigger: 'scheduled' | 'manual'): Promise { + const settings = await readBackupDbSettings(env.DATABASE_URL); + const target = resolveRemoteTarget(settings, readSecrets()); + return { + backupsDir: env.BACKUPS_DIR, + retentionDays: settings.localRetentionDays ?? env.BACKUP_RETENTION_DAYS, + now: () => new Date(), + dump: (outFile) => pgDump(env.DATABASE_URL, outFile), + archive: (outFile) => createArchive(outFile, [env.UPLOADS_DIR, env.PLUGINS_DIR]), + onFailure: async (run) => { + const sent = await sendFailureMail(env, run); + if (!sent) + log.warn({ backupId: run.backupId }, 'no BACKUP_MAIL_TO configured, alert not sent'); + }, + upload: target + ? async ({ backupId, previous }) => { + const due = + trigger === 'manual' || + uploadDue( + settings.nextcloud.uploadSchedule, + previous?.lastSuccessfulUpload?.finishedAt ?? null, + new Date(), + ); + if (!due) return undefined; + return uploadSet({ + env, + backupsDir: env.BACKUPS_DIR, + target, + remoteRetentionDays: settings.remoteRetentionDays, + backupId, + previous, + now: () => new Date(), + log, + }); + } + : undefined, + log, + }; +} /** * runBackup handles run errors itself (failed status + alert); this guard @@ -36,9 +86,9 @@ const deps: RunnerDeps = { * status.json fails. Logging instead of crashing keeps the container out * of a restart loop; the missed run shows up through the freshness check. */ -async function guardedRun(): Promise<'succeeded' | 'failed'> { +async function guardedRun(trigger: 'scheduled' | 'manual'): Promise<'succeeded' | 'failed'> { try { - return (await runBackup(deps)).lastRun.outcome; + return (await runBackup(await buildRunnerDeps(trigger))).lastRun.outcome; } catch (error) { log.error({ error: String(error) }, 'backup run could not even record its status'); return 'failed'; @@ -46,18 +96,103 @@ async function guardedRun(): Promise<'succeeded' | 'failed'> { } if (process.env.BACKUP_RUN_ONCE === '1') { - process.exit((await guardedRun()) === 'succeeded' ? 0 : 1); + process.exit((await guardedRun('manual')) === 'succeeded' ? 0 : 1); +} + +/** + * All work — nightly runs, manual runs, restores — flows through one + * serial queue: a restore must never race a running backup, and command + * bursts must not overlap. Tasks never reject (each wraps its own errors). + */ +let queue: Promise = Promise.resolve(); +function enqueue(name: string, task: () => Promise): void { + queue = queue.then(async () => { + try { + await task(); + } catch (error) { + log.error({ task: name, error: String(error) }, 'queued task failed unexpectedly'); + } + }); +} + +async function notifyMaintenance(event: { phase: 'enter' | 'exit' }): Promise { + const client = new Client({ connectionString: env.DATABASE_URL }); + try { + await client.connect(); + await client.query('SELECT pg_notify($1, $2)', [ + BACKUP_MAINTENANCE_CHANNEL, + JSON.stringify(event), + ]); + } finally { + await client.end().catch(() => undefined); + } +} + +/** + * Kills every other connection to the database right before pg_restore — + * the in-app equivalent of restore.sh stopping the app services. The api + * is already holding requests in maintenance mode and restarts afterwards; + * pools and listeners (including our own command listener) reconnect. + */ +async function terminateOtherConnections(): Promise { + const client = new Client({ connectionString: env.DATABASE_URL }); + try { + await client.connect(); + await client.query( + `SELECT pg_terminate_backend(pid) FROM pg_stat_activity + WHERE datname = current_database() AND pid <> pg_backend_pid()`, + ); + } finally { + await client.end().catch(() => undefined); + } +} + +async function handleRestore(command: Extract): Promise { + const settings = await readBackupDbSettings(env.DATABASE_URL); + const target = resolveRemoteTarget(settings, readSecrets()); + await orchestrateRestore( + { + backupsDir: env.BACKUPS_DIR, + now: () => new Date(), + notifyMaintenance, + terminateOtherConnections, + fetchRemoteSet: async (backupId) => { + if (!target) throw new Error('no Nextcloud target configured'); + await fetchRemoteSet({ backupsDir: env.BACKUPS_DIR, target, backupId, log }); + }, + restoreSet: (backupId) => performRestore(env, backupId, log), + log, + }, + command, + ); } log.info( { time: env.BACKUP_TIME, retentionDays: env.BACKUP_RETENTION_DAYS, dir: env.BACKUPS_DIR }, 'backup sidecar started', ); -const schedule = scheduleDaily(env.BACKUP_TIME, async () => void (await guardedRun()), log); +const schedule = scheduleDaily( + env.BACKUP_TIME, + async () => enqueue('scheduled backup', () => guardedRun('scheduled')), + log, +); +const commands = createCommandListener({ + createClient: () => new Client({ connectionString: env.DATABASE_URL }), + onCommand: (command) => { + if (command.kind === 'run') { + log.info({ requestedBy: command.requestedBy }, 'manual backup requested'); + enqueue('manual backup', () => guardedRun('manual')); + } else { + enqueue('restore', () => handleRestore(command)); + } + }, + log, +}); +await commands.start(); for (const signal of ['SIGTERM', 'SIGINT'] as const) { process.on(signal, () => { schedule.stop(); - process.exit(0); + void commands.stop().finally(() => process.exit(0)); }); } diff --git a/apps/backup/src/mail.ts b/apps/backup/src/mail.ts index 1fff616..27459ca 100644 --- a/apps/backup/src/mail.ts +++ b/apps/backup/src/mail.ts @@ -27,49 +27,85 @@ export interface RenderedFailureMail { const CATALOGS = { de: deMails, en: enMails } as const; +/** Both alert mails share one key shape; the namespace picks the texts. */ +type AlertNamespace = 'backupFailed' | 'backupUploadFailed'; + function t( locale: 'de' | 'en', + namespace: AlertNamespace, key: keyof (typeof CATALOGS)['en']['backupFailed'], params: Record = {}, ): string { - let text: string = CATALOGS[locale].backupFailed[key]; + let text: string = CATALOGS[locale][namespace][key]; for (const [name, value] of Object.entries(params)) { text = text.replaceAll(`{{${name}}}`, value); } return text; } +function renderAlertMail( + namespace: AlertNamespace, + input: FailureMailInput, + locale: 'de' | 'en', + instanceLabel: string, +): RenderedFailureMail { + const lastSuccess = input.lastSuccessAt + ? t(locale, namespace, 'lastSuccess', { finishedAt: input.lastSuccessAt }) + : t(locale, namespace, 'lastSuccessNever'); + return { + subject: t(locale, namespace, 'subject', { + instance: instanceLabel, + backupId: input.backupId, + }), + text: [ + t(locale, namespace, 'intro', { instance: instanceLabel }), + '', + t(locale, namespace, 'backupId', { backupId: input.backupId }), + t(locale, namespace, 'error', { error: input.error }), + lastSuccess, + '', + t(locale, namespace, 'hint'), + ].join('\n'), + }; +} + export function renderFailureMail( input: FailureMailInput, locale: 'de' | 'en', instanceLabel: string, ): RenderedFailureMail { - const lastSuccess = input.lastSuccessAt - ? t(locale, 'lastSuccess', { finishedAt: input.lastSuccessAt }) - : t(locale, 'lastSuccessNever'); - return { - subject: t(locale, 'subject', { instance: instanceLabel, backupId: input.backupId }), - text: [ - t(locale, 'intro', { instance: instanceLabel }), - '', - t(locale, 'backupId', { backupId: input.backupId }), - t(locale, 'error', { error: input.error }), - lastSuccess, - '', - t(locale, 'hint'), - ].join('\n'), - }; + return renderAlertMail('backupFailed', input, locale, instanceLabel); +} + +/** Alert for a failed Nextcloud upload after a successful local run (#103). */ +export function renderUploadFailureMail( + input: FailureMailInput, + locale: 'de' | 'en', + instanceLabel: string, +): RenderedFailureMail { + return renderAlertMail('backupUploadFailed', input, locale, instanceLabel); } /** Sends the alert; returns false (after logging upstream) when no relay or * recipient is configured — a missing mail must never fail the run. */ export async function sendFailureMail(env: BackupEnv, input: FailureMailInput): Promise { + return sendAlertMail(env, renderFailureMail(input, env.BACKUP_MAIL_LOCALE, label(env))); +} + +/** Same delivery path for the upload alert (issue #103). */ +export async function sendUploadFailureMail( + env: BackupEnv, + input: FailureMailInput, +): Promise { + return sendAlertMail(env, renderUploadFailureMail(input, env.BACKUP_MAIL_LOCALE, label(env))); +} + +function label(env: BackupEnv): string { + return env.BACKUP_INSTANCE_LABEL || 'Dorfteich'; +} + +async function sendAlertMail(env: BackupEnv, mail: RenderedFailureMail): Promise { if (!env.BACKUP_MAIL_TO) return false; - const mail = renderFailureMail( - input, - env.BACKUP_MAIL_LOCALE, - env.BACKUP_INSTANCE_LABEL || 'Dorfteich', - ); const transport = createTransport({ host: env.SMTP_HOST, port: env.SMTP_PORT, diff --git a/apps/backup/src/perform-restore.ts b/apps/backup/src/perform-restore.ts new file mode 100644 index 0000000..fbf261e --- /dev/null +++ b/apps/backup/src/perform-restore.ts @@ -0,0 +1,34 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; + +import type { BackupEnv } from '@dorfteich/shared'; + +import { extractArchive } from './archive.js'; +import { archiveFileName, dumpFileName } from './backup-set.js'; +import { pgRestore } from './pg.js'; +import type { RemoteLogger } from './remote.js'; + +/** + * Restores one local set into the live database and data volumes: + * `pg_restore --clean --if-exists` of the dump, then the volume archive + * back over the uploads/plugins mounts. Shared by the operator CLI + * (restore.js via restore.sh) and the in-app restore orchestrator (#103) — + * one restore path, exercised by drills and the app alike. + */ +export async function performRestore( + env: Pick, + backupId: string, + log: RemoteLogger, +): Promise { + const dumpFile = join(env.BACKUPS_DIR, dumpFileName(backupId)); + const archiveFile = join(env.BACKUPS_DIR, archiveFileName(backupId)); + for (const file of [dumpFile, archiveFile]) { + if (!existsSync(file)) { + throw new Error(`restore set is incomplete — ${file} not found`); + } + } + log.info({ backupId }, 'restoring database dump'); + await pgRestore(env.DATABASE_URL, dumpFile); + log.info({ backupId }, 'restoring uploads/plugins archive'); + await extractArchive(archiveFile, [env.UPLOADS_DIR, env.PLUGINS_DIR]); +} diff --git a/apps/backup/src/remote.test.ts b/apps/backup/src/remote.test.ts new file mode 100644 index 0000000..03f6b53 --- /dev/null +++ b/apps/backup/src/remote.test.ts @@ -0,0 +1,273 @@ +import { createServer, type Server } from 'node:http'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import type { BackupEnv } from '@dorfteich/shared'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { extractArchiveTo } from './archive.js'; +import { buildBundle, fetchRemoteSet, pruneRemote, uploadDue, uploadSet } from './remote.js'; +import { archiveFileName, dumpFileName } from './backup-set.js'; + +/** + * In-memory WebDAV server covering the subset the sidecar uses — the tests + * exercise the real HTTP path including streamed PUT bodies, not a mock of + * our own client. + */ +function createDavServer(): { + server: Server; + files: Map; + collections: Set; + start(): Promise; + stop(): Promise; + failWith?: number; + setFailWith(status: number | undefined): void; +} { + const files = new Map(); + const collections = new Set(['/remote.php/dav/files/tester']); + let failWith: number | undefined; + + const server = createServer((req, res) => { + if (failWith) { + res.statusCode = failWith; + return res.end(); + } + const path = decodeURIComponent(new URL(req.url!, 'http://x').pathname).replace(/\/+$/, ''); + const chunks: Buffer[] = []; + req.on('data', (chunk: Buffer) => chunks.push(chunk)); + req.on('end', () => { + switch (req.method) { + case 'PROPFIND': { + if (!collections.has(path) && !files.has(path)) { + res.statusCode = 404; + return res.end(); + } + const children = + req.headers.depth === '1' + ? [...files.keys()].filter((name) => name.startsWith(`${path}/`)) + : []; + res.statusCode = 207; + res.setHeader('Content-Type', 'application/xml'); + return res.end( + ` + ${path}/ + + + ${children + .map( + (name) => `${encodeURI(name)} + ${files.get(name)!.length} + + `, + ) + .join('')} + `, + ); + } + case 'MKCOL': + collections.add(path); + res.statusCode = 201; + return res.end(); + case 'PUT': + files.set(path, Buffer.concat(chunks)); + res.statusCode = 201; + return res.end(); + case 'GET': { + const body = files.get(path); + if (!body) { + res.statusCode = 404; + return res.end(); + } + res.statusCode = 200; + return res.end(body); + } + case 'DELETE': { + const existed = files.delete(path); + res.statusCode = existed ? 204 : 404; + return res.end(); + } + default: + res.statusCode = 405; + return res.end(); + } + }); + }); + + return { + server, + files, + collections, + setFailWith: (status) => { + failWith = status; + }, + start: () => + new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => { + const address = server.address() as { port: number }; + resolve(`http://127.0.0.1:${address.port}`); + }); + }), + stop: () => + new Promise((resolve) => { + server.close(() => resolve()); + }), + }; +} + +const silentLog = { info: () => {}, warn: () => {}, error: () => {} }; +const noMailEnv = { BACKUP_MAIL_TO: undefined } as unknown as BackupEnv; + +describe('uploadDue', () => { + const now = new Date('2026-07-12T03:00:00Z'); + it('never uploads on schedule "off"', () => { + expect(uploadDue('off', null, now)).toBe(false); + expect(uploadDue('off', '2026-01-01T00:00:00Z', now)).toBe(false); + }); + it('uploads after every set on "daily"', () => { + expect(uploadDue('daily', now.toISOString(), now)).toBe(true); + }); + it('uploads weekly once the last copy is ~a week old', () => { + expect(uploadDue('weekly', null, now)).toBe(true); + expect(uploadDue('weekly', '2026-07-10T03:00:00Z', now)).toBe(false); + expect(uploadDue('weekly', '2026-07-05T02:00:00Z', now)).toBe(true); + }); +}); + +describe('remote bundle roundtrip', () => { + let dir: string; + let restoreDir: string; + const dav = createDavServer(); + let baseUrl: string; + const backupId = '20260712-030000'; + + const target = (): { baseUrl: string; username: string; password: string; folder: string } => ({ + baseUrl, + username: 'tester', + password: 'secret', + folder: 'dorfteich-backups', + }); + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'dorfteich-remote-')); + restoreDir = await mkdtemp(join(tmpdir(), 'dorfteich-remote-restore-')); + baseUrl = await dav.start(); + dav.setFailWith(undefined); + dav.files.clear(); + await writeFile(join(dir, dumpFileName(backupId)), 'dump-bytes'); + await writeFile(join(dir, archiveFileName(backupId)), 'archive-bytes'); + }); + + afterEach(async () => { + await dav.stop(); + await rm(dir, { recursive: true, force: true }); + await rm(restoreDir, { recursive: true, force: true }); + }); + + it('builds a self-contained bundle with a manifest', async () => { + const bundle = await buildBundle(dir, backupId, new Date('2026-07-12T03:00:05Z')); + expect(bundle.sizeBytes).toBeGreaterThan(0); + await extractArchiveTo(bundle.path, restoreDir); + const manifest = JSON.parse(await readFile(join(restoreDir, 'manifest.json'), 'utf8')); + expect(manifest).toMatchObject({ schemaVersion: 1, backupId }); + expect(await readFile(join(restoreDir, dumpFileName(backupId)), 'utf8')).toBe('dump-bytes'); + expect(await readFile(join(restoreDir, archiveFileName(backupId)), 'utf8')).toBe( + 'archive-bytes', + ); + }); + + it('uploads the bundle, creates the folder, and cleans up locally', async () => { + const result = await uploadSet({ + env: noMailEnv, + backupsDir: dir, + target: target(), + remoteRetentionDays: 30, + backupId, + previous: undefined, + now: () => new Date('2026-07-12T03:00:10Z'), + log: silentLog, + }); + expect(result.lastUpload.outcome).toBe('succeeded'); + expect(result.lastSuccessfulUpload?.backupId).toBe(backupId); + const uploaded = [...dav.files.keys()]; + expect(uploaded).toEqual([ + `/remote.php/dav/files/tester/dorfteich-backups/dorfteich-backup-${backupId}.tar.gz`, + ]); + // Bundle and manifest are derived data — gone after the upload. + expect(existsSync(join(dir, `dorfteich-backup-${backupId}.tar.gz`))).toBe(false); + expect(existsSync(join(dir, 'manifest.json'))).toBe(false); + }); + + it('reports a failed upload without throwing and keeps the previous success', async () => { + dav.setFailWith(401); + const previous = { + lastUpload: { + backupId: '20260711-030000', + finishedAt: '2026-07-11T03:01:00Z', + outcome: 'succeeded' as const, + sizeBytes: 10, + }, + lastSuccessfulUpload: { + backupId: '20260711-030000', + finishedAt: '2026-07-11T03:01:00Z', + sizeBytes: 10, + }, + }; + const result = await uploadSet({ + env: noMailEnv, + backupsDir: dir, + target: target(), + remoteRetentionDays: 30, + backupId, + previous, + now: () => new Date('2026-07-12T03:00:10Z'), + log: silentLog, + }); + expect(result.lastUpload.outcome).toBe('failed'); + expect(result.lastUpload.error).toContain('authentication failed'); + expect(result.lastSuccessfulUpload).toEqual(previous.lastSuccessfulUpload); + }); + + it('prunes expired remote bundles but never the newest one', async () => { + const folder = '/remote.php/dav/files/tester/dorfteich-backups'; + dav.collections.add(folder); + dav.files.set(`${folder}/dorfteich-backup-20260101-030000.tar.gz`, Buffer.from('old')); + dav.files.set(`${folder}/dorfteich-backup-20260102-030000.tar.gz`, Buffer.from('old2')); + dav.files.set(`${folder}/unrelated-file.txt`, Buffer.from('keep')); + await pruneRemote(target(), 30, new Date('2026-07-12T03:00:00Z'), silentLog); + expect([...dav.files.keys()].sort()).toEqual([ + `${folder}/dorfteich-backup-20260102-030000.tar.gz`, + `${folder}/unrelated-file.txt`, + ]); + }); + + it('downloads and unpacks a remote set, verifying the manifest', async () => { + await uploadSet({ + env: noMailEnv, + backupsDir: dir, + target: target(), + remoteRetentionDays: 30, + backupId, + previous: undefined, + now: () => new Date(), + log: silentLog, + }); + await fetchRemoteSet({ backupsDir: restoreDir, target: target(), backupId, log: silentLog }); + expect(await readFile(join(restoreDir, dumpFileName(backupId)), 'utf8')).toBe('dump-bytes'); + expect(await readFile(join(restoreDir, archiveFileName(backupId)), 'utf8')).toBe( + 'archive-bytes', + ); + }); + + it('fails a download of a set that does not exist remotely', async () => { + await expect( + fetchRemoteSet({ + backupsDir: restoreDir, + target: target(), + backupId: '20250101-000000', + log: silentLog, + }), + ).rejects.toThrow(/download/); + }); +}); diff --git a/apps/backup/src/remote.ts b/apps/backup/src/remote.ts new file mode 100644 index 0000000..27132cf --- /dev/null +++ b/apps/backup/src/remote.ts @@ -0,0 +1,252 @@ +import { createReadStream, createWriteStream } from 'node:fs'; +import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; + +import { + remoteBundleId, + remoteBundleName, + type BackupEnv, + type BackupRemoteStatus, +} from '@dorfteich/shared'; +import { + webdavCheck, + webdavDelete, + webdavGet, + webdavList, + webdavPut, + type WebDavTarget, +} from '@dorfteich/shared/webdav'; + +import { createArchiveOfFiles, extractArchiveTo } from './archive.js'; +import { archiveFileName, backupIdTime, dumpFileName } from './backup-set.js'; +import { sendUploadFailureMail } from './mail.js'; +import { NEXTCLOUD_PASSWORD_SECRET_KEY, type BackupDbSettings } from './settings.js'; + +/** + * Off-host half of a backup run (issue #103): bundle a complete local set + * into ONE self-contained archive, upload it to the admin-configured + * Nextcloud folder via WebDAV, and prune expired remote bundles — never the + * newest one, mirroring the local guarantee (#83). + */ + +export interface RemoteLogger { + info(details: object, message: string): void; + warn(details: object, message: string): void; + error(details: object, message: string): void; +} + +/** + * The effective WebDAV target, or null when the feature is off or not fully + * configured. The app password comes from the wizard-written secret store + * (never the database); base URL and username from instance settings. + */ +export function resolveRemoteTarget( + settings: BackupDbSettings, + secrets: Record, +): WebDavTarget | null { + const { enabled, baseUrl, username, folder } = settings.nextcloud; + const password = secrets[NEXTCLOUD_PASSWORD_SECRET_KEY] ?? ''; + if (!enabled || !baseUrl || !username || !password) return null; + return { baseUrl, username, password, folder }; +} + +/** + * Whether a scheduled run should upload: `off` never (manual trigger only), + * `daily` after every successful set, `weekly` when the last remote copy is + * at least ~a week old (half a day of slack so a slightly early nightly run + * does not skip its week). + */ +export function uploadDue( + schedule: BackupDbSettings['nextcloud']['uploadSchedule'], + lastSuccessfulUploadAt: string | null, + now: Date, +): boolean { + if (schedule === 'off') return false; + if (schedule === 'daily') return true; + if (!lastSuccessfulUploadAt) return true; + const ageMs = now.getTime() - new Date(lastSuccessfulUploadAt).getTime(); + return ageMs >= 6.5 * 24 * 60 * 60 * 1000; +} + +/** Contents of the bundle's manifest.json — the self-description a rebuild + * after total loss relies on (documented in the restore runbook). */ +export interface BundleManifest { + schemaVersion: 1; + backupId: string; + createdAt: string; + files: string[]; +} + +/** + * Packs one complete local set into `dorfteich-backup-.tar.gz` next to + * the set (staged as `.partial`, like every other artifact). Returns the + * bundle path; the caller uploads and then deletes it — bundles are derived + * data and would double the volume's footprint. + */ +export async function buildBundle( + backupsDir: string, + backupId: string, + now: Date, +): Promise<{ path: string; sizeBytes: number }> { + const files = [dumpFileName(backupId), archiveFileName(backupId)]; + const manifest: BundleManifest = { + schemaVersion: 1, + backupId, + createdAt: now.toISOString(), + files, + }; + const manifestName = 'manifest.json'; + await writeFile(join(backupsDir, manifestName), JSON.stringify(manifest, null, 2) + '\n'); + + const bundlePath = join(backupsDir, remoteBundleName(backupId)); + await createArchiveOfFiles(`${bundlePath}.partial`, backupsDir, [...files, manifestName]); + await rename(`${bundlePath}.partial`, bundlePath); + await rm(join(backupsDir, manifestName), { force: true }); + return { path: bundlePath, sizeBytes: (await stat(bundlePath)).size }; +} + +/** + * Uploads the bundle for `backupId` and prunes expired remote bundles. + * Returns the new remote status; an upload failure alerts by mail (the + * local run stays succeeded — the admin card and readyz surface the gap). + */ +export async function uploadSet(deps: { + env: BackupEnv; + backupsDir: string; + target: WebDavTarget; + remoteRetentionDays: number; + backupId: string; + previous: BackupRemoteStatus | undefined; + now(): Date; + log: RemoteLogger; +}): Promise { + const { backupId, target } = deps; + const previousSuccess = deps.previous?.lastSuccessfulUpload ?? null; + let bundle: { path: string; sizeBytes: number } | null = null; + try { + const check = await webdavCheck(target); + if (!check.ok) throw new Error(check.error); + + bundle = await buildBundle(deps.backupsDir, backupId, deps.now()); + const put = await webdavPut( + target, + remoteBundleName(backupId), + Readable.toWeb(createReadStream(bundle.path)) as ReadableStream, + { contentLength: bundle.sizeBytes }, + ); + if (!put.ok) throw new Error(put.error); + + await pruneRemote(target, deps.remoteRetentionDays, deps.now(), deps.log); + const finishedAt = deps.now().toISOString(); + deps.log.info({ backupId, sizeBytes: bundle.sizeBytes }, 'remote upload succeeded'); + return { + lastUpload: { backupId, finishedAt, outcome: 'succeeded', sizeBytes: bundle.sizeBytes }, + lastSuccessfulUpload: { backupId, finishedAt, sizeBytes: bundle.sizeBytes }, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + deps.log.error({ backupId, error: message }, 'remote upload failed'); + try { + const sent = await sendUploadFailureMail(deps.env, { + backupId, + error: message, + lastSuccessAt: previousSuccess?.finishedAt ?? null, + }); + if (!sent) deps.log.warn({ backupId }, 'no BACKUP_MAIL_TO configured, upload alert not sent'); + } catch (mailError) { + deps.log.error({ backupId, error: String(mailError) }, 'upload alert could not be sent'); + } + return { + lastUpload: { + backupId, + finishedAt: deps.now().toISOString(), + outcome: 'failed', + error: message, + }, + lastSuccessfulUpload: previousSuccess, + }; + } finally { + if (bundle) await rm(bundle.path, { force: true }); + await rm(join(deps.backupsDir, 'manifest.json'), { force: true }); + } +} + +/** + * Deletes remote bundles older than the retention window — never the newest + * one, even when expired (same guarantee as the local prune, #83). Foreign + * files in the folder are never touched. Prune failures are logged, not + * thrown: the upload itself succeeded and must count. + */ +export async function pruneRemote( + target: WebDavTarget, + retentionDays: number, + now: Date, + log: RemoteLogger, +): Promise { + const listed = await webdavList(target); + if (!listed.ok) { + log.warn({ error: listed.error }, 'remote prune skipped: listing failed'); + return; + } + const bundles = listed.value + .filter((entry) => !entry.isCollection) + .map((entry) => ({ name: entry.name, id: remoteBundleId(entry.name) })) + .filter((entry): entry is { name: string; id: string } => entry.id !== null) + .sort((a, b) => a.id.localeCompare(b.id)); + const cutoff = now.getTime() - retentionDays * 24 * 60 * 60 * 1000; + const newest = bundles.at(-1); + for (const bundle of bundles) { + if (bundle === newest) continue; + const time = backupIdTime(bundle.id); + if (!time || time.getTime() >= cutoff) continue; + const deleted = await webdavDelete(target, bundle.name); + if (deleted.ok) log.info({ name: bundle.name }, 'pruned expired remote bundle'); + else log.warn({ name: bundle.name, error: deleted.error }, 'remote prune of bundle failed'); + } +} + +/** + * Downloads a remote bundle and unpacks its set artifacts into the backups + * directory (the in-app restore path, and documented for operators in the + * runbook). Verifies the manifest matches the requested id. + */ +export async function fetchRemoteSet(deps: { + backupsDir: string; + target: WebDavTarget; + backupId: string; + log: RemoteLogger; +}): Promise { + const name = remoteBundleName(deps.backupId); + const response = await webdavGet(deps.target, name); + if (!response.ok) throw new Error(response.error); + if (!response.value.body) throw new Error(`download ${name}: empty response body`); + + const downloadPath = join(deps.backupsDir, `${name}.download`); + const scratchDir = join(deps.backupsDir, `.restore-${deps.backupId}`); + try { + await pipeline( + Readable.fromWeb(response.value.body as import('node:stream/web').ReadableStream), + createWriteStream(downloadPath), + ); + await mkdir(scratchDir, { recursive: true }); + await extractArchiveTo(downloadPath, scratchDir); + + const manifest = JSON.parse( + await readFile(join(scratchDir, 'manifest.json'), 'utf8'), + ) as BundleManifest; + if (manifest.backupId !== deps.backupId) { + throw new Error( + `bundle manifest mismatch: requested ${deps.backupId}, bundle contains ${manifest.backupId}`, + ); + } + for (const file of [dumpFileName(deps.backupId), archiveFileName(deps.backupId)]) { + await rename(join(scratchDir, file), join(deps.backupsDir, file)); + } + deps.log.info({ backupId: deps.backupId }, 'remote set downloaded and unpacked'); + } finally { + await rm(downloadPath, { force: true }); + await rm(scratchDir, { recursive: true, force: true }); + } +} diff --git a/apps/backup/src/restore-orchestrator.test.ts b/apps/backup/src/restore-orchestrator.test.ts new file mode 100644 index 0000000..b9d5313 --- /dev/null +++ b/apps/backup/src/restore-orchestrator.test.ts @@ -0,0 +1,108 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { archiveFileName, dumpFileName } from './backup-set.js'; +import { orchestrateRestore, type RestoreOrchestratorDeps } from './restore-orchestrator.js'; +import { readRestoreStatus } from './status.js'; + +const silentLog = { info: () => {}, warn: () => {}, error: () => {} }; +const backupId = '20260712-030000'; + +describe('orchestrateRestore', () => { + let dir: string; + let calls: string[]; + + const deps = (overrides: Partial = {}): RestoreOrchestratorDeps => ({ + backupsDir: dir, + now: () => new Date('2026-07-12T10:00:00Z'), + notifyMaintenance: async (event) => void calls.push(`notify:${event.phase}`), + terminateOtherConnections: async () => void calls.push('terminate'), + fetchRemoteSet: async (id) => { + calls.push(`fetch:${id}`); + await writeFile(join(dir, dumpFileName(id)), 'dump'); + await writeFile(join(dir, archiveFileName(id)), 'archive'); + }, + restoreSet: async (id) => void calls.push(`restore:${id}`), + graceMs: 0, + log: silentLog, + ...overrides, + }); + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'dorfteich-orchestrator-')); + calls = []; + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('runs the local restore in maintenance order and records success', async () => { + await writeFile(join(dir, dumpFileName(backupId)), 'dump'); + await writeFile(join(dir, archiveFileName(backupId)), 'archive'); + const result = await orchestrateRestore(deps(), { + kind: 'restore', + source: 'local', + backupId, + requestedBy: 'admin', + }); + expect(result.state).toBe('succeeded'); + expect(calls).toEqual(['notify:enter', 'terminate', `restore:${backupId}`, 'notify:exit']); + expect(readRestoreStatus(dir)).toMatchObject({ + state: 'succeeded', + backupId, + source: 'local', + requestedBy: 'admin', + }); + }); + + it('downloads the set first for a remote restore', async () => { + const result = await orchestrateRestore(deps(), { + kind: 'restore', + source: 'remote', + backupId, + requestedBy: null, + }); + expect(result.state).toBe('succeeded'); + expect(calls).toEqual([ + 'notify:enter', + `fetch:${backupId}`, + 'terminate', + `restore:${backupId}`, + 'notify:exit', + ]); + }); + + it('fails an incomplete local set before touching the database', async () => { + const result = await orchestrateRestore(deps(), { + kind: 'restore', + source: 'local', + backupId, + requestedBy: null, + }); + expect(result.state).toBe('failed'); + expect(result.error).toContain('incomplete'); + expect(calls).toEqual(['notify:enter', 'notify:exit']); + expect(readRestoreStatus(dir)?.state).toBe('failed'); + }); + + it('records a failed restore and still exits maintenance', async () => { + await writeFile(join(dir, dumpFileName(backupId)), 'dump'); + await writeFile(join(dir, archiveFileName(backupId)), 'archive'); + const result = await orchestrateRestore( + deps({ + restoreSet: async () => { + throw new Error('pg_restore failed: boom'); + }, + }), + { kind: 'restore', source: 'local', backupId, requestedBy: null }, + ); + expect(result.state).toBe('failed'); + expect(result.error).toContain('boom'); + expect(calls.at(-1)).toBe('notify:exit'); + expect(readRestoreStatus(dir)?.state).toBe('failed'); + }); +}); diff --git a/apps/backup/src/restore-orchestrator.ts b/apps/backup/src/restore-orchestrator.ts new file mode 100644 index 0000000..35dbf24 --- /dev/null +++ b/apps/backup/src/restore-orchestrator.ts @@ -0,0 +1,110 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; + +import type { BackupCommand, MaintenanceEvent, RestoreStatus } from '@dorfteich/shared'; + +import { archiveFileName, dumpFileName } from './backup-set.js'; +import type { RemoteLogger } from './remote.js'; +import { writeRestoreStatus } from './status.js'; + +/** + * The in-app restore, orchestrated by the sidecar (issue #103): + * + * 1. `restore-status.json` → `running` — the api's maintenance gate flips + * to 503 for everything but health and the status endpoint. + * 2. NOTIFY maintenance `enter` — the collab server persists + closes every + * live session and refuses new ones, so no in-memory document writes + * pre-restore content back afterwards. + * 3. Grace period, then terminate all other database connections. + * 4. Fetch the set (remote source: download + unpack the bundle), verify + * both artifacts exist, `pg_restore --clean` + volume archive extract — + * the exact path of the operator's `restore.sh`. + * 5. `restore-status.json` → final state; NOTIFY `exit`. The api restarts + * itself on `succeeded` (fresh caches, migrate-on-start for older dumps). + * + * Every failure lands in `restore-status.json` — the admin watches that + * file through the exempt status endpoint, so it must always resolve. + */ + +export interface RestoreOrchestratorDeps { + backupsDir: string; + now(): Date; + /** NOTIFY on a short-lived connection (maintenance enter/exit). */ + notifyMaintenance(event: MaintenanceEvent): Promise; + /** Kills every other DB connection right before pg_restore. */ + terminateOtherConnections(): Promise; + /** Downloads + unpacks the remote bundle into backupsDir (remote.ts). */ + fetchRemoteSet(backupId: string): Promise; + /** pg_restore + volume extract — shared with the restore.js CLI. */ + restoreSet(backupId: string): Promise; + /** Milliseconds between maintenance enter and connection termination. */ + graceMs?: number; + log: RemoteLogger; +} + +export async function orchestrateRestore( + deps: RestoreOrchestratorDeps, + command: Extract, +): Promise { + const startedAt = deps.now().toISOString(); + const base: Omit = { + schemaVersion: 1, + backupId: command.backupId, + source: command.source, + requestedBy: command.requestedBy, + startedAt, + }; + await writeRestoreStatus(deps.backupsDir, { ...base, state: 'running', finishedAt: null }); + deps.log.info( + { backupId: command.backupId, source: command.source, requestedBy: command.requestedBy }, + 'restore started — instance entering maintenance mode', + ); + + let entered = false; + try { + await deps.notifyMaintenance({ phase: 'enter' }); + entered = true; + await sleep(deps.graceMs ?? 5000); + + if (command.source === 'remote') { + await deps.fetchRemoteSet(command.backupId); + } + for (const file of [dumpFileName(command.backupId), archiveFileName(command.backupId)]) { + if (!existsSync(join(deps.backupsDir, file))) { + throw new Error(`restore set is incomplete — ${file} not found`); + } + } + + await deps.terminateOtherConnections(); + await deps.restoreSet(command.backupId); + + const status: RestoreStatus = { + ...base, + state: 'succeeded', + finishedAt: deps.now().toISOString(), + }; + await writeRestoreStatus(deps.backupsDir, status); + deps.log.info({ backupId: command.backupId }, 'restore succeeded — api will restart'); + return status; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const status: RestoreStatus = { + ...base, + state: 'failed', + finishedAt: deps.now().toISOString(), + error: message, + }; + // Best effort — if even this write fails the api's staleness bound + // (RESTORE_STALE_MAX_AGE_MINUTES) unblocks the instance eventually. + await writeRestoreStatus(deps.backupsDir, status).catch(() => undefined); + deps.log.error({ backupId: command.backupId, error: message }, 'restore failed'); + return status; + } finally { + if (entered) { + await deps.notifyMaintenance({ phase: 'exit' }).catch((error: unknown) => { + deps.log.warn({ error: String(error) }, 'maintenance exit notify failed'); + }); + } + } +} diff --git a/apps/backup/src/restore.ts b/apps/backup/src/restore.ts index 6f99582..a7fbcf0 100644 --- a/apps/backup/src/restore.ts +++ b/apps/backup/src/restore.ts @@ -1,19 +1,15 @@ -import { existsSync } from 'node:fs'; -import { join } from 'node:path'; - import { pino } from 'pino'; -import { extractArchive } from './archive.js'; -import { archiveFileName, backupIdTime, dumpFileName } from './backup-set.js'; +import { backupIdTime } from './backup-set.js'; import { loadBackupEnv } from './config.js'; -import { pgRestore } from './pg.js'; +import { performRestore } from './perform-restore.js'; /** * In-container half of the restore runbook (operations.md §Backup & restore): - * `pg_restore --clean --if-exists` of the set's dump, then the volume archive - * back over the uploads/plugins mounts. The host-side `restore.sh` wraps this - * with stopping/starting the app services — run through that, not directly, - * unless you know the api and collab are down. + * the host-side `restore.sh` wraps this with stopping/starting the app + * services — run through that, not directly, unless you know the api and + * collab are down. The actual restore lives in perform-restore.ts, shared + * with the in-app restore orchestrator (issue #103). */ const env = loadBackupEnv(); const log = pino({ level: env.LOG_LEVEL, base: { service: 'backup-restore' } }); @@ -24,17 +20,10 @@ if (!backupId || !backupIdTime(backupId)) { process.exit(2); } -const dumpFile = join(env.BACKUPS_DIR, dumpFileName(backupId)); -const archiveFile = join(env.BACKUPS_DIR, archiveFileName(backupId)); -for (const file of [dumpFile, archiveFile]) { - if (!existsSync(file)) { - log.error({ file }, 'restore set is incomplete — file not found'); - process.exit(2); - } +try { + await performRestore(env, backupId, log); +} catch (error) { + log.error({ backupId, error: String(error) }, 'restore failed'); + process.exit(1); } - -log.info({ backupId }, 'restoring database dump'); -await pgRestore(env.DATABASE_URL, dumpFile); -log.info({ backupId }, 'restoring uploads/plugins archive'); -await extractArchive(archiveFile, [env.UPLOADS_DIR, env.PLUGINS_DIR]); log.info({ backupId }, 'restore complete — start the stack and verify /readyz'); diff --git a/apps/backup/src/runner.ts b/apps/backup/src/runner.ts index fd0e5ff..63929f9 100644 --- a/apps/backup/src/runner.ts +++ b/apps/backup/src/runner.ts @@ -2,6 +2,8 @@ import { mkdir, readdir, rename, rm, stat } from 'node:fs/promises'; import { join } from 'node:path'; import { archiveFileName, dumpFileName, expiredSets, listSets, newBackupId } from './backup-set.js'; +import type { BackupRemoteStatus } from '@dorfteich/shared'; + import { readStatus, writeStatus, @@ -29,6 +31,18 @@ export interface RunnerDeps { archive(outFile: string): Promise; /** Failure alert (mail.ts); errors here are logged, never rethrown. */ onFailure(run: { backupId: string; error: string; lastSuccessAt: string | null }): Promise; + /** + * Off-host upload of the completed set (issue #103, remote.ts). Called + * only after a successful local run; returns the new remote status, or + * undefined when no upload happened (not configured / not due) — the + * previous remote status is then carried unchanged. Upload failures are + * reported inside the returned status, never thrown: the local set exists + * and the run must count as succeeded. + */ + upload?(input: { + backupId: string; + previous: BackupRemoteStatus | undefined; + }): Promise; log: { info(details: object, message: string): void; error(details: object, message: string): void; @@ -40,6 +54,7 @@ export async function runBackup(deps: RunnerDeps): Promise { const backupId = newBackupId(startedAt); const previous = readStatus(deps.backupsDir); const lastSuccess = previous?.lastSuccess ?? null; + let remote = previous?.remote; await mkdir(deps.backupsDir, { recursive: true }); const dumpFile = join(deps.backupsDir, dumpFileName(backupId)); @@ -60,6 +75,16 @@ export async function runBackup(deps: RunnerDeps): Promise { }; success = { backupId, finishedAt: run.finishedAt, sizes }; deps.log.info({ backupId, sizes }, 'backup run succeeded'); + if (deps.upload) { + try { + remote = (await deps.upload({ backupId, previous: remote })) ?? remote; + } catch (uploadError) { + // Defensive: remote.ts reports failures in its return value; a throw + // here is a bug, but it must never turn a good local run into a + // failed one. + deps.log.error({ backupId, error: String(uploadError) }, 'upload hook threw unexpectedly'); + } + } } catch (error) { await removePartials(deps.backupsDir); const message = error instanceof Error ? error.message : String(error); @@ -92,6 +117,7 @@ export async function runBackup(deps: RunnerDeps): Promise { retentionDays: deps.retentionDays, lastRun: run, lastSuccess: success, + ...(remote ? { remote } : {}), }; await writeStatus(deps.backupsDir, status); return status; diff --git a/apps/backup/src/settings.test.ts b/apps/backup/src/settings.test.ts new file mode 100644 index 0000000..b329d1e --- /dev/null +++ b/apps/backup/src/settings.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest'; + +import { parseCommand } from './commands.js'; +import { resolveRemoteTarget } from './remote.js'; +import { parseBackupSettings } from './settings.js'; + +describe('parseBackupSettings', () => { + it('returns pure defaults for a database without the keys', () => { + expect(parseBackupSettings([])).toEqual({ + localRetentionDays: null, + remoteRetentionDays: 30, + nextcloud: { + enabled: false, + baseUrl: '', + username: '', + folder: 'dorfteich-backups', + uploadSchedule: 'daily', + }, + }); + }); + + it('applies stored rows and falls back per key on invalid values', () => { + const settings = parseBackupSettings([ + { key: 'backup.localRetentionDays', value: 14 }, + { key: 'backup.remoteRetentionDays', value: 'not-a-number' }, + { key: 'backup.nextcloud.enabled', value: true }, + { key: 'backup.nextcloud.baseUrl', value: 'https://cloud.example.com' }, + { key: 'backup.nextcloud.username', value: 'backupuser' }, + { key: 'backup.nextcloud.uploadSchedule', value: 'weekly' }, + ]); + expect(settings.localRetentionDays).toBe(14); + expect(settings.remoteRetentionDays).toBe(30); + expect(settings.nextcloud).toMatchObject({ + enabled: true, + baseUrl: 'https://cloud.example.com', + username: 'backupuser', + uploadSchedule: 'weekly', + }); + }); +}); + +describe('resolveRemoteTarget', () => { + const settings = parseBackupSettings([ + { key: 'backup.nextcloud.enabled', value: true }, + { key: 'backup.nextcloud.baseUrl', value: 'https://cloud.example.com' }, + { key: 'backup.nextcloud.username', value: 'backupuser' }, + ]); + + it('combines settings with the secret-store app password', () => { + expect(resolveRemoteTarget(settings, { BACKUP_NEXTCLOUD_PASSWORD: 'app-pass' })).toEqual({ + baseUrl: 'https://cloud.example.com', + username: 'backupuser', + password: 'app-pass', + folder: 'dorfteich-backups', + }); + }); + + it('is null when disabled or incompletely configured', () => { + expect(resolveRemoteTarget(settings, {})).toBeNull(); + expect( + resolveRemoteTarget(parseBackupSettings([]), { BACKUP_NEXTCLOUD_PASSWORD: 'app-pass' }), + ).toBeNull(); + }); +}); + +describe('parseCommand', () => { + it('parses run and restore commands', () => { + expect(parseCommand('{"kind":"run","requestedBy":"admin"}')).toEqual({ + kind: 'run', + requestedBy: 'admin', + }); + expect( + parseCommand('{"kind":"restore","source":"remote","backupId":"20260712-030000"}'), + ).toEqual({ + kind: 'restore', + source: 'remote', + backupId: '20260712-030000', + requestedBy: null, + }); + }); + + it('rejects malformed payloads and bad backup ids', () => { + expect(parseCommand('not json')).toBeNull(); + expect(parseCommand('{"kind":"nuke"}')).toBeNull(); + expect( + parseCommand('{"kind":"restore","source":"remote","backupId":"../../etc/passwd"}'), + ).toBeNull(); + expect( + parseCommand('{"kind":"restore","source":"elsewhere","backupId":"20260712-030000"}'), + ).toBeNull(); + }); +}); diff --git a/apps/backup/src/settings.ts b/apps/backup/src/settings.ts new file mode 100644 index 0000000..541e20a --- /dev/null +++ b/apps/backup/src/settings.ts @@ -0,0 +1,78 @@ +import { Client } from 'pg'; +import { z } from 'zod'; + +/** + * The sidecar's read-only view of the backup-related instance settings + * (issue #103). The api owns the `instance_settings` registry; the sidecar + * reads the raw rows with the same defaults, so admin changes apply on the + * next run without a restart or a config channel. A row that is missing or + * fails validation falls back to its default — never to a crash. + */ + +export const NEXTCLOUD_PASSWORD_SECRET_KEY = 'BACKUP_NEXTCLOUD_PASSWORD'; + +const SETTING_SCHEMAS = { + // Local retention: `null` means "no admin override" — the env value + // (BACKUP_RETENTION_DAYS, stage-specific) stays authoritative until a + // Site Admin saves the setting once. + 'backup.localRetentionDays': z.number().int().min(1).nullable().default(null), + 'backup.remoteRetentionDays': z.number().int().min(1).default(30), + 'backup.nextcloud.enabled': z.boolean().default(false), + 'backup.nextcloud.baseUrl': z.string().default(''), + 'backup.nextcloud.username': z.string().default(''), + 'backup.nextcloud.folder': z.string().default('dorfteich-backups'), + 'backup.nextcloud.uploadSchedule': z.enum(['off', 'daily', 'weekly']).default('daily'), +} as const; + +export interface BackupDbSettings { + localRetentionDays: number | null; + remoteRetentionDays: number; + nextcloud: { + enabled: boolean; + baseUrl: string; + username: string; + folder: string; + uploadSchedule: 'off' | 'daily' | 'weekly'; + }; +} + +export function parseBackupSettings(rows: { key: string; value: unknown }[]): BackupDbSettings { + const byKey = new Map(rows.map((row) => [row.key, row.value])); + const get = ( + key: K, + ): z.infer<(typeof SETTING_SCHEMAS)[K]> => { + const parsed = SETTING_SCHEMAS[key].safeParse(byKey.get(key)); + return parsed.success ? parsed.data : SETTING_SCHEMAS[key].parse(undefined); + }; + return { + localRetentionDays: get('backup.localRetentionDays'), + remoteRetentionDays: get('backup.remoteRetentionDays'), + nextcloud: { + enabled: get('backup.nextcloud.enabled'), + baseUrl: get('backup.nextcloud.baseUrl'), + username: get('backup.nextcloud.username'), + folder: get('backup.nextcloud.folder'), + uploadSchedule: get('backup.nextcloud.uploadSchedule'), + }, + }; +} + +/** + * Reads the settings rows with a short-lived connection. A database that is + * unreachable or predates the settings keys yields plain defaults — the + * local backup must still run when the instance is at its most broken. + */ +export async function readBackupDbSettings(databaseUrl: string): Promise { + const client = new Client({ connectionString: databaseUrl }); + try { + await client.connect(); + const result = await client.query<{ key: string; value: unknown }>( + `SELECT key, value FROM instance_settings WHERE key LIKE 'backup.%'`, + ); + return parseBackupSettings(result.rows); + } catch { + return parseBackupSettings([]); + } finally { + await client.end().catch(() => undefined); + } +} diff --git a/apps/backup/src/status.ts b/apps/backup/src/status.ts index b613e9d..e0863f9 100644 --- a/apps/backup/src/status.ts +++ b/apps/backup/src/status.ts @@ -2,7 +2,12 @@ import { existsSync, readFileSync } from 'node:fs'; import { rename, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { BACKUP_STATUS_FILE, type BackupStatus } from '@dorfteich/shared'; +import { + BACKUP_STATUS_FILE, + RESTORE_STATUS_FILE, + type BackupStatus, + type RestoreStatus, +} from '@dorfteich/shared'; /** * File I/O for `status.json` on the backups volume. The shape itself is a @@ -26,8 +31,27 @@ export function readStatus(backupsDir: string): BackupStatus | null { /** Atomic write (staging + rename) so readers never see a torn file. */ export async function writeStatus(backupsDir: string, status: BackupStatus): Promise { - const path = join(backupsDir, BACKUP_STATUS_FILE); + await writeJsonAtomic(join(backupsDir, BACKUP_STATUS_FILE), status); +} + +/** The in-app restore progress file (issue #103) — same contract discipline + * as status.json: the api's maintenance gate and admin card read it. */ +export function readRestoreStatus(backupsDir: string): RestoreStatus | null { + const path = join(backupsDir, RESTORE_STATUS_FILE); + if (!existsSync(path)) return null; + try { + return JSON.parse(readFileSync(path, 'utf8')) as RestoreStatus; + } catch { + return null; + } +} + +export async function writeRestoreStatus(backupsDir: string, status: RestoreStatus): Promise { + await writeJsonAtomic(join(backupsDir, RESTORE_STATUS_FILE), status); +} + +async function writeJsonAtomic(path: string, value: unknown): Promise { const staging = `${path}.tmp-${process.pid}`; - await writeFile(staging, JSON.stringify(status, null, 2) + '\n'); + await writeFile(staging, JSON.stringify(value, null, 2) + '\n'); await rename(staging, path); } diff --git a/apps/collab/src/index.ts b/apps/collab/src/index.ts index 866ac6d..c65f527 100644 --- a/apps/collab/src/index.ts +++ b/apps/collab/src/index.ts @@ -4,6 +4,7 @@ import { Client } from 'pg'; import { createAccessListener } from './access-listener.js'; import { createPool, pingDatabase } from './db.js'; import { createLogger } from './logger.js'; +import { createMaintenanceListener, type MaintenanceListener } from './maintenance-listener.js'; import { PostgresPagePersistence } from './persistence.js'; import { createRestoreListener } from './restore-listener.js'; import { closeDocumentConnections, createCollabServer } from './server.js'; @@ -27,6 +28,17 @@ async function bootstrap(): Promise { const sessionRegistry = new PostgresSessionRegistry({ pool, logger }); const versionStore = new PostgresVersionStore({ pool, logger }); + // Persists + closes all sessions before the backup sidecar replaces the + // database, and refuses new ones until the restore is over (issue #103). + // The close action is bound after the server exists (mutual reference, + // same trick as the session registry above). + let closeAllConnections = (): void => {}; + const maintenanceListener: MaintenanceListener = createMaintenanceListener({ + createClient: () => new Client({ connectionString: env.DATABASE_URL }), + closeAllConnections: () => closeAllConnections(), + logger, + }); + const server = createCollabServer({ version: env.APP_VERSION, logger, @@ -35,7 +47,13 @@ async function bootstrap(): Promise { persistence: new PostgresPagePersistence(pool), sessionRegistry, versionStore, + isMaintenanceActive: () => maintenanceListener.isActive(), }); + closeAllConnections = () => { + for (const documentName of server.hocuspocus.documents.keys()) { + closeDocumentConnections(server.hocuspocus, documentName); + } + }; // Terminate live sessions when access to a pond is revoked (issue #39). The // listener owns a dedicated connection because `LISTEN` is connection-bound @@ -60,6 +78,7 @@ async function bootstrap(): Promise { await server.listen(env.PORT); await accessListener.start(); await restoreListener.start(); + await maintenanceListener.start(); sessionRegistry.start(() => [...server.hocuspocus.documents.keys()]); logger.info({ event: 'listen', port: env.PORT }, 'collaboration server listening'); @@ -69,6 +88,7 @@ async function bootstrap(): Promise { void Promise.allSettled([ accessListener.stop(), restoreListener.stop(), + maintenanceListener.stop(), server.destroy(), pool.end(), ]).then(() => process.exit(0)); diff --git a/apps/collab/src/maintenance-listener.test.ts b/apps/collab/src/maintenance-listener.test.ts new file mode 100644 index 0000000..55f161f --- /dev/null +++ b/apps/collab/src/maintenance-listener.test.ts @@ -0,0 +1,76 @@ +import { EventEmitter } from 'node:events'; + +import { BACKUP_MAINTENANCE_CHANNEL } from '@dorfteich/shared'; +import { pino } from 'pino'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { createMaintenanceListener, type MaintenanceListener } from './maintenance-listener.js'; + +const logger = pino({ enabled: false }); + +class FakeClient extends EventEmitter { + connect = vi.fn(async () => undefined); + query = vi.fn(async () => ({ rows: [] })); + end = vi.fn(async () => undefined); + notify(payload: string): void { + this.emit('notification', { channel: BACKUP_MAINTENANCE_CHANNEL, payload }); + } +} + +describe('maintenance listener (issue #103)', () => { + const listeners: MaintenanceListener[] = []; + + afterEach(async () => { + await Promise.all(listeners.splice(0).map((listener) => listener.stop())); + vi.restoreAllMocks(); + }); + + function build(maxMaintenanceMs?: number) { + const client = new FakeClient(); + const closeAllConnections = vi.fn(); + const listener = createMaintenanceListener({ + createClient: () => client as unknown as never, + closeAllConnections, + logger, + reconnectDelayMs: 10, + maxMaintenanceMs, + }); + listeners.push(listener); + return { client, closeAllConnections, listener }; + } + + it('closes all sessions on enter and refuses until exit', async () => { + const { client, closeAllConnections, listener } = build(); + await listener.start(); + expect(listener.isActive()).toBe(false); + + client.notify(JSON.stringify({ phase: 'enter' })); + expect(listener.isActive()).toBe(true); + expect(closeAllConnections).toHaveBeenCalledTimes(1); + + // A duplicate enter must not close twice. + client.notify(JSON.stringify({ phase: 'enter' })); + expect(closeAllConnections).toHaveBeenCalledTimes(1); + + client.notify(JSON.stringify({ phase: 'exit' })); + expect(listener.isActive()).toBe(false); + }); + + it('ignores malformed payloads', async () => { + const { client, listener } = build(); + await listener.start(); + client.notify('not json'); + expect(listener.isActive()).toBe(false); + }); + + it('leaves maintenance via the failsafe when no exit ever arrives', async () => { + vi.useFakeTimers(); + const { client, listener } = build(60_000); + await listener.start(); + client.notify(JSON.stringify({ phase: 'enter' })); + expect(listener.isActive()).toBe(true); + vi.advanceTimersByTime(61_000); + expect(listener.isActive()).toBe(false); + vi.useRealTimers(); + }); +}); diff --git a/apps/collab/src/maintenance-listener.ts b/apps/collab/src/maintenance-listener.ts new file mode 100644 index 0000000..4817ff7 --- /dev/null +++ b/apps/collab/src/maintenance-listener.ts @@ -0,0 +1,148 @@ +import { BACKUP_MAINTENANCE_CHANNEL, type MaintenanceEvent } from '@dorfteich/shared'; +import type { Client } from 'pg'; +import type { Logger } from 'pino'; + +export interface MaintenanceListenerDeps { + /** Dedicated `LISTEN` connection factory (connection-bound, not pooled). */ + createClient: () => Client; + /** Persist + close every live session (server.ts closes the sockets). */ + closeAllConnections: () => void; + logger: Logger; + reconnectDelayMs?: number; + /** Failsafe: leave maintenance after this long even without an exit event. */ + maxMaintenanceMs?: number; +} + +export interface MaintenanceListener { + start(): Promise; + stop(): Promise; + /** Checked by onAuthenticate — no new sessions while a restore runs. */ + isActive(): boolean; +} + +const DEFAULT_RECONNECT_DELAY_MS = 1000; +const DEFAULT_MAX_MAINTENANCE_MS = 30 * 60 * 1000; + +/** + * Maintenance mode during an in-app restore (issue #103): the backup + * sidecar notifies `enter` before it replaces the database. Every live + * session is persisted and closed NOW — before the restore starts — so no + * in-memory document can write pre-restore content back over the restored + * database, and new connections are refused until `exit`. The failsafe + * timeout mirrors the api's staleness bound: a crashed sidecar must not + * leave the collab server refusing connections forever. + */ +export function createMaintenanceListener(deps: MaintenanceListenerDeps): MaintenanceListener { + const reconnectDelayMs = deps.reconnectDelayMs ?? DEFAULT_RECONNECT_DELAY_MS; + const maxMaintenanceMs = deps.maxMaintenanceMs ?? DEFAULT_MAX_MAINTENANCE_MS; + let client: Client | null = null; + let stopped = false; + let reconnectTimer: NodeJS.Timeout | null = null; + let active = false; + let failsafeTimer: NodeJS.Timeout | null = null; + + function enter(): void { + if (active) return; + active = true; + deps.logger.warn( + { event: 'maintenance.enter' }, + 'restore starting — closing all sessions, refusing new connections', + ); + deps.closeAllConnections(); + failsafeTimer = setTimeout(() => { + deps.logger.error( + { event: 'maintenance.timeout' }, + 'no maintenance exit received — leaving maintenance via failsafe', + ); + exit(); + }, maxMaintenanceMs); + failsafeTimer.unref?.(); + } + + function exit(): void { + if (!active) return; + active = false; + if (failsafeTimer) { + clearTimeout(failsafeTimer); + failsafeTimer = null; + } + deps.logger.info({ event: 'maintenance.exit' }, 'maintenance over — accepting connections'); + } + + function scheduleReconnect(): void { + if (stopped || reconnectTimer) return; + reconnectTimer = setTimeout(() => { + reconnectTimer = null; + void connect(); + }, reconnectDelayMs); + reconnectTimer.unref?.(); + } + + async function connect(): Promise { + if (stopped) return; + const next = deps.createClient(); + next.on('error', (error) => { + // Expected mid-restore: the sidecar terminates every connection. + deps.logger.info( + { event: 'maintenance.listen.error', err: error.message }, + 'maintenance listener connection error; will reconnect', + ); + if (client === next) client = null; + scheduleReconnect(); + }); + next.on('end', () => { + if (client === next) client = null; + scheduleReconnect(); + }); + next.on('notification', (message) => { + if (message.channel !== BACKUP_MAINTENANCE_CHANNEL || !message.payload) return; + let event: MaintenanceEvent; + try { + event = JSON.parse(message.payload) as MaintenanceEvent; + } catch { + return; + } + if (event.phase === 'enter') enter(); + else if (event.phase === 'exit') exit(); + }); + + try { + await next.connect(); + await next.query(`LISTEN ${BACKUP_MAINTENANCE_CHANNEL}`); + client = next; + deps.logger.info( + { event: 'maintenance.listen.ready', channel: BACKUP_MAINTENANCE_CHANNEL }, + 'listening for maintenance events', + ); + } catch (error) { + deps.logger.warn( + { event: 'maintenance.listen.connect_failed', err: (error as Error).message }, + 'could not start maintenance listener; will retry', + ); + await next.end().catch(() => undefined); + scheduleReconnect(); + } + } + + return { + async start(): Promise { + stopped = false; + await connect(); + }, + async stop(): Promise { + stopped = true; + if (reconnectTimer) { + clearTimeout(reconnectTimer); + reconnectTimer = null; + } + if (failsafeTimer) { + clearTimeout(failsafeTimer); + failsafeTimer = null; + } + const current = client; + client = null; + if (current) await current.end().catch(() => undefined); + }, + isActive: () => active, + }; +} diff --git a/apps/collab/src/server.ts b/apps/collab/src/server.ts index 33ff49b..8f70237 100644 --- a/apps/collab/src/server.ts +++ b/apps/collab/src/server.ts @@ -36,6 +36,12 @@ export interface CollabServerDeps { * Optional for the same reason as `sessionRegistry`. */ versionStore?: VersionStore; + /** + * Maintenance mode during an in-app restore (issue #103): while true, new + * connections are refused so no session comes up mid-restore. Optional — + * servers without the listener (tests) never enter maintenance. + */ + isMaintenanceActive?: () => boolean; } /** @@ -100,6 +106,13 @@ export function createCollabServer(deps: CollabServerDeps): Server { * dropped server-side via Hocuspocus' read-only connection flag. */ async onAuthenticate({ documentName, token, connectionConfig }): Promise { + if (deps.isMaintenanceActive?.()) { + logger.info( + { event: 'auth.rejected', documentName, reason: 'maintenance' }, + 'collab authentication rejected', + ); + throw new Error('maintenance'); + } const result = verifyCollabToken(token, tokenSecret); if (!result.valid) { logger.info( diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index f7ab259..02cf501 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -1,6 +1,9 @@ +import { useEffect, useState } from 'react'; import { Navigate, Route, Routes } from 'react-router-dom'; import { RequireAnonymous, RequireAuth, RequireSiteAdmin } from './auth/guards'; +import { MAINTENANCE_EVENT } from './lib/api'; +import { MaintenancePage } from './pages/MaintenancePage'; import { AppLayout } from './layout/AppLayout'; import { AdminSettingsPage } from './pages/AdminSettingsPage'; import { AdminSystemPage } from './pages/AdminSystemPage'; @@ -27,6 +30,18 @@ import { useSetupStatus } from './setup/use-setup-status'; export function App(): React.JSX.Element { const setup = useSetupStatus(); + // Maintenance gate (issue #103): once any api call answers 503 + // maintenance_mode — a backup restore is running — the whole UI becomes + // the maintenance screen, which polls the exempt status endpoint and + // reloads when the instance is back. + const [maintenance, setMaintenance] = useState(false); + useEffect(() => { + const onMaintenance = (): void => setMaintenance(true); + window.addEventListener(MAINTENANCE_EVENT, onMaintenance); + return () => window.removeEventListener(MAINTENANCE_EVENT, onMaintenance); + }, []); + if (maintenance) return ; + // First-run gate (issue #81): while setup is pending the api 503s almost // everything, so the SPA offers only the wizard, the login (to resume a // started wizard as the Site Admin), and a pending notice — without the diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts index ea94ab5..f3796ad 100644 --- a/apps/web/src/lib/api.ts +++ b/apps/web/src/lib/api.ts @@ -14,6 +14,19 @@ export class ApiError extends Error { } } +/** + * Fired once any api call answers 503 `maintenance_mode` (an in-app backup + * restore is running, issue #103) — App.tsx listens and swaps the UI for + * the maintenance screen, whatever the user was doing. + */ +export const MAINTENANCE_EVENT = 'dorfteich:maintenance'; + +function noteMaintenance(status: number, body: ApiErrorBody | null): void { + if (status === 503 && body?.code === 'maintenance_mode') { + window.dispatchEvent(new CustomEvent(MAINTENANCE_EVENT)); + } +} + async function requestJson(method: string, path: string, body?: unknown): Promise { let response: Response; try { @@ -30,6 +43,7 @@ async function requestJson(method: string, path: string, body?: unknown): Pro } if (!response.ok) { const parsed = (await response.json().catch(() => null)) as ApiErrorBody | null; + noteMaintenance(response.status, parsed); throw new ApiError( response.status, parsed ?? { code: `http_${response.status}`, message: response.statusText }, diff --git a/apps/web/src/pages/AdminBackupSection.tsx b/apps/web/src/pages/AdminBackupSection.tsx new file mode 100644 index 0000000..98562e2 --- /dev/null +++ b/apps/web/src/pages/AdminBackupSection.tsx @@ -0,0 +1,504 @@ +import type { + BackupConnectionTestResult, + BackupSetView, + BackupSetsView, + BackupSettingsInput, + BackupSettingsView, + SystemBackupView, +} from '@dorfteich/shared'; +import { useQuery, useQueryClient } from '@tanstack/react-query'; +import { useState } from 'react'; +import { useTranslation } from 'react-i18next'; + +import { formatBytes } from '../files/file-format'; +import { ApiError, apiGet, apiPost, apiPut } from '../lib/api'; + +const BACKUP_QUERY_KEY = ['admin', 'system', 'backup'] as const; + +/** + * The backup area of the Site-Admin system panel (issue #86 card extended + * by issue #103): status card with a manual trigger, the Nextcloud target + * configuration, and the in-app restore with a type-to-confirm gate. + */ +export function BackupSection(): React.JSX.Element { + const { t } = useTranslation('system'); + const query = useQuery({ + queryKey: BACKUP_QUERY_KEY, + queryFn: () => apiGet('/admin/system/backup'), + // A requested run/restore progresses in the sidecar — keep the card live. + refetchInterval: 15_000, + }); + const view = query.data; + if (!view) return <>; + + return ( +
+

{t('backup.title')}

+ + + +
+ ); +} + +function StatusCard({ view }: { view: SystemBackupView }): React.JSX.Element { + const { t } = useTranslation('system'); + const queryClient = useQueryClient(); + const [runState, setRunState] = useState<'idle' | 'pending' | 'requested'>('idle'); + + const runNow = async (): Promise => { + setRunState('pending'); + try { + await apiPost('/admin/system/backup/run', {}); + setRunState('requested'); + } catch { + setRunState('idle'); + } finally { + await queryClient.invalidateQueries({ queryKey: BACKUP_QUERY_KEY }); + } + }; + + return ( +
+ {!view.available &&

{t('backup.unavailable')}

} + {view.available && view.status && ( + <> +

+ + {view.fresh ? t('backup.fresh') : t('backup.stale', { hours: view.maxAgeHours })} + +

+
+
{t('backup.lastSuccess')}
+
+ {view.status.lastSuccess + ? `${new Date(view.status.lastSuccess.finishedAt).toLocaleString()} (${view.status.lastSuccess.backupId}) — ${t( + 'backup.sizes', + { + dump: formatBytes(view.status.lastSuccess.sizes.dumpBytes), + archive: formatBytes(view.status.lastSuccess.sizes.archiveBytes), + }, + )}` + : t('backup.never')} +
+
{t('backup.lastRun')}
+
+ {new Date(view.status.lastRun.finishedAt).toLocaleString()} —{' '} + {t(`backup.outcome.${view.status.lastRun.outcome}`)} +
+ {view.status.lastRun.error && ( + <> +
{t('backup.error')}
+
+ {view.status.lastRun.error} +
+ + )} +
{t('backup.remote.title')}
+
+ {!view.remoteConfigured && t('backup.remote.notConfigured')} + {view.remoteConfigured && ( + <> + {t('backup.remote.lastSuccessfulUpload')}:{' '} + {view.status.remote?.lastSuccessfulUpload + ? `${new Date( + view.status.remote.lastSuccessfulUpload.finishedAt, + ).toLocaleString()} (${view.status.remote.lastSuccessfulUpload.backupId}, ${t( + 'backup.remote.bundleSize', + { + size: formatBytes(view.status.remote.lastSuccessfulUpload.sizeBytes), + }, + )})` + : t('backup.never')} + {view.status.remote?.lastUpload.outcome === 'failed' && ( + <> + {' — '} + {view.status.remote.lastUpload.error} + + )} + + )} +
+
+

+ {t('backup.retention', { days: view.status.retentionDays })} +

+ + )} +

+ +

+ {runState === 'requested' &&

{t('backup.runRequested')}

} +
+ ); +} + +interface SettingsDraft { + localRetentionDays: string; + remoteRetentionDays: string; + enabled: boolean; + baseUrl: string; + username: string; + password: string; + folder: string; + uploadSchedule: 'off' | 'daily' | 'weekly'; +} + +function toDraft(view: BackupSettingsView): SettingsDraft { + return { + localRetentionDays: view.localRetentionDays === null ? '' : String(view.localRetentionDays), + remoteRetentionDays: String(view.remoteRetentionDays), + enabled: view.nextcloud.enabled, + baseUrl: view.nextcloud.baseUrl, + username: view.nextcloud.username, + password: '', + folder: view.nextcloud.folder, + uploadSchedule: view.nextcloud.uploadSchedule, + }; +} + +function toInput(draft: SettingsDraft): BackupSettingsInput { + return { + localRetentionDays: draft.localRetentionDays.trim() ? Number(draft.localRetentionDays) : null, + remoteRetentionDays: Number(draft.remoteRetentionDays) || 30, + nextcloud: { + enabled: draft.enabled, + baseUrl: draft.baseUrl.trim(), + username: draft.username.trim(), + folder: draft.folder.trim() || 'dorfteich-backups', + uploadSchedule: draft.uploadSchedule, + ...(draft.password ? { password: draft.password } : {}), + }, + }; +} + +function BackupSettingsForm(): React.JSX.Element { + const { t } = useTranslation('system'); + const { t: tErrors } = useTranslation('errors'); + const queryClient = useQueryClient(); + const [draft, setDraft] = useState(null); + const [notice, setNotice] = useState<{ kind: 'ok' | 'error'; text: string } | null>(null); + const [busy, setBusy] = useState<'test' | 'save' | null>(null); + + const query = useQuery({ + queryKey: ['admin', 'system', 'backup', 'settings'], + queryFn: () => apiGet('/admin/system/backup/settings'), + }); + const view = query.data; + if (!view) return <>; + const form = draft ?? toDraft(view); + const update = (patch: Partial): void => setDraft({ ...form, ...patch }); + + const describeError = (error: unknown): string => { + if (error instanceof ApiError) { + const detail = (error.body.details as { nextcloud?: string[] } | undefined)?.nextcloud?.[0]; + const translated = tErrors(error.body.code, { defaultValue: error.body.code }); + return detail ? `${translated} ${detail}` : translated; + } + return String(error); + }; + + const test = async (): Promise => { + setBusy('test'); + setNotice(null); + try { + const result = await apiPost( + '/admin/system/backup/nextcloud/test', + { + baseUrl: form.baseUrl.trim(), + username: form.username.trim(), + folder: form.folder.trim() || 'dorfteich-backups', + ...(form.password ? { password: form.password } : {}), + }, + ); + setNotice( + result.ok + ? { kind: 'ok', text: t('backup.settings.testOk') } + : { kind: 'error', text: result.error ?? t('backup.settings.test') }, + ); + } catch (error) { + setNotice({ kind: 'error', text: describeError(error) }); + } finally { + setBusy(null); + } + }; + + const save = async (): Promise => { + setBusy('save'); + setNotice(null); + try { + await apiPut('/admin/system/backup/settings', toInput(form)); + setDraft(null); + setNotice({ kind: 'ok', text: t('backup.settings.saved') }); + await queryClient.invalidateQueries({ queryKey: ['admin', 'system', 'backup'] }); + } catch (error) { + setNotice({ kind: 'error', text: describeError(error) }); + } finally { + setBusy(null); + } + }; + + return ( +
{ + e.preventDefault(); + void save(); + }} + > +

{t('backup.settings.title')}

+ {notice && ( +

+ {notice.text} +

+ )} + +

{t('backup.settings.localRetentionHint')}

+ +
+ +

{t('backup.settings.baseUrlHint')}

+ + +

{t('backup.settings.passwordHint')}

+ + + + +
+ +
+ ); +} + +function RestorePicker({ restore }: { restore: SystemBackupView['restore'] }): React.JSX.Element { + const { t } = useTranslation('system'); + const { t: tErrors } = useTranslation('errors'); + const [open, setOpen] = useState(false); + const [selected, setSelected] = useState<{ + source: 'local' | 'remote'; + set: BackupSetView; + } | null>(null); + const [confirm, setConfirm] = useState(''); + const [state, setState] = useState<'idle' | 'pending' | 'requested'>('idle'); + const [error, setError] = useState(null); + + const sets = useQuery({ + queryKey: ['admin', 'system', 'backup', 'sets'], + queryFn: () => apiGet('/admin/system/backup/sets'), + enabled: open, + }); + + const request = async (): Promise => { + if (!selected) return; + setState('pending'); + setError(null); + try { + await apiPost('/admin/system/backup/restore', { + source: selected.source, + backupId: selected.set.backupId, + confirm, + }); + setState('requested'); + } catch (err) { + setState('idle'); + setError( + err instanceof ApiError + ? tErrors(err.body.code, { defaultValue: err.body.code }) + : String(err), + ); + } + }; + + const setList = (source: 'local' | 'remote', list: BackupSetView[]): React.JSX.Element => ( +
    + {list.map((set) => ( +
  • + +
    • + ))} +
+ ); + + return ( +
+

{t('backup.restore.title')}

+ {restore && ( +

+ {t(`backup.restore.status.${restore.state}`, { + id: restore.backupId, + startedAt: new Date(restore.startedAt).toLocaleString(), + finishedAt: restore.finishedAt ? new Date(restore.finishedAt).toLocaleString() : '', + error: restore.error ?? '', + })} +

+ )} + {!open && ( + + )} + {open && ( + <> +

+ {t('backup.restore.warning')} +

+ {sets.isPending &&

{t('backup.restore.loading')}

} + {sets.data && ( + <> +

{t('backup.restore.localTitle')}

+ {sets.data.local.length === 0 &&

{t('backup.restore.empty')}

} + {setList('local', sets.data.local)} + {sets.data.remoteConfigured && ( + <> +

{t('backup.restore.remoteTitle')}

+ {sets.data.remoteError && ( +

+ {t('backup.restore.remoteError', { error: sets.data.remoteError })} +

+ )} + {!sets.data.remoteError && sets.data.remote.length === 0 && ( +

{t('backup.restore.empty')}

+ )} + {setList('remote', sets.data.remote)} + + )} + + )} + {selected && ( +
+ + +
+ )} + {state === 'requested' &&

{t('backup.restore.requested')}

} + {error &&

{error}

} + + )} +
+ ); +} diff --git a/apps/web/src/pages/AdminSystemPage.tsx b/apps/web/src/pages/AdminSystemPage.tsx index 909860c..3b68f9f 100644 --- a/apps/web/src/pages/AdminSystemPage.tsx +++ b/apps/web/src/pages/AdminSystemPage.tsx @@ -4,7 +4,6 @@ import type { JobTriggerOutcome, JobTriggerResult, StorageOverviewView, - SystemBackupView, SystemJobView, } from '@dorfteich/shared'; import { keepPreviousData, useQuery } from '@tanstack/react-query'; @@ -14,6 +13,7 @@ import { Link } from 'react-router-dom'; import { formatBytes } from '../files/file-format'; import { apiGet, apiPost } from '../lib/api'; +import { BackupSection } from './AdminBackupSection'; /** * Site-Admin "System" panel (issue #86): maintenance jobs with a manual @@ -30,7 +30,7 @@ export function AdminSystemPage(): React.JSX.Element { ← {t('backLink')}

- + @@ -147,64 +147,6 @@ function JobOutcome({ job }: { job: SystemJobView }): React.JSX.Element { return {t('jobs.outcome.ok')}; } -function BackupCard(): React.JSX.Element { - const { t } = useTranslation('system'); - const query = useQuery({ - queryKey: ['admin', 'system', 'backup'], - queryFn: () => apiGet('/admin/system/backup'), - }); - const view = query.data; - if (!view) return <>; - - return ( -
-

{t('backup.title')}

- {!view.available &&

{t('backup.unavailable')}

} - {view.available && view.status && ( - <> -

- - {view.fresh ? t('backup.fresh') : t('backup.stale', { hours: view.maxAgeHours })} - -

-
-
{t('backup.lastSuccess')}
-
- {view.status.lastSuccess - ? `${new Date(view.status.lastSuccess.finishedAt).toLocaleString()} (${view.status.lastSuccess.backupId}) — ${t( - 'backup.sizes', - { - dump: formatBytes(view.status.lastSuccess.sizes.dumpBytes), - archive: formatBytes(view.status.lastSuccess.sizes.archiveBytes), - }, - )}` - : t('backup.never')} -
-
{t('backup.lastRun')}
-
- {new Date(view.status.lastRun.finishedAt).toLocaleString()} —{' '} - {t(`backup.outcome.${view.status.lastRun.outcome}`)} -
- {view.status.lastRun.error && ( - <> -
{t('backup.error')}
-
- {view.status.lastRun.error} -
- - )} -
-

- {t('backup.retention', { days: view.status.retentionDays })} -

- - )} -
- ); -} - interface AuditFilter { actor: string; action: string; @@ -356,6 +298,9 @@ const KNOWN_ACTIONS = [ 'auth.login_succeeded', 'auth.password_reset', 'job.triggered', + 'backup.settings_changed', + 'backup.run_triggered', + 'backup.restore_requested', ]; function AuditRow({ entry }: { entry: AuditEntryView }): React.JSX.Element { diff --git a/apps/web/src/pages/MaintenancePage.tsx b/apps/web/src/pages/MaintenancePage.tsx new file mode 100644 index 0000000..181acfd --- /dev/null +++ b/apps/web/src/pages/MaintenancePage.tsx @@ -0,0 +1,68 @@ +import type { RestoreStatusResponse } from '@dorfteich/shared'; +import { useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; + +import { apiGet } from '../lib/api'; + +const POLL_INTERVAL_MS = 3000; + +/** + * The status page behind maintenance mode (issue #103): while a backup + * restore runs, every api call answers 503, and this screen polls the one + * exempt endpoint. Once the restore is over and the api answers again, the + * page reloads itself — after a successful restore the whole client state + * is stale anyway. + */ +export function MaintenancePage(): React.JSX.Element { + const { t } = useTranslation('system'); + const [status, setStatus] = useState(null); + + useEffect(() => { + let cancelled = false; + const poll = async (): Promise => { + let current: RestoreStatusResponse | null = null; + try { + current = await apiGet('/backup/restore-status'); + } catch { + current = null; // api restarting — keep waiting + } + if (cancelled) return; + setStatus(current); + if (current && current.state !== 'running' && current.state !== 'failed') { + // idle or succeeded: check the api serves normally again, then reload. + try { + await apiGet('/healthz'); + if (!cancelled) window.location.reload(); + return; + } catch { + // still restarting + } + } + if (!cancelled) setTimeout(() => void poll(), POLL_INTERVAL_MS); + }; + void poll(); + return () => { + cancelled = true; + }; + }, []); + + const failedStatus = status !== null && status.state === 'failed' ? status : null; + return ( +
+

{t('maintenance.title')}

+

{t('maintenance.body')}

+ {failedStatus ? ( + <> +

+ {t('maintenance.failed', { error: failedStatus.error ?? '' })} +

+ + + ) : ( +

{t('maintenance.waiting')}

+ )} +
+ ); +} diff --git a/apps/web/src/styles/base.css b/apps/web/src/styles/base.css index 77a3ea5..d886a2d 100644 --- a/apps/web/src/styles/base.css +++ b/apps/web/src/styles/base.css @@ -2522,6 +2522,98 @@ button { color: var(--color-text-muted); } +/* Backup settings + in-app restore (issue #103) */ +.system-backup__settings, +.system-backup__restore { + margin-top: var(--space-4); + padding-top: var(--space-3); + border-top: 1px solid var(--color-border, #cbd5e1); +} + +.system-backup__settings label, +.system-backup__confirm label { + display: block; + margin-top: var(--space-2); + font-weight: 600; +} + +.system-backup__settings input[type='text'], +.system-backup__settings input[type='url'], +.system-backup__settings input[type='password'], +.system-backup__settings input[type='number'], +.system-backup__settings select, +.system-backup__confirm input[type='text'] { + display: block; + margin-top: var(--space-1); + width: min(28rem, 100%); +} + +.system-backup__checkbox, +.system-backup__settings .system-backup__checkbox { + font-weight: 600; +} + +.system-backup__checkbox input { + margin-right: var(--space-1); +} + +.system-backup__nextcloud { + border: none; + padding: 0; + margin: 0; +} + +.system-backup__nextcloud:disabled { + opacity: 0.5; +} + +.system-backup__hint { + color: var(--color-text-muted); + font-size: 0.875rem; + margin: var(--space-1) 0; +} + +.system-backup__notice--error { + color: var(--color-danger); +} + +.system-backup__warning { + color: var(--color-danger); + font-weight: 600; +} + +.system-backup__sets { + list-style: none; + padding: 0; + margin: var(--space-1) 0; +} + +.system-backup__save, +.system-backup__test { + margin-top: var(--space-3); +} + +.system-backup__restore-status--failed { + color: var(--color-danger); +} + +.button--danger { + border-color: var(--color-danger); + color: var(--color-danger); +} + +/* Maintenance screen during an in-app restore (issue #103) */ +.maintenance-page { + max-width: 32rem; + margin: 15vh auto 0; + padding: var(--space-4); + text-align: center; +} + +.maintenance-page__error { + color: var(--color-danger); +} + .system-audit__filters { display: flex; gap: var(--space-2); diff --git a/deploy/compose/.env.example b/deploy/compose/.env.example index 1f32459..13a8357 100644 --- a/deploy/compose/.env.example +++ b/deploy/compose/.env.example @@ -68,7 +68,11 @@ SMTP_FROM=Dorfteich #TZ=Europe/Berlin #BACKUP_TIME=03:00 # Local retention in days: 30 (default) for Prod, 7 for Test/Int (ADR 0015). +# A Site Admin can override this in the admin UI (issue #103) — the saved +# setting then wins over this value. #BACKUP_RETENTION_DAYS=30 +# Off-host copies to a Nextcloud (issue #103) are configured entirely in the +# admin UI (Admin -> System -> Backups) — no env values needed here. # Failure alert: recipient (unset = no mail, failures only in the logs and # status.json), mail language (de|en), and the label used in the subject # (defaults to the compose project name). diff --git a/deploy/monitoring.md b/deploy/monitoring.md index a78ba07..7f887d3 100644 --- a/deploy/monitoring.md +++ b/deploy/monitoring.md @@ -17,7 +17,16 @@ existing **Uptime-Kuma** instance. Checks enumerated in the body: `database`, `migrations` (hard), `converter`, `renderer`, `backup` (warning-level; `backup` reads the sidecar's `status.json` and warns when the last successful backup is -older than 26 h — ADR 0015). +older than 26 h — ADR 0015). With a Nextcloud backup target configured +(issue #103) a `backup_remote` check appears too: it warns when the last +successful off-host upload is older than its schedule allows (26 h daily, +170 h weekly; manual-only schedules are never stale). + +During an **in-app restore** (issue #103) every application route answers +503 `maintenance_mode` for a few minutes and the api restarts itself once +afterwards; `healthz`/`readyz` and `GET /api/v1/backup/restore-status` +keep answering throughout. A short monitor blip around a restore is +expected. **Degraded never restarts containers**: the Docker healthchecks use the liveness endpoints only (api `/api/v1/healthz`, web `/healthz`, collab diff --git a/docs/architecture/operations.md b/docs/architecture/operations.md index 0e05f06..0eb105d 100644 --- a/docs/architecture/operations.md +++ b/docs/architecture/operations.md @@ -45,10 +45,20 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack. - Nightly at 03:00 stage-local time (sidecar `backup`, issue #83; env `BACKUP_TIME`/`TZ`): `pg_dump -Fc` → uploads/plugins volume archive (one - tar, same backup id `YYYYMMDD-HHMMSS`) → prune (`BACKUP_RETENTION_DAYS`, - 30 default / 7 Test+Int; the newest complete set always survives) → - `status.json` on the `backups` volume → on failure a mail directly via - the instance SMTP to `BACKUP_MAIL_TO`. Mirror to BASEL is issue #84. + tar, same backup id `YYYYMMDD-HHMMSS`) → optional Nextcloud upload + (issue #103, see below) → prune (`BACKUP_RETENTION_DAYS`, 30 default / + 7 Test+Int; a Site-Admin setting overrides the env; the newest complete + set always survives) → `status.json` on the `backups` volume → on + failure a mail directly via the instance SMTP to `BACKUP_MAIL_TO`. + Mirror to BASEL is issue #84. +- **Off-host copies** (issue #103): with a Nextcloud target configured in + the admin UI (WebDAV base URL + username + folder in instance settings, + app password in the secret store), each successful set is bundled into + ONE `dorfteich-backup-.tar.gz` (dump + files archive + manifest) and + uploaded per schedule (daily/weekly/manual); remote prune mirrors the + local guarantee. The api and the sidecar talk over pg NOTIFY/LISTEN + (`backup_command`); upload failures mail like run failures, staleness + shows up as the `backup_remote` readyz check. - **On-demand run**: `docker compose run --rm -e BACKUP_RUN_ONCE=1 backup` (exit code = outcome); list sets with `docker compose exec backup ls /backups`. - **Restore runbook** (also the Prod-relocation procedure) — automated by @@ -63,8 +73,13 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack. drills" issue); manual procedure and relocation notes in `docs/operations/restore-runbook.md`. Quarterly manual full-runbook drill on Test. -- **Admin UI**: Site Admin can download the latest dump/archive and trigger - an on-demand backup run (ADR 0015 — restore stays CLI-only). +- **Admin UI** (issue #103): Site Admin triggers on-demand backups + ("Back up now") and restores a local or Nextcloud set in-app — a + type-to-confirm prompt, then the sidecar orchestrates: maintenance mode + (global 503 with an exempt status endpoint), collab sessions closed via + the `backup_maintenance` NOTIFY channel, connections terminated, + `pg_restore` + volume extract, api restart. `restore.sh` remains the + disaster fallback when the app itself is gone. ## Maintenance jobs (in-app scheduler, `jobs` table) diff --git a/docs/operations/restore-runbook.md b/docs/operations/restore-runbook.md index 4b71309..933f98c 100644 --- a/docs/operations/restore-runbook.md +++ b/docs/operations/restore-runbook.md @@ -32,6 +32,31 @@ never corruption. (`docker compose up -d db backup`), copy the set into the backups volume (`docker run --rm -v -v _backups:/backups …`), then steps 2–3. +## In-app restore (issue #103) + +With a Nextcloud target configured, Site Admins can restore without shell +access: _Admin → System → Backups → Restore_ lists local and remote sets; +after a type-to-confirm prompt the backup sidecar orchestrates the whole +restore (maintenance mode → download + verify → terminate connections → +`pg_restore` + volume extract → api restart). Progress lands in +`restore-status.json` next to `status.json`; the public +`GET /api/v1/backup/restore-status` endpoint keeps answering while +everything else serves 503 `maintenance_mode`. This runbook stays the +disaster path for when the app itself is gone. + +## Fetching a set from Nextcloud (total loss) + +When the host is gone but the off-host copies exist, rebuild from the +Nextcloud bundle alone: + +1. Download `dorfteich-backup-.tar.gz` from the configured Nextcloud + folder (browser or `curl -u : -O +https://cloud.example.com/remote.php/dav/files///dorfteich-backup-.tar.gz`). +2. Unpack it: `tar -xzf dorfteich-backup-.tar.gz` → `db-.dump`, + `files-.tar.gz`, and a `manifest.json` describing the set. +3. Copy the two artifacts into the (fresh) stack's backups volume — see + _Relocation to a new host_ above — and run `./restore.sh `. + ## Automated monthly drill (`.gitea/workflows/drill.yml`) Runs on the 1st of each month — and on demand by pushing a `drill-*` tag diff --git a/docs/self-hosting/README.md b/docs/self-hosting/README.md index c01407b..d7ee076 100644 --- a/docs/self-hosting/README.md +++ b/docs/self-hosting/README.md @@ -94,13 +94,45 @@ archives the uploads/plugins volumes nightly at `BACKUP_TIME` onto the `backups` volume, prunes by `BACKUP_RETENTION_DAYS`, writes `status.json`, and — with `BACKUP_MAIL_TO` set — mails you on failure. -- On-demand backup: `docker compose run --rm -e BACKUP_RUN_ONCE=1 backup` +- On-demand backup: `docker compose run --rm -e BACKUP_RUN_ONCE=1 backup`, + or the **Back up now** button under _Admin → System_. - List sets: `docker compose exec backup ls /backups` - Restore: `./restore.sh ` (fetch `deploy/backup/restore.sh` next to your compose file) — details in `docs/operations/restore-runbook.md`. -- Copy the `backups` volume off the host regularly; a backup on the same - disk protects against mistakes, not against losing the host. + +### Off-host copies to a Nextcloud + +Get the backups off the host — a backup on the same disk protects against +mistakes, not against losing the host. Any Nextcloud you can reach works +as the target; configure it entirely in the admin UI (_Admin → System → +Backups_): + +1. In Nextcloud, create an **app password** for the account that should + hold the backups (Settings → Security → Devices & sessions). +2. In Dorfteich, enable _Upload backups to Nextcloud_, enter the plain + Nextcloud address (e.g. `https://cloud.example.com`), the username, the + app password and a folder, and use **Test connection** — it verifies + the credentials and creates the folder. The password is kept in the + secret store on the `secrets` volume, never in the database. +3. Pick the upload schedule (after every nightly backup, weekly, or manual + only) and the retention for both sides. After each successful upload, + old remote bundles beyond the retention are pruned — never the newest + one. + +Each upload is ONE self-contained archive +(`dorfteich-backup-.tar.gz` = database dump + files archive + +manifest) — everything needed to rebuild the instance after total loss. +`readyz` warns (`backup_remote` check) when the off-host copy grows stale, +and upload failures alert through the backup failure mail. + +**Restore from the admin UI:** _Admin → System → Backups → Restore_ lists +local and Nextcloud sets. Restoring asks you to re-type the backup id, +then the instance enters maintenance mode (everything answers 503 plus a +status page), restores itself through the backup sidecar, and restarts. +If the app itself is gone, use the operator path in +`docs/operations/restore-runbook.md` instead — it documents fetching a +bundle from Nextcloud by hand. ## Health & troubleshooting diff --git a/packages/shared/i18n/de/errors.json b/packages/shared/i18n/de/errors.json index 452314b..1b3e66a 100644 --- a/packages/shared/i18n/de/errors.json +++ b/packages/shared/i18n/de/errors.json @@ -95,5 +95,11 @@ "comment_not_author": "Nur die Autorin/der Autor kann diesen Kommentar ändern.", "comment_has_replies": "Dieser Kommentar hat Antworten — den ganzen Thread kann nur eine Teich-Administration löschen.", "comment_parent_invalid": "Antworten müssen sich auf einen Kommentar der obersten Ebene derselben Seite beziehen.", - "comment_not_root": "Nur Kommentare der obersten Ebene können als erledigt markiert werden." + "comment_not_root": "Nur Kommentare der obersten Ebene können als erledigt markiert werden.", + "maintenance_mode": "Die Instanz ist im Wartungsmodus, während ein Backup wiederhergestellt wird.", + "backup_connection_failed": "Der Nextcloud-Verbindungstest ist fehlgeschlagen.", + "backup_restore_confirm_mismatch": "Der Bestätigungstext stimmt nicht mit der Backup-ID überein.", + "backup_restore_running": "Es läuft bereits eine Wiederherstellung.", + "backup_remote_not_configured": "Es ist kein Nextcloud-Backup-Ziel konfiguriert.", + "backup_set_not_found": "Das gewählte Backup-Set wurde nicht gefunden." } diff --git a/packages/shared/i18n/de/mails.json b/packages/shared/i18n/de/mails.json index 1a8d8ba..6149d19 100644 --- a/packages/shared/i18n/de/mails.json +++ b/packages/shared/i18n/de/mails.json @@ -31,6 +31,15 @@ "lastSuccessNever": "Letztes erfolgreiches Backup: noch keines", "hint": "Prüfe die Sidecar-Logs (docker compose logs backup) und die status.json auf dem Backups-Volume." }, + "backupUploadFailed": { + "subject": "[{{instance}}] Backup-Upload zu Nextcloud fehlgeschlagen ({{backupId}})", + "intro": "Das lokale Backup auf {{instance}} war erfolgreich, aber der Upload zum Nextcloud-Ziel ist fehlgeschlagen.", + "backupId": "Backup-ID: {{backupId}}", + "error": "Fehler: {{error}}", + "lastSuccess": "Letzter erfolgreicher Upload: {{finishedAt}}", + "lastSuccessNever": "Letzter erfolgreicher Upload: noch keiner", + "hint": "Prüfe die Nextcloud-Verbindungseinstellungen im Admin-Bereich (Verbindungstest) und die Sidecar-Logs (docker compose logs backup)." + }, "digest": { "subject": "Dorfteich: {{count}} Neuigkeiten für dich", "intro": "Das ist auf von dir beobachteten Seiten passiert ({{count}} Neuigkeiten):", diff --git a/packages/shared/i18n/de/system.json b/packages/shared/i18n/de/system.json index d601fc5..5ba51ac 100644 --- a/packages/shared/i18n/de/system.json +++ b/packages/shared/i18n/de/system.json @@ -56,7 +56,62 @@ "never": "noch keines", "sizes": "Dump {{dump}}, Dateien {{archive}}", "retention": "Aufbewahrung: {{days}} Tage", - "error": "Letzter Fehler" + "error": "Letzter Fehler", + "runNow": "Jetzt sichern", + "runPending": "Wird angefordert…", + "runRequested": "Backup angefordert — der Sidecar führt es im Hintergrund aus. Diese Karte aktualisiert sich automatisch.", + "remote": { + "title": "Nextcloud-Ziel", + "notConfigured": "Kein Nextcloud-Ziel konfiguriert — Backups bleiben nur auf diesem Host.", + "lastUpload": "Letzter Upload", + "lastSuccessfulUpload": "Letzter erfolgreicher Upload", + "bundleSize": "Bundle {{size}}" + }, + "settings": { + "title": "Backup-Einstellungen", + "localRetention": "Lokale Aufbewahrung (Tage)", + "localRetentionHint": "Leer = Container-Standard behalten (BACKUP_RETENTION_DAYS).", + "remoteRetention": "Nextcloud-Aufbewahrung (Tage)", + "enabled": "Backups zu Nextcloud hochladen", + "baseUrl": "Nextcloud-Adresse (URL)", + "baseUrlHint": "Die einfache Server-Adresse, z. B. https://cloud.example.com — der WebDAV-Pfad wird abgeleitet.", + "username": "Benutzername", + "password": "App-Passwort", + "passwordStored": "Gespeichert — leer lassen, um es zu behalten", + "passwordHint": "In Nextcloud anlegen unter Einstellungen → Sicherheit → Geräte & Sitzungen.", + "folder": "Ordner", + "schedule": "Upload-Zeitplan", + "scheduleOptions": { + "off": "Nur manuell", + "daily": "Nach jedem Backup (täglich)", + "weekly": "Wöchentlich" + }, + "test": "Verbindung testen", + "testPending": "Teste…", + "testOk": "Verbindung ok — der Ordner ist erreichbar.", + "save": "Backup-Einstellungen speichern", + "savePending": "Speichere…", + "saved": "Backup-Einstellungen gespeichert." + }, + "restore": { + "title": "Wiederherstellung", + "open": "Ein Backup wiederherstellen…", + "warning": "Die Wiederherstellung ersetzt ALLE aktuellen Inhalte, Dateien und Einstellungen dieser Instanz durch den Stand des gewählten Backups. Während der Wiederherstellung ist die Instanz im Wartungsmodus.", + "loading": "Lade Backup-Sets…", + "localTitle": "Auf diesem Host", + "remoteTitle": "Auf Nextcloud", + "empty": "Keine wiederherstellbaren Sets gefunden.", + "remoteError": "Die Nextcloud-Sets konnten nicht aufgelistet werden: {{error}}", + "confirmLabel": "Zur Bestätigung die Backup-ID eintippen: {{id}}", + "button": "Dieses Backup wiederherstellen", + "pending": "Wiederherstellung wird angefordert…", + "requested": "Wiederherstellung angefordert — die Instanz geht in den Wartungsmodus.", + "status": { + "running": "Wiederherstellung von {{id}} läuft (gestartet {{startedAt}})…", + "succeeded": "Wiederherstellung von {{id}} war erfolgreich ({{finishedAt}}).", + "failed": "Wiederherstellung von {{id}} ist fehlgeschlagen: {{error}}" + } + } }, "audit": { "title": "Audit-Log", @@ -107,7 +162,10 @@ "auth.login_failed": "Anmeldung fehlgeschlagen", "auth.login_succeeded": "Anmeldung", "auth.password_reset": "Passwort zurückgesetzt", - "job.triggered": "Job manuell gestartet" + "job.triggered": "Job manuell gestartet", + "backup.settings_changed": "Backup-Einstellungen geändert", + "backup.run_triggered": "Manuelles Backup ausgelöst", + "backup.restore_requested": "Backup-Wiederherstellung angefordert" } }, "storage": { @@ -123,5 +181,12 @@ "shared": "geteilt" }, "empty": "Noch keine Belegung erfasst." + }, + "maintenance": { + "title": "Wartungsmodus", + "body": "Ein Backup wird wiederhergestellt. Die Instanz ist vorübergehend nicht erreichbar.", + "waiting": "Warte darauf, dass die Instanz zurückkommt…", + "failed": "Die Wiederherstellung ist fehlgeschlagen: {{error}}", + "reload": "Neu laden" } } diff --git a/packages/shared/i18n/en/errors.json b/packages/shared/i18n/en/errors.json index aec3f2c..bc103a0 100644 --- a/packages/shared/i18n/en/errors.json +++ b/packages/shared/i18n/en/errors.json @@ -95,5 +95,11 @@ "comment_not_author": "Only the author can change this comment.", "comment_has_replies": "This comment has replies — only a pond admin can delete the whole thread.", "comment_parent_invalid": "Replies must reference a top-level comment on the same page.", - "comment_not_root": "Only top-level comments can be resolved." + "comment_not_root": "Only top-level comments can be resolved.", + "maintenance_mode": "The instance is in maintenance mode while a backup is being restored.", + "backup_connection_failed": "The Nextcloud connection test failed.", + "backup_restore_confirm_mismatch": "The confirmation text does not match the backup id.", + "backup_restore_running": "A restore is already running.", + "backup_remote_not_configured": "No Nextcloud backup target is configured.", + "backup_set_not_found": "The selected backup set was not found." } diff --git a/packages/shared/i18n/en/mails.json b/packages/shared/i18n/en/mails.json index f7b64fe..b75793b 100644 --- a/packages/shared/i18n/en/mails.json +++ b/packages/shared/i18n/en/mails.json @@ -31,6 +31,15 @@ "lastSuccessNever": "Last successful backup: none yet", "hint": "Check the sidecar logs (docker compose logs backup) and status.json on the backups volume." }, + "backupUploadFailed": { + "subject": "[{{instance}}] Backup upload to Nextcloud failed ({{backupId}})", + "intro": "The local backup on {{instance}} succeeded, but uploading it to the Nextcloud target failed.", + "backupId": "Backup id: {{backupId}}", + "error": "Error: {{error}}", + "lastSuccess": "Last successful upload: {{finishedAt}}", + "lastSuccessNever": "Last successful upload: none yet", + "hint": "Check the Nextcloud connection settings in the admin panel (test connection) and the sidecar logs (docker compose logs backup)." + }, "digest": { "subject": "Dorfteich: {{count}} updates for you", "intro": "Here is what happened on pages you watch ({{count}} updates):", diff --git a/packages/shared/i18n/en/system.json b/packages/shared/i18n/en/system.json index cea8ae8..605ed5a 100644 --- a/packages/shared/i18n/en/system.json +++ b/packages/shared/i18n/en/system.json @@ -56,7 +56,62 @@ "never": "none yet", "sizes": "Dump {{dump}}, files {{archive}}", "retention": "Retention: {{days}} days", - "error": "Last error" + "error": "Last error", + "runNow": "Back up now", + "runPending": "Requesting…", + "runRequested": "Backup requested — the sidecar is running it in the background. This card refreshes automatically.", + "remote": { + "title": "Nextcloud target", + "notConfigured": "No Nextcloud target configured — backups stay on this host only.", + "lastUpload": "Last upload", + "lastSuccessfulUpload": "Last successful upload", + "bundleSize": "bundle {{size}}" + }, + "settings": { + "title": "Backup settings", + "localRetention": "Local retention (days)", + "localRetentionHint": "Empty = keep the container default (BACKUP_RETENTION_DAYS).", + "remoteRetention": "Nextcloud retention (days)", + "enabled": "Upload backups to Nextcloud", + "baseUrl": "Nextcloud address (URL)", + "baseUrlHint": "The plain server address, e.g. https://cloud.example.com — the WebDAV path is derived.", + "username": "Username", + "password": "App password", + "passwordStored": "Stored — leave empty to keep it", + "passwordHint": "Create one in Nextcloud under Settings → Security → Devices & sessions.", + "folder": "Folder", + "schedule": "Upload schedule", + "scheduleOptions": { + "off": "Manual only", + "daily": "After every backup (daily)", + "weekly": "Weekly" + }, + "test": "Test connection", + "testPending": "Testing…", + "testOk": "Connection ok — folder is reachable.", + "save": "Save backup settings", + "savePending": "Saving…", + "saved": "Backup settings saved." + }, + "restore": { + "title": "Restore", + "open": "Restore a backup…", + "warning": "Restoring replaces ALL current content, files and settings of this instance with the state of the selected backup. The instance goes into maintenance mode while the restore runs.", + "loading": "Loading backup sets…", + "localTitle": "On this host", + "remoteTitle": "On Nextcloud", + "empty": "No restorable sets found.", + "remoteError": "The Nextcloud sets could not be listed: {{error}}", + "confirmLabel": "Type the backup id to confirm: {{id}}", + "button": "Restore this backup", + "pending": "Requesting restore…", + "requested": "Restore requested — the instance is entering maintenance mode.", + "status": { + "running": "Restore of {{id}} is running (started {{startedAt}})…", + "succeeded": "Restore of {{id}} succeeded ({{finishedAt}}).", + "failed": "Restore of {{id}} failed: {{error}}" + } + } }, "audit": { "title": "Audit log", @@ -107,7 +162,10 @@ "auth.login_failed": "Login failed", "auth.login_succeeded": "Login", "auth.password_reset": "Password reset", - "job.triggered": "Job triggered manually" + "job.triggered": "Job triggered manually", + "backup.settings_changed": "Backup settings changed", + "backup.run_triggered": "Manual backup triggered", + "backup.restore_requested": "Backup restore requested" } }, "storage": { @@ -123,5 +181,12 @@ "shared": "shared" }, "empty": "No usage recorded yet." + }, + "maintenance": { + "title": "Maintenance mode", + "body": "A backup is being restored. The instance is temporarily unavailable.", + "waiting": "Waiting for the instance to come back…", + "failed": "The restore failed: {{error}}", + "reload": "Reload" } } diff --git a/packages/shared/package.json b/packages/shared/package.json index a5fc8a9..134b3a5 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -18,13 +18,21 @@ "import": "./dist/token-crypto.mjs", "require": "./dist/token-crypto.js" }, + "./webdav": { + "types": "./dist/webdav.d.ts", + "import": "./dist/webdav.mjs", + "require": "./dist/webdav.js" + }, "./i18n/*": "./i18n/*" }, - "//": "typesVersions maps the ./token-crypto subpath for the api, which uses classic (node10) module resolution that ignores the exports field.", + "//": "typesVersions maps the server-only subpaths for the api, which uses classic (node10) module resolution that ignores the exports field.", "typesVersions": { "*": { "token-crypto": [ "./dist/token-crypto.d.ts" + ], + "webdav": [ + "./dist/webdav.d.ts" ] } }, @@ -33,7 +41,7 @@ "i18n" ], "scripts": { - "build": "tsup src/index.ts src/token-crypto.ts --format esm,cjs --dts --clean", + "build": "tsup src/index.ts src/token-crypto.ts src/webdav.ts --format esm,cjs --dts --clean", "typecheck": "tsc --noEmit", "test": "vitest run --passWithNoTests" }, diff --git a/packages/shared/src/backup-set.ts b/packages/shared/src/backup-set.ts new file mode 100644 index 0000000..766d1fc --- /dev/null +++ b/packages/shared/src/backup-set.ts @@ -0,0 +1,91 @@ +/** + * A restore set (ADR 0015) is one nightly `pg_dump` plus the matching + * uploads/plugins archive, tied together by a shared backup id derived from + * the run's UTC start time. This module owns the naming scheme and the pure + * prune decision; the sidecar's runner applies it to + * the filesystem, and the api reads it to list local sets for the in-app + * restore (issue #103). + */ + +export interface BackupSet { + id: string; + /** File names (not paths) present in the backups directory. */ + files: string[]; + /** Complete = both the dump and the volume archive exist. */ + complete: boolean; +} + +const ID_PATTERN = /^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})$/; +const SET_FILE_PATTERN = /^(?:db-|files-)(\d{8}-\d{6})\.(?:dump|tar\.gz)$/; + +/** Backup id for a run starting now: UTC timestamp, filesystem-safe. */ +export function newBackupId(now: Date): string { + const pad = (value: number): string => String(value).padStart(2, '0'); + return ( + `${now.getUTCFullYear()}${pad(now.getUTCMonth() + 1)}${pad(now.getUTCDate())}` + + `-${pad(now.getUTCHours())}${pad(now.getUTCMinutes())}${pad(now.getUTCSeconds())}` + ); +} + +/** The UTC time encoded in a backup id, or null for a malformed id. */ +export function backupIdTime(id: string): Date | null { + const match = ID_PATTERN.exec(id); + if (!match) return null; + const [, year, month, day, hour, minute, second] = match; + return new Date( + Date.UTC( + Number(year), + Number(month) - 1, + Number(day), + Number(hour), + Number(minute), + Number(second), + ), + ); +} + +export function dumpFileName(id: string): string { + return `db-${id}.dump`; +} + +export function archiveFileName(id: string): string { + return `files-${id}.tar.gz`; +} + +/** + * Groups the backup directory's file names into sets, oldest first. Files + * that do not belong to the naming scheme (status.json, `.partial` staging + * files of a running or crashed run) are ignored — prune never touches them. + */ +export function listSets(fileNames: string[]): BackupSet[] { + const byId = new Map(); + for (const name of fileNames) { + const match = SET_FILE_PATTERN.exec(name); + if (!match || !backupIdTime(match[1]!)) continue; + const files = byId.get(match[1]!) ?? []; + files.push(name); + byId.set(match[1]!, files); + } + return [...byId.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([id, files]) => ({ + id, + files: files.sort(), + complete: files.includes(dumpFileName(id)) && files.includes(archiveFileName(id)), + })); +} + +/** + * The sets prune may delete: older than the retention cutoff — but never + * the newest complete set, even when it is expired. A stalled instance must + * always keep one restorable set (issue #83 acceptance criteria). + */ +export function expiredSets(sets: BackupSet[], now: Date, retentionDays: number): BackupSet[] { + const cutoff = now.getTime() - retentionDays * 24 * 60 * 60 * 1000; + const newestComplete = [...sets].reverse().find((set) => set.complete); + return sets.filter((set) => { + if (set === newestComplete) return false; + const time = backupIdTime(set.id); + return time !== null && time.getTime() < cutoff; + }); +} diff --git a/packages/shared/src/backup-status.ts b/packages/shared/src/backup-status.ts index eea69f3..fee579b 100644 --- a/packages/shared/src/backup-status.ts +++ b/packages/shared/src/backup-status.ts @@ -15,6 +15,13 @@ export const BACKUP_STATUS_FILE = 'status.json'; */ export const BACKUP_FRESH_MAX_AGE_HOURS = 26; +/** + * Freshness bound of the off-host copy (issue #103) when the upload + * schedule is `weekly`: seven nightly cadences plus the same grace window. + * `daily` uploads share {@link BACKUP_FRESH_MAX_AGE_HOURS}. + */ +export const BACKUP_REMOTE_WEEKLY_MAX_AGE_HOURS = 7 * 24 + 2; + export interface BackupSizes { dumpBytes: number; archiveBytes: number; @@ -39,4 +46,96 @@ export interface BackupStatus { lastRun: BackupRun; /** Carried across failed runs so freshness checks see the real gap. */ lastSuccess: { backupId: string; finishedAt: string; sizes: BackupSizes } | null; + /** + * Off-host upload state (issue #103). Absent until a Nextcloud target was + * configured and the sidecar attempted its first upload; carried across + * runs like `lastSuccess` so a broken target keeps its last good copy + * visible. + */ + remote?: BackupRemoteStatus; +} + +/** One WebDAV upload attempt of a complete restore set (issue #103). */ +export interface BackupRemoteUpload { + backupId: string; + finishedAt: string; + outcome: 'succeeded' | 'failed'; + /** Present on failure: the first error the upload hit, as a plain string. */ + error?: string; + /** Present on success: size of the uploaded bundle. */ + sizeBytes?: number; +} + +export interface BackupRemoteStatus { + lastUpload: BackupRemoteUpload; + /** Carried across failed uploads — the newest copy known to exist remotely. */ + lastSuccessfulUpload: { backupId: string; finishedAt: string; sizeBytes: number } | null; +} + +/** + * The remote bundle naming scheme (issue #103): one self-contained archive + * per restore set, everything needed to rebuild an instance after total + * loss (`db-.dump`, `files-.tar.gz`, `manifest.json`). + */ +export function remoteBundleName(backupId: string): string { + return `dorfteich-backup-${backupId}.tar.gz`; +} + +const REMOTE_BUNDLE_PATTERN = /^dorfteich-backup-(\d{8}-\d{6})\.tar\.gz$/; + +/** The backup id encoded in a remote bundle name, or null for foreign files. */ +export function remoteBundleId(name: string): string | null { + return REMOTE_BUNDLE_PATTERN.exec(name)?.[1] ?? null; +} + +/** + * The in-app restore contract (issue #103): the sidecar orchestrates a + * restore and mirrors its progress into `restore-status.json` next to + * `status.json`. The api's maintenance gate answers 503 while `state` is + * `running`, and restarts itself once it flips to `succeeded`. + */ +export const RESTORE_STATUS_FILE = 'restore-status.json'; + +/** + * Safety valve: a `running` restore older than this is treated as crashed + * (sidecar died before writing a final state), so the api leaves maintenance + * mode instead of serving 503 forever. + */ +export const RESTORE_STALE_MAX_AGE_MINUTES = 30; + +export interface RestoreStatus { + schemaVersion: 1; + state: 'running' | 'succeeded' | 'failed'; + backupId: string; + source: 'local' | 'remote'; + /** Username of the requesting Site Admin (display only). */ + requestedBy: string | null; + startedAt: string; + finishedAt: string | null; + /** Present when `state` is `failed`. */ + error?: string; +} + +/** + * PostgreSQL `NOTIFY` channel over which the api sends commands to the + * backup sidecar (issue #103) — the same primitive as the api↔collab bus + * (collab-token.ts). Payload is a JSON {@link BackupCommand}. + */ +export const BACKUP_COMMAND_CHANNEL = 'backup_command'; + +export type BackupCommand = + | { kind: 'run'; requestedBy: string | null } + | { kind: 'restore'; source: 'local' | 'remote'; backupId: string; requestedBy: string | null }; + +/** + * PostgreSQL `NOTIFY` channel over which the backup sidecar announces + * maintenance-mode transitions during an in-app restore (issue #103). The + * collab server listens and closes/refuses live sessions so no in-memory + * document persists pre-restore content back over the restored database. + * Payload is a JSON {@link MaintenanceEvent}. + */ +export const BACKUP_MAINTENANCE_CHANNEL = 'backup_maintenance'; + +export interface MaintenanceEvent { + phase: 'enter' | 'exit'; } diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 7d91c44..b41a993 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1,6 +1,7 @@ export * from './admin-users'; export * from './api-error'; export * from './auth'; +export * from './backup-set'; export * from './backup-status'; export * from './collab-token'; export * from './comments'; diff --git a/packages/shared/src/system.ts b/packages/shared/src/system.ts index aaf7224..6bc2c0c 100644 --- a/packages/shared/src/system.ts +++ b/packages/shared/src/system.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; -import type { BackupStatus } from './backup-status'; +import type { BackupStatus, RestoreStatus } from './backup-status'; /** * Site-Admin system panel (issue #86): maintenance jobs, backup status, @@ -33,8 +33,93 @@ export interface SystemBackupView { fresh: boolean; status: BackupStatus | null; maxAgeHours: number; + /** Whether a Nextcloud target is fully configured (issue #103). */ + remoteConfigured: boolean; + /** Progress/result of the last in-app restore, if any (issue #103). */ + restore: RestoreStatus | null; } +/** + * Backup configuration surface of the admin settings page (issue #103). + * The app password is write-only: the view only says whether one is stored. + */ +export interface BackupSettingsView { + localRetentionDays: number | null; + remoteRetentionDays: number; + nextcloud: { + enabled: boolean; + baseUrl: string; + username: string; + folder: string; + uploadSchedule: 'off' | 'daily' | 'weekly'; + passwordSet: boolean; + }; +} + +export const backupSettingsInputSchema = z.object({ + /** `null` = no override; the sidecar's env value stays authoritative. */ + localRetentionDays: z.number().int().min(1).max(3650).nullable(), + remoteRetentionDays: z.number().int().min(1).max(3650), + nextcloud: z.object({ + enabled: z.boolean(), + baseUrl: z.string().trim().url({ message: 'validation.url' }).or(z.literal('')), + username: z.string().trim().max(200), + folder: z.string().trim().min(1).max(500), + uploadSchedule: z.enum(['off', 'daily', 'weekly']), + /** Empty or omitted = keep the stored app password. */ + password: z.string().max(500).optional(), + }), +}); + +export type BackupSettingsInput = z.infer; + +export const backupConnectionTestInputSchema = z.object({ + baseUrl: z.string().trim().url({ message: 'validation.url' }), + username: z.string().trim().min(1).max(200), + folder: z.string().trim().min(1).max(500), + /** Empty or omitted = test with the stored app password. */ + password: z.string().max(500).optional(), +}); + +export type BackupConnectionTestInput = z.infer; + +export interface BackupConnectionTestResult { + ok: boolean; + /** Raw transport/server error for the admin, like the SMTP test (#80). */ + error?: string; +} + +/** One restorable set as offered in the restore picker (issue #103). */ +export interface BackupSetView { + backupId: string; + /** UTC run start derived from the backup id. */ + startedAt: string; + /** Bundle size (remote) or dump+archive sum (local); null when unknown. */ + sizeBytes: number | null; +} + +export interface BackupSetsView { + local: BackupSetView[]; + remoteConfigured: boolean; + remote: BackupSetView[]; + /** Present when a configured remote target could not be listed. */ + remoteError?: string; +} + +export const backupIdSchema = z.string().regex(/^\d{8}-\d{6}$/); + +export const backupRestoreInputSchema = z.object({ + source: z.enum(['local', 'remote']), + backupId: backupIdSchema, + /** Type-to-confirm safety: must repeat the backup id verbatim. */ + confirm: z.string(), +}); + +export type BackupRestoreInput = z.infer; + +/** Response of the public, maintenance-exempt restore status endpoint. */ +export type RestoreStatusResponse = { state: 'idle' } | RestoreStatus; + export interface AuditActorView { id: string; username: string; diff --git a/packages/shared/src/webdav.test.ts b/packages/shared/src/webdav.test.ts new file mode 100644 index 0000000..e1044e7 --- /dev/null +++ b/packages/shared/src/webdav.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from 'vitest'; + +import { + basicAuthHeader, + folderSegments, + parsePropfind, + webdavCheck, + webdavFileUrl, + webdavFolderUrl, + webdavList, +} from './webdav'; + +/** A trimmed real-world Nextcloud PROPFIND (depth 1) multistatus body. */ +const NEXTCLOUD_PROPFIND = ` + + + /remote.php/dav/files/backupuser/dorfteich-backups/ + + + Sat, 11 Jul 2026 03:00:22 GMT + + + HTTP/1.1 200 OK + + + + /remote.php/dav/files/backupuser/dorfteich-backups/dorfteich-backup-20260711-030001.tar.gz + + + Sat, 11 Jul 2026 03:00:22 GMT + 1048576 + + + HTTP/1.1 200 OK + + + + /remote.php/dav/files/backupuser/dorfteich-backups/notes%20%26%20misc + + + + + HTTP/1.1 200 OK + + +`; + +const target = { + baseUrl: 'https://cloud.example.com', + username: 'backupuser', + password: 'app-password', + folder: 'dorfteich-backups', +}; + +describe('webdav url derivation', () => { + it('derives the Nextcloud DAV path from the base url', () => { + expect(webdavFolderUrl(target)).toBe( + 'https://cloud.example.com/remote.php/dav/files/backupuser/dorfteich-backups', + ); + }); + + it('keeps an explicit DAV base verbatim (generic WebDAV servers)', () => { + expect(webdavFolderUrl({ ...target, baseUrl: 'https://dav.example.com/dav/home/' })).toBe( + 'https://dav.example.com/dav/home/dorfteich-backups', + ); + }); + + it('encodes folder segments and file names', () => { + expect(webdavFileUrl({ ...target, folder: 'backups/my instance' }, 'a b.tar.gz')).toBe( + 'https://cloud.example.com/remote.php/dav/files/backupuser/backups/my%20instance/a%20b.tar.gz', + ); + }); + + it('rejects traversal segments and slashes in file names', () => { + expect(() => folderSegments('../etc')).toThrow(/segments/); + expect(() => webdavFileUrl(target, 'x/y')).toThrow(/file name/); + }); +}); + +describe('parsePropfind', () => { + it('parses a Nextcloud multistatus and skips the folder itself', () => { + const entries = parsePropfind( + NEXTCLOUD_PROPFIND, + '/remote.php/dav/files/backupuser/dorfteich-backups/', + ); + expect(entries).toEqual([ + { + name: 'dorfteich-backup-20260711-030001.tar.gz', + isCollection: false, + sizeBytes: 1_048_576, + lastModified: 'Sat, 11 Jul 2026 03:00:22 GMT', + }, + { + name: 'notes & misc', + isCollection: true, + sizeBytes: null, + lastModified: null, + }, + ]); + }); + + it('tolerates uppercase and missing namespace prefixes', () => { + const xml = ` + /dav/f/file.bin + 7 + `; + expect(parsePropfind(xml, '/dav/f/')).toEqual([ + { name: 'file.bin', isCollection: false, sizeBytes: 7, lastModified: null }, + ]); + }); +}); + +describe('webdavCheck', () => { + it('creates missing folder segments via MKCOL', async () => { + const calls: { url: string; method: string }[] = []; + const fetchLike = async (url: string, init?: RequestInit): Promise => { + const method = init?.method ?? 'GET'; + calls.push({ url, method }); + if (method === 'PROPFIND' && url.endsWith('/dorfteich-backups')) { + return new Response('', { status: 404 }); + } + return new Response('', { status: 207 }); + }; + const result = await webdavCheck(target, fetchLike); + expect(result.ok).toBe(true); + expect(calls.map((c) => c.method)).toEqual(['PROPFIND', 'PROPFIND', 'MKCOL']); + }); + + it('reports authentication failures readably', async () => { + const fetchLike = async (): Promise => + new Response('', { status: 401, statusText: 'Unauthorized' }); + const result = await webdavCheck(target, fetchLike); + expect(result).toEqual({ + ok: false, + error: expect.stringContaining('authentication failed') as unknown as string, + }); + }); + + it('turns thrown network errors into readable results', async () => { + const fetchLike = async (): Promise => { + throw new Error('getaddrinfo ENOTFOUND cloud.example.com'); + }; + const result = await webdavCheck(target, fetchLike); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain('ENOTFOUND'); + }); +}); + +describe('webdavList', () => { + it('sends PROPFIND depth 1 with basic auth and parses the body', async () => { + let seen: RequestInit | undefined; + const fetchLike = async (_url: string, init?: RequestInit): Promise => { + seen = init; + return new Response(NEXTCLOUD_PROPFIND, { status: 207 }); + }; + const result = await webdavList(target, fetchLike); + expect(result.ok).toBe(true); + if (result.ok) expect(result.value).toHaveLength(2); + expect((seen?.headers as Record).Depth).toBe('1'); + expect((seen?.headers as Record).Authorization).toBe(basicAuthHeader(target)); + }); +}); diff --git a/packages/shared/src/webdav.ts b/packages/shared/src/webdav.ts new file mode 100644 index 0000000..aefb319 --- /dev/null +++ b/packages/shared/src/webdav.ts @@ -0,0 +1,273 @@ +/** + * Minimal WebDAV client for the Nextcloud backup target (issue #103), shared + * by the api (connection test, remote set listing) and the backup sidecar + * (upload, prune, download). Plain `fetch` with basic auth — no heavy DAV + * dependency; the subset used here (PROPFIND/MKCOL/PUT/GET/DELETE) is stable + * across Nextcloud versions and generic WebDAV servers. + * + * Like `token-crypto`, this module is a separate package entry (not part of + * the barrel) because it is server-only. + */ + +export interface WebDavTarget { + /** + * The Nextcloud base URL (e.g. `https://cloud.example.com`) — the DAV + * path `remote.php/dav/files/` is derived. A URL that already + * contains `remote.php` or `/dav/` is used as the DAV base verbatim, so + * generic WebDAV servers work too. + */ + baseUrl: string; + username: string; + password: string; + /** Target folder under the DAV base, may contain `/` for nesting. */ + folder: string; +} + +export interface WebDavEntry { + /** Decoded file or collection name (last path segment). */ + name: string; + isCollection: boolean; + sizeBytes: number | null; + lastModified: string | null; +} + +export type WebDavResult = { ok: true; value: T } | { ok: false; error: string }; + +type FetchLike = (url: string, init?: RequestInit) => Promise; + +/** Sane bound for a wrong host that answers slowly — uploads set their own. */ +const REQUEST_TIMEOUT_MS = 20_000; + +function trimSlashes(value: string): string { + return value.replace(/^\/+|\/+$/g, ''); +} + +/** Folder segments, each URI-encoded; rejects `.`/`..` traversal segments. */ +export function folderSegments(folder: string): string[] { + const segments = trimSlashes(folder.trim()) + .split('/') + .filter((segment) => segment.length > 0); + if (segments.some((segment) => segment === '.' || segment === '..')) { + throw new Error('folder must not contain "." or ".." segments'); + } + return segments; +} + +/** The DAV base URL (without the folder), normalized without trailing slash. */ +export function webdavBaseUrl(target: Pick): string { + const base = target.baseUrl.replace(/\/+$/, ''); + if (/remote\.php|\/dav\//i.test(base)) return base; + return `${base}/remote.php/dav/files/${encodeURIComponent(target.username)}`; +} + +/** Absolute URL of the target folder (no trailing slash). */ +export function webdavFolderUrl(target: WebDavTarget): string { + const segments = folderSegments(target.folder).map(encodeURIComponent); + return [webdavBaseUrl(target), ...segments].join('/'); +} + +/** Absolute URL of a file inside the target folder. */ +export function webdavFileUrl(target: WebDavTarget, name: string): string { + if (name.includes('/')) throw new Error('file name must not contain "/"'); + return `${webdavFolderUrl(target)}/${encodeURIComponent(name)}`; +} + +export function basicAuthHeader(target: Pick): string { + const credentials = `${target.username}:${target.password}`; + return `Basic ${Buffer.from(credentials, 'utf8').toString('base64')}`; +} + +function decodeXmlEntities(value: string): string { + return value + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&#(\d+);/g, (_, code: string) => String.fromCodePoint(Number(code))) + .replace(/&/g, '&'); +} + +/** First text content of `` inside `block`, prefix-agnostic. */ +function xmlText(block: string, tag: string): string | null { + const match = new RegExp(`<(?:[A-Za-z0-9-]+:)?${tag}[^>]*>([^<]*)][\s\S]*?<\/(?:[A-Za-z0-9-]+:)?response>/gi) ?? []; + const normalizedRequest = trimSlashes(decodeURIComponent(requestPath)); + for (const block of responseBlocks) { + const href = xmlText(block, 'href'); + if (!href) continue; + const path = trimSlashes(decodeURIComponent(href)); + if (path === normalizedRequest) continue; + const name = path.split('/').pop() ?? ''; + if (!name) continue; + const lengthText = xmlText(block, 'getcontentlength'); + const sizeBytes = lengthText !== null && /^\d+$/.test(lengthText) ? Number(lengthText) : null; + entries.push({ + name, + isCollection: /<(?:[A-Za-z0-9-]+:)?collection\b/i.test(block), + sizeBytes, + lastModified: xmlText(block, 'getlastmodified'), + }); + } + return entries; +} + +async function davRequest( + fetchLike: FetchLike, + target: WebDavTarget, + url: string, + init: RequestInit & { timeoutMs?: number }, +): Promise { + const { timeoutMs, ...rest } = init; + return fetchLike(url, { + ...rest, + headers: { + Authorization: basicAuthHeader(target), + ...(rest.headers ?? {}), + }, + signal: AbortSignal.timeout(timeoutMs ?? REQUEST_TIMEOUT_MS), + }); +} + +function describeFailure(action: string, response: Response): string { + const auth = response.status === 401 || response.status === 403; + return auth + ? `${action}: authentication failed (HTTP ${response.status}) — check username and app password` + : `${action}: HTTP ${response.status} ${response.statusText}`.trim(); +} + +function describeError(action: string, error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return `${action}: ${message}`; +} + +/** + * Verifies the target is usable: credentials accepted and the folder exists, + * creating missing folder segments via MKCOL on the way (like the setup + * wizard's SMTP test, nothing else is touched). + */ +export async function webdavCheck( + target: WebDavTarget, + fetchLike: FetchLike = fetch, +): Promise> { + let url = webdavBaseUrl(target); + try { + const probe = await davRequest(fetchLike, target, url, { + method: 'PROPFIND', + headers: { Depth: '0' }, + }); + if (!probe.ok) return { ok: false, error: describeFailure('connect', probe) }; + + for (const segment of folderSegments(target.folder)) { + url = `${url}/${encodeURIComponent(segment)}`; + const exists = await davRequest(fetchLike, target, url, { + method: 'PROPFIND', + headers: { Depth: '0' }, + }); + if (exists.ok) continue; + if (exists.status !== 404) + return { ok: false, error: describeFailure('check folder', exists) }; + const created = await davRequest(fetchLike, target, url, { method: 'MKCOL' }); + if (!created.ok) return { ok: false, error: describeFailure('create folder', created) }; + } + return { ok: true, value: undefined }; + } catch (error) { + return { ok: false, error: describeError('connect', error) }; + } +} + +/** Lists the target folder (depth 1), excluding sub-collections' contents. */ +export async function webdavList( + target: WebDavTarget, + fetchLike: FetchLike = fetch, +): Promise> { + const url = webdavFolderUrl(target); + try { + const response = await davRequest(fetchLike, target, url, { + method: 'PROPFIND', + headers: { Depth: '1' }, + }); + if (!response.ok) return { ok: false, error: describeFailure('list folder', response) }; + const requestPath = new URL(url).pathname; + return { ok: true, value: parsePropfind(await response.text(), requestPath) }; + } catch (error) { + return { ok: false, error: describeError('list folder', error) }; + } +} + +/** + * Uploads a file into the target folder. The body is a Buffer or a byte + * stream (pass `contentLength` for streams so the server can reject early + * on quota). No practical timeout — bundles can be large and slow links are + * fine; the caller's run wraps the whole upload. + */ +export async function webdavPut( + target: WebDavTarget, + name: string, + body: Buffer | ReadableStream, + options: { contentLength?: number; timeoutMs?: number } = {}, + fetchLike: FetchLike = fetch, +): Promise> { + try { + const response = await davRequest(fetchLike, target, webdavFileUrl(target, name), { + method: 'PUT', + body: body as RequestInit['body'], + headers: options.contentLength ? { 'Content-Length': String(options.contentLength) } : {}, + timeoutMs: options.timeoutMs ?? 6 * 60 * 60 * 1000, + // Node's fetch requires half-duplex for streamed request bodies. + ...(body instanceof Buffer ? {} : { duplex: 'half' as const }), + } as RequestInit & { timeoutMs?: number }); + if (!response.ok) return { ok: false, error: describeFailure(`upload ${name}`, response) }; + return { ok: true, value: undefined }; + } catch (error) { + return { ok: false, error: describeError(`upload ${name}`, error) }; + } +} + +/** Fetches a file; the caller streams `response.body` to disk. */ +export async function webdavGet( + target: WebDavTarget, + name: string, + fetchLike: FetchLike = fetch, +): Promise> { + try { + const response = await davRequest(fetchLike, target, webdavFileUrl(target, name), { + method: 'GET', + timeoutMs: 6 * 60 * 60 * 1000, + }); + if (!response.ok) return { ok: false, error: describeFailure(`download ${name}`, response) }; + return { ok: true, value: response }; + } catch (error) { + return { ok: false, error: describeError(`download ${name}`, error) }; + } +} + +export async function webdavDelete( + target: WebDavTarget, + name: string, + fetchLike: FetchLike = fetch, +): Promise> { + try { + const response = await davRequest(fetchLike, target, webdavFileUrl(target, name), { + method: 'DELETE', + }); + // 404 counts as deleted — prune must be idempotent across retries. + if (!response.ok && response.status !== 404) { + return { ok: false, error: describeFailure(`delete ${name}`, response) }; + } + return { ok: true, value: undefined }; + } catch (error) { + return { ok: false, error: describeError(`delete ${name}`, error) }; + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6151895..e4a991d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -180,6 +180,9 @@ importers: nodemailer: specifier: ^9.0.3 version: 9.0.3 + pg: + specifier: ^8.16.0 + version: 8.22.0 pino: specifier: ^9.6.0 version: 9.14.0 @@ -193,6 +196,9 @@ importers: '@types/nodemailer': specifier: ^8.0.1 version: 8.0.1 + '@types/pg': + specifier: ^8.11.0 + version: 8.20.0 tsx: specifier: ^4.19.0 version: 4.23.0