All checks were successful
CI / Lint, typecheck, test (push) Successful in 3m45s
CD / Build and push images (push) Successful in 3m49s
CI / Build container images (push) Has been skipped
CD / Deploy to Test (push) Successful in 9s
CD / Smoke tests against Test (push) Successful in 1m18s
CD / Promote to Int (push) Successful in 11s
CI / Auth e2e pack (push) Successful in 5m35s
CI / Import/export fidelity gate (push) Successful in 47s
Off-host backups for every self-hoster, configured entirely in the admin UI — supersedes the host-specific mirror plan behind #84. shared: - webdav.ts (new package entry like token-crypto): minimal WebDAV client with basic auth — PROPFIND (tolerant multistatus parser), MKCOL, PUT (streamed), GET, DELETE; Nextcloud DAV path derived from the plain server URL, explicit DAV bases pass through - backup-status.ts: additive remote-upload status in status.json, the restore-status.json contract (running/succeeded/failed + staleness bound), the backup_command/backup_maintenance NOTIFY channels, and the one-bundle-per-set naming (dorfteich-backup-<id>.tar.gz) - backup-set.ts moved here from apps/backup (api lists local sets) backup sidecar: - reads the backup.* instance settings directly from the database (admin changes apply next run; local retention row overrides the env) and the app password from the secret store - after each successful set: bundle dump + files archive + manifest into ONE self-contained tar.gz, upload via WebDAV per schedule (off/daily/weekly; manual runs always upload), prune remote bundles — never the newest — and record the outcome in status.json; upload failures alert via a new backupUploadFailed mail (de+en) - command listener on backup_command (run / restore) with a serial queue against the nightly timer - restore orchestrator: restore-status.json → maintenance NOTIFY → grace → (remote: download + manifest-verify bundle) → terminate other DB connections → shared perform-restore path (same code as restore.sh) → final status + maintenance exit api: - MaintenanceGuard (global, registered before the setup gate): 503 maintenance_mode while restore-status says running; health endpoints and the new public GET /backup/restore-status stay exempt; a stale running state (crashed sidecar) unblocks after 30 min - MaintenanceStateService watches the file and restarts the api after a successful restore (fresh caches, migrate-on-start for older dumps); main.ts refuses to touch the database while a restore runs — a container restarting mid-restore must not race pg_restore with migrate deploy - worker sweeps (conversion, mail outbox, scheduler) catch transient database failures instead of dying on an unhandled rejection — the restore's connection termination crashed the api in verification - backup admin endpoints under /admin/system/backup: settings (live connection test before save, password write-only into the secret store), nextcloud/test, sets (local via the ro backups mount + remote via WebDAV), run + restore (type-to-confirm backstop, source validation) — commands travel as NOTIFY payloads; audit actions backup.settings_changed/run_triggered/restore_requested - readyz: new warning-level backup_remote check while a target is configured (26 h daily / 170 h weekly bound) collab: - maintenance listener: on enter, persist + close every live session and refuse new connections until exit (failsafe timeout 30 min) — no in-memory document may write pre-restore content back afterwards web: - Admin → System backup section: status card with remote facts and a "Back up now" button, the Nextcloud settings form with test button, and the restore picker (local + remote sets, type-to-confirm) - global maintenance screen: any 503 maintenance_mode flips the SPA to a status page polling the exempt endpoint, reloading when the instance returns Verified end-to-end against a live stack (fresh DB, native api + sidecar, fake WebDAV server): configure → test → manual backup → bundle upload → readyz/sets/status surfaces → remote restore with maintenance gate, marker rollback and api restart; suites: shared 21, backup 9, collab 11, api 58 files green, lint + i18n:check + typecheck clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1
406 lines
14 KiB
TypeScript
406 lines
14 KiB
TypeScript
import { createServer, type Server } from 'node:http';
|
|
import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { setTimeout as sleep } from 'node:timers/promises';
|
|
|
|
import { INestApplication } from '@nestjs/common';
|
|
import {
|
|
BACKUP_COMMAND_CHANNEL,
|
|
RESTORE_STATUS_FILE,
|
|
archiveFileName,
|
|
dumpFileName,
|
|
type BackupCommand,
|
|
type BackupSetsView,
|
|
type BackupSettingsView,
|
|
type RestoreStatus,
|
|
type SystemBackupView,
|
|
} from '@dorfteich/shared';
|
|
import { PrismaClient } from '@prisma/client';
|
|
import { Client } from 'pg';
|
|
import request from 'supertest';
|
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
|
|
import { createTestApp, sessionCookieOf } from '../testing/test-app';
|
|
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
|
|
import { UsersService } from '../users/users.service';
|
|
|
|
/**
|
|
* A fake Nextcloud endpoint covering the WebDAV subset the admin endpoints
|
|
* use (PROPFIND/MKCOL) plus a remote bundle listing — the connection test
|
|
* and the restore picker run against real HTTP.
|
|
*/
|
|
function createDavServer(): {
|
|
server: Server;
|
|
start(): Promise<string>;
|
|
stop(): Promise<void>;
|
|
setAuthOk(ok: boolean): void;
|
|
} {
|
|
let authOk = true;
|
|
const files = ['dorfteich-backup-20260710-030000.tar.gz'];
|
|
const server = createServer((req, res) => {
|
|
if (!authOk) {
|
|
res.statusCode = 401;
|
|
return res.end();
|
|
}
|
|
if (req.method === 'PROPFIND') {
|
|
const depth = req.headers.depth;
|
|
res.statusCode = 207;
|
|
res.setHeader('Content-Type', 'application/xml');
|
|
const children =
|
|
depth === '1'
|
|
? files
|
|
.map(
|
|
(name) => `<d:response><d:href>${req.url}/${name}</d:href><d:propstat><d:prop>
|
|
<d:getcontentlength>2048</d:getcontentlength><d:resourcetype/>
|
|
</d:prop></d:propstat></d:response>`,
|
|
)
|
|
.join('')
|
|
: '';
|
|
return res.end(
|
|
`<?xml version="1.0"?><d:multistatus xmlns:d="DAV:">
|
|
<d:response><d:href>${req.url}/</d:href><d:propstat><d:prop>
|
|
<d:resourcetype><d:collection/></d:resourcetype>
|
|
</d:prop></d:propstat></d:response>${children}</d:multistatus>`,
|
|
);
|
|
}
|
|
if (req.method === 'MKCOL') {
|
|
res.statusCode = 201;
|
|
return res.end();
|
|
}
|
|
res.statusCode = 405;
|
|
res.end();
|
|
});
|
|
return {
|
|
server,
|
|
setAuthOk: (ok) => {
|
|
authOk = ok;
|
|
},
|
|
start: () =>
|
|
new Promise((resolve) => {
|
|
server.listen(0, '127.0.0.1', () => {
|
|
resolve(`http://127.0.0.1:${(server.address() as { port: number }).port}`);
|
|
});
|
|
}),
|
|
stop: () => new Promise((resolve) => server.close(() => resolve())),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Backup administration end to end (issue #103): settings roundtrip with
|
|
* the app password landing in the secret store (never the database), the
|
|
* live connection test, the restore picker's set listing, command NOTIFYs
|
|
* for run/restore, the public restore-status endpoint, and the maintenance
|
|
* gate's 503 semantics including staleness.
|
|
*/
|
|
describe.skipIf(!hasTestDb)('backup admin (e2e, issue #103)', () => {
|
|
let app: INestApplication;
|
|
let prisma: PrismaClient;
|
|
let backupsDir: string;
|
|
let secretsFile: string;
|
|
let davUrl: string;
|
|
const dav = createDavServer();
|
|
const suffix = uniqueSuffix();
|
|
const password = 'backupadmin ist vorsichtig 1';
|
|
const ids: Record<string, string> = {};
|
|
const cookies: Record<string, string> = {};
|
|
const baseSecretsFile = process.env.SECRETS_FILE;
|
|
|
|
const commands: BackupCommand[] = [];
|
|
let listenClient: Client;
|
|
|
|
const api = () => request(app.getHttpServer());
|
|
|
|
async function makeUser(handle: string, siteAdmin: boolean): Promise<void> {
|
|
const users = app.get(UsersService);
|
|
const username = `bak-${handle}-${suffix}`;
|
|
const user = await users.createUser({
|
|
username,
|
|
email: `${username}@example.org`,
|
|
displayName: `Bak ${handle}`,
|
|
password,
|
|
locale: 'en',
|
|
});
|
|
await users.markEmailVerified(user.id);
|
|
if (siteAdmin)
|
|
await prisma.user.update({ where: { id: user.id }, data: { isSiteAdmin: true } });
|
|
ids[handle] = user.id;
|
|
cookies[handle] = sessionCookieOf(
|
|
await api()
|
|
.post('/api/v1/auth/login')
|
|
.send({ usernameOrEmail: username, password })
|
|
.expect(200),
|
|
);
|
|
}
|
|
|
|
function settingsInput(overrides: Record<string, unknown> = {}): Record<string, unknown> {
|
|
return {
|
|
localRetentionDays: 14,
|
|
remoteRetentionDays: 60,
|
|
nextcloud: {
|
|
enabled: true,
|
|
baseUrl: davUrl,
|
|
username: 'clouduser',
|
|
folder: `dorfteich-e2e-${suffix}`,
|
|
uploadSchedule: 'weekly',
|
|
password: 'app-password-123',
|
|
...((overrides.nextcloud as object) ?? {}),
|
|
},
|
|
...Object.fromEntries(Object.entries(overrides).filter(([k]) => k !== 'nextcloud')),
|
|
};
|
|
}
|
|
|
|
beforeAll(async () => {
|
|
backupsDir = mkdtempSync(join(tmpdir(), 'dorfteich-backup-admin-'));
|
|
secretsFile = join(mkdtempSync(join(tmpdir(), 'dorfteich-backup-secrets-')), 'secrets.env');
|
|
process.env.BACKUPS_DIR = backupsDir;
|
|
process.env.SECRETS_FILE = secretsFile;
|
|
davUrl = await dav.start();
|
|
prisma = createTestPrisma();
|
|
await prisma.rateLimit.deleteMany({});
|
|
// Clean leftovers of earlier runs — the settings keys are singletons.
|
|
await prisma.instanceSetting.deleteMany({ where: { key: { startsWith: 'backup.' } } });
|
|
app = await createTestApp();
|
|
await makeUser('admin', true);
|
|
await makeUser('user', false);
|
|
|
|
listenClient = new Client({ connectionString: process.env.TEST_DATABASE_URL });
|
|
await listenClient.connect();
|
|
listenClient.on('notification', (message) => {
|
|
if (message.channel === BACKUP_COMMAND_CHANNEL && message.payload) {
|
|
commands.push(JSON.parse(message.payload) as BackupCommand);
|
|
}
|
|
});
|
|
await listenClient.query(`LISTEN ${BACKUP_COMMAND_CHANNEL}`);
|
|
});
|
|
|
|
afterAll(async () => {
|
|
delete process.env.BACKUPS_DIR;
|
|
if (baseSecretsFile === undefined) delete process.env.SECRETS_FILE;
|
|
else process.env.SECRETS_FILE = baseSecretsFile;
|
|
await dav.stop();
|
|
await listenClient.end().catch(() => undefined);
|
|
const all = Object.values(ids);
|
|
await prisma.instanceSetting.deleteMany({ where: { key: { startsWith: 'backup.' } } });
|
|
await prisma.auditEntry.deleteMany({ where: { actorId: { in: all } } });
|
|
await prisma.session.deleteMany({ where: { userId: { in: all } } });
|
|
await prisma.userIdentity.deleteMany({ where: { userId: { in: all } } });
|
|
await prisma.user.deleteMany({ where: { id: { in: all } } });
|
|
await prisma.$disconnect();
|
|
await app.close();
|
|
});
|
|
|
|
it('rejects non-admins on every backup admin route', async () => {
|
|
for (const [method, path] of [
|
|
['get', '/api/v1/admin/system/backup/settings'],
|
|
['put', '/api/v1/admin/system/backup/settings'],
|
|
['post', '/api/v1/admin/system/backup/nextcloud/test'],
|
|
['get', '/api/v1/admin/system/backup/sets'],
|
|
['post', '/api/v1/admin/system/backup/run'],
|
|
['post', '/api/v1/admin/system/backup/restore'],
|
|
] as const) {
|
|
await api()[method](path).set('Cookie', cookies.user!).expect(403);
|
|
}
|
|
});
|
|
|
|
it('tests the connection against a live WebDAV endpoint', async () => {
|
|
const ok = await api()
|
|
.post('/api/v1/admin/system/backup/nextcloud/test')
|
|
.set('Cookie', cookies.admin!)
|
|
.send({
|
|
baseUrl: davUrl,
|
|
username: 'clouduser',
|
|
folder: 'dorfteich-e2e',
|
|
password: 'app-password-123',
|
|
})
|
|
.expect(200);
|
|
expect(ok.body).toEqual({ ok: true });
|
|
|
|
dav.setAuthOk(false);
|
|
const bad = await api()
|
|
.post('/api/v1/admin/system/backup/nextcloud/test')
|
|
.set('Cookie', cookies.admin!)
|
|
.send({
|
|
baseUrl: davUrl,
|
|
username: 'clouduser',
|
|
folder: 'dorfteich-e2e',
|
|
password: 'wrong',
|
|
})
|
|
.expect(200);
|
|
expect(bad.body.ok).toBe(false);
|
|
expect(String(bad.body.error)).toContain('authentication failed');
|
|
dav.setAuthOk(true);
|
|
});
|
|
|
|
it('refuses to save an enabled target that fails the connection test', async () => {
|
|
dav.setAuthOk(false);
|
|
const res = await api()
|
|
.put('/api/v1/admin/system/backup/settings')
|
|
.set('Cookie', cookies.admin!)
|
|
.send(settingsInput())
|
|
.expect(400);
|
|
expect(res.body.code).toBe('backup_connection_failed');
|
|
dav.setAuthOk(true);
|
|
// Nothing was persisted.
|
|
const view = await api()
|
|
.get('/api/v1/admin/system/backup/settings')
|
|
.set('Cookie', cookies.admin!)
|
|
.expect(200);
|
|
expect((view.body as BackupSettingsView).nextcloud.enabled).toBe(false);
|
|
});
|
|
|
|
it('saves settings, keeping the app password out of the database', async () => {
|
|
const res = await api()
|
|
.put('/api/v1/admin/system/backup/settings')
|
|
.set('Cookie', cookies.admin!)
|
|
.send(settingsInput())
|
|
.expect(200);
|
|
const view = res.body as BackupSettingsView;
|
|
expect(view).toMatchObject({
|
|
localRetentionDays: 14,
|
|
remoteRetentionDays: 60,
|
|
nextcloud: {
|
|
enabled: true,
|
|
baseUrl: davUrl,
|
|
username: 'clouduser',
|
|
uploadSchedule: 'weekly',
|
|
passwordSet: true,
|
|
},
|
|
});
|
|
|
|
// Secret store holds the password; instance_settings never does.
|
|
expect(readFileSync(secretsFile, 'utf8')).toMatch(
|
|
/BACKUP_NEXTCLOUD_PASSWORD="?app-password-123"?/,
|
|
);
|
|
const rows = await prisma.instanceSetting.findMany({
|
|
where: { key: { startsWith: 'backup.' } },
|
|
});
|
|
expect(JSON.stringify(rows.map((row) => row.value))).not.toContain('app-password-123');
|
|
|
|
// Re-saving without a password keeps the stored one (write-only field).
|
|
await api()
|
|
.put('/api/v1/admin/system/backup/settings')
|
|
.set('Cookie', cookies.admin!)
|
|
.send(settingsInput({ nextcloud: { password: undefined } }))
|
|
.expect(200);
|
|
expect(readFileSync(secretsFile, 'utf8')).toMatch(
|
|
/BACKUP_NEXTCLOUD_PASSWORD="?app-password-123"?/,
|
|
);
|
|
});
|
|
|
|
it('lists local and remote restore sets, and reflects the target on the card', async () => {
|
|
writeFileSync(join(backupsDir, dumpFileName('20260712-030000')), 'dump');
|
|
writeFileSync(join(backupsDir, archiveFileName('20260712-030000')), 'archive');
|
|
// An incomplete set never shows up as restorable.
|
|
writeFileSync(join(backupsDir, dumpFileName('20260712-040000')), 'dump-only');
|
|
|
|
const res = await api()
|
|
.get('/api/v1/admin/system/backup/sets')
|
|
.set('Cookie', cookies.admin!)
|
|
.expect(200);
|
|
const sets = res.body as BackupSetsView;
|
|
expect(sets.remoteConfigured).toBe(true);
|
|
expect(sets.local.map((s) => s.backupId)).toEqual(['20260712-030000']);
|
|
expect(sets.remote.map((s) => s.backupId)).toEqual(['20260710-030000']);
|
|
expect(sets.remote[0]!.sizeBytes).toBe(2048);
|
|
|
|
const card = await api()
|
|
.get('/api/v1/admin/system/backup')
|
|
.set('Cookie', cookies.admin!)
|
|
.expect(200);
|
|
expect((card.body as SystemBackupView).remoteConfigured).toBe(true);
|
|
});
|
|
|
|
it('sends run and restore commands over NOTIFY, audit-logged', async () => {
|
|
commands.length = 0;
|
|
await api().post('/api/v1/admin/system/backup/run').set('Cookie', cookies.admin!).expect(202);
|
|
|
|
await api()
|
|
.post('/api/v1/admin/system/backup/restore')
|
|
.set('Cookie', cookies.admin!)
|
|
.send({ source: 'local', backupId: '20260712-030000', confirm: '20260712-030000' })
|
|
.expect(202);
|
|
|
|
await sleep(300);
|
|
expect(commands).toEqual([
|
|
{ kind: 'run', requestedBy: `bak-admin-${suffix}` },
|
|
{
|
|
kind: 'restore',
|
|
source: 'local',
|
|
backupId: '20260712-030000',
|
|
requestedBy: `bak-admin-${suffix}`,
|
|
},
|
|
]);
|
|
|
|
const audit = await prisma.auditEntry.findMany({
|
|
where: { actorId: ids.admin!, action: { startsWith: 'backup.' } },
|
|
});
|
|
const actions = audit.map((entry) => entry.action);
|
|
expect(actions).toContain('backup.run_triggered');
|
|
expect(actions).toContain('backup.restore_requested');
|
|
});
|
|
|
|
it('guards the restore trigger: confirm mismatch, unknown sets, bad sources', async () => {
|
|
await api()
|
|
.post('/api/v1/admin/system/backup/restore')
|
|
.set('Cookie', cookies.admin!)
|
|
.send({ source: 'local', backupId: '20260712-030000', confirm: 'nope' })
|
|
.expect(400);
|
|
|
|
await api()
|
|
.post('/api/v1/admin/system/backup/restore')
|
|
.set('Cookie', cookies.admin!)
|
|
.send({ source: 'local', backupId: '20260712-040000', confirm: '20260712-040000' })
|
|
.expect(404);
|
|
|
|
await api()
|
|
.post('/api/v1/admin/system/backup/restore')
|
|
.set('Cookie', cookies.admin!)
|
|
.send({ source: 'remote', backupId: '20260712-050000', confirm: '20260712-050000' })
|
|
.expect(404);
|
|
});
|
|
|
|
it('serves the public restore status and gates the api during a restore', async () => {
|
|
// No restore yet → idle, and the api serves normally.
|
|
const idle = await api().get('/api/v1/backup/restore-status').expect(200);
|
|
expect(idle.body).toEqual({ state: 'idle' });
|
|
|
|
const running: RestoreStatus = {
|
|
schemaVersion: 1,
|
|
state: 'running',
|
|
backupId: '20260712-030000',
|
|
source: 'local',
|
|
requestedBy: 'admin',
|
|
startedAt: new Date().toISOString(),
|
|
finishedAt: null,
|
|
};
|
|
writeFileSync(join(backupsDir, RESTORE_STATUS_FILE), JSON.stringify(running));
|
|
await sleep(1600); // maintenance state cache TTL
|
|
|
|
// Anonymous status endpoint keeps answering; everything else 503s.
|
|
const status = await api().get('/api/v1/backup/restore-status').expect(200);
|
|
expect((status.body as RestoreStatus).state).toBe('running');
|
|
const gated = await api().get('/api/v1/ponds').set('Cookie', cookies.admin!).expect(503);
|
|
expect(gated.body.code).toBe('maintenance_mode');
|
|
await api().get('/api/v1/healthz').expect(200);
|
|
|
|
// A crashed restore (stale running state) must not brick the instance.
|
|
writeFileSync(
|
|
join(backupsDir, RESTORE_STATUS_FILE),
|
|
JSON.stringify({ ...running, startedAt: new Date(Date.now() - 31 * 60_000).toISOString() }),
|
|
);
|
|
await sleep(1600);
|
|
await api().get('/api/v1/ponds').set('Cookie', cookies.admin!).expect(200);
|
|
|
|
// A finished restore leaves the gate open and reports its result.
|
|
writeFileSync(
|
|
join(backupsDir, RESTORE_STATUS_FILE),
|
|
JSON.stringify({ ...running, state: 'succeeded', finishedAt: new Date().toISOString() }),
|
|
);
|
|
await sleep(1600);
|
|
const done = await api().get('/api/v1/backup/restore-status').expect(200);
|
|
expect((done.body as RestoreStatus).state).toBe('succeeded');
|
|
await api().get('/api/v1/ponds').set('Cookie', cookies.admin!).expect(200);
|
|
});
|
|
});
|