All checks were successful
CI / Lint, typecheck, test (push) Successful in 3m45s
CD / Build and push images (push) Successful in 3m49s
CI / Build container images (push) Has been skipped
CD / Deploy to Test (push) Successful in 9s
CD / Smoke tests against Test (push) Successful in 1m18s
CD / Promote to Int (push) Successful in 11s
CI / Auth e2e pack (push) Successful in 5m35s
CI / Import/export fidelity gate (push) Successful in 47s
Off-host backups for every self-hoster, configured entirely in the admin UI — supersedes the host-specific mirror plan behind #84. shared: - webdav.ts (new package entry like token-crypto): minimal WebDAV client with basic auth — PROPFIND (tolerant multistatus parser), MKCOL, PUT (streamed), GET, DELETE; Nextcloud DAV path derived from the plain server URL, explicit DAV bases pass through - backup-status.ts: additive remote-upload status in status.json, the restore-status.json contract (running/succeeded/failed + staleness bound), the backup_command/backup_maintenance NOTIFY channels, and the one-bundle-per-set naming (dorfteich-backup-<id>.tar.gz) - backup-set.ts moved here from apps/backup (api lists local sets) backup sidecar: - reads the backup.* instance settings directly from the database (admin changes apply next run; local retention row overrides the env) and the app password from the secret store - after each successful set: bundle dump + files archive + manifest into ONE self-contained tar.gz, upload via WebDAV per schedule (off/daily/weekly; manual runs always upload), prune remote bundles — never the newest — and record the outcome in status.json; upload failures alert via a new backupUploadFailed mail (de+en) - command listener on backup_command (run / restore) with a serial queue against the nightly timer - restore orchestrator: restore-status.json → maintenance NOTIFY → grace → (remote: download + manifest-verify bundle) → terminate other DB connections → shared perform-restore path (same code as restore.sh) → final status + maintenance exit api: - MaintenanceGuard (global, registered before the setup gate): 503 maintenance_mode while restore-status says running; health endpoints and the new public GET /backup/restore-status stay exempt; a stale running state (crashed sidecar) unblocks after 30 min - MaintenanceStateService watches the file and restarts the api after a successful restore (fresh caches, migrate-on-start for older dumps); main.ts refuses to touch the database while a restore runs — a container restarting mid-restore must not race pg_restore with migrate deploy - worker sweeps (conversion, mail outbox, scheduler) catch transient database failures instead of dying on an unhandled rejection — the restore's connection termination crashed the api in verification - backup admin endpoints under /admin/system/backup: settings (live connection test before save, password write-only into the secret store), nextcloud/test, sets (local via the ro backups mount + remote via WebDAV), run + restore (type-to-confirm backstop, source validation) — commands travel as NOTIFY payloads; audit actions backup.settings_changed/run_triggered/restore_requested - readyz: new warning-level backup_remote check while a target is configured (26 h daily / 170 h weekly bound) collab: - maintenance listener: on enter, persist + close every live session and refuse new connections until exit (failsafe timeout 30 min) — no in-memory document may write pre-restore content back afterwards web: - Admin → System backup section: status card with remote facts and a "Back up now" button, the Nextcloud settings form with test button, and the restore picker (local + remote sets, type-to-confirm) - global maintenance screen: any 503 maintenance_mode flips the SPA to a status page polling the exempt endpoint, reloading when the instance returns Verified end-to-end against a live stack (fresh DB, native api + sidecar, fake WebDAV server): configure → test → manual backup → bundle upload → readyz/sets/status surfaces → remote restore with maintenance gate, marker rollback and api restart; suites: shared 21, backup 9, collab 11, api 58 files green, lint + i18n:check + typecheck clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1
274 lines
9.4 KiB
TypeScript
274 lines
9.4 KiB
TypeScript
import { createServer, type Server } from 'node:http';
|
|
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import { existsSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
|
|
import type { BackupEnv } from '@dorfteich/shared';
|
|
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
|
|
import { extractArchiveTo } from './archive.js';
|
|
import { buildBundle, fetchRemoteSet, pruneRemote, uploadDue, uploadSet } from './remote.js';
|
|
import { archiveFileName, dumpFileName } from './backup-set.js';
|
|
|
|
/**
|
|
* In-memory WebDAV server covering the subset the sidecar uses — the tests
|
|
* exercise the real HTTP path including streamed PUT bodies, not a mock of
|
|
* our own client.
|
|
*/
|
|
function createDavServer(): {
|
|
server: Server;
|
|
files: Map<string, Buffer>;
|
|
collections: Set<string>;
|
|
start(): Promise<string>;
|
|
stop(): Promise<void>;
|
|
failWith?: number;
|
|
setFailWith(status: number | undefined): void;
|
|
} {
|
|
const files = new Map<string, Buffer>();
|
|
const collections = new Set<string>(['/remote.php/dav/files/tester']);
|
|
let failWith: number | undefined;
|
|
|
|
const server = createServer((req, res) => {
|
|
if (failWith) {
|
|
res.statusCode = failWith;
|
|
return res.end();
|
|
}
|
|
const path = decodeURIComponent(new URL(req.url!, 'http://x').pathname).replace(/\/+$/, '');
|
|
const chunks: Buffer[] = [];
|
|
req.on('data', (chunk: Buffer) => chunks.push(chunk));
|
|
req.on('end', () => {
|
|
switch (req.method) {
|
|
case 'PROPFIND': {
|
|
if (!collections.has(path) && !files.has(path)) {
|
|
res.statusCode = 404;
|
|
return res.end();
|
|
}
|
|
const children =
|
|
req.headers.depth === '1'
|
|
? [...files.keys()].filter((name) => name.startsWith(`${path}/`))
|
|
: [];
|
|
res.statusCode = 207;
|
|
res.setHeader('Content-Type', 'application/xml');
|
|
return res.end(
|
|
`<?xml version="1.0"?><d:multistatus xmlns:d="DAV:">
|
|
<d:response><d:href>${path}/</d:href><d:propstat><d:prop>
|
|
<d:resourcetype><d:collection/></d:resourcetype>
|
|
</d:prop></d:propstat></d:response>
|
|
${children
|
|
.map(
|
|
(name) => `<d:response><d:href>${encodeURI(name)}</d:href><d:propstat><d:prop>
|
|
<d:getcontentlength>${files.get(name)!.length}</d:getcontentlength>
|
|
<d:resourcetype/>
|
|
</d:prop></d:propstat></d:response>`,
|
|
)
|
|
.join('')}
|
|
</d:multistatus>`,
|
|
);
|
|
}
|
|
case 'MKCOL':
|
|
collections.add(path);
|
|
res.statusCode = 201;
|
|
return res.end();
|
|
case 'PUT':
|
|
files.set(path, Buffer.concat(chunks));
|
|
res.statusCode = 201;
|
|
return res.end();
|
|
case 'GET': {
|
|
const body = files.get(path);
|
|
if (!body) {
|
|
res.statusCode = 404;
|
|
return res.end();
|
|
}
|
|
res.statusCode = 200;
|
|
return res.end(body);
|
|
}
|
|
case 'DELETE': {
|
|
const existed = files.delete(path);
|
|
res.statusCode = existed ? 204 : 404;
|
|
return res.end();
|
|
}
|
|
default:
|
|
res.statusCode = 405;
|
|
return res.end();
|
|
}
|
|
});
|
|
});
|
|
|
|
return {
|
|
server,
|
|
files,
|
|
collections,
|
|
setFailWith: (status) => {
|
|
failWith = status;
|
|
},
|
|
start: () =>
|
|
new Promise((resolve) => {
|
|
server.listen(0, '127.0.0.1', () => {
|
|
const address = server.address() as { port: number };
|
|
resolve(`http://127.0.0.1:${address.port}`);
|
|
});
|
|
}),
|
|
stop: () =>
|
|
new Promise((resolve) => {
|
|
server.close(() => resolve());
|
|
}),
|
|
};
|
|
}
|
|
|
|
const silentLog = { info: () => {}, warn: () => {}, error: () => {} };
|
|
const noMailEnv = { BACKUP_MAIL_TO: undefined } as unknown as BackupEnv;
|
|
|
|
describe('uploadDue', () => {
|
|
const now = new Date('2026-07-12T03:00:00Z');
|
|
it('never uploads on schedule "off"', () => {
|
|
expect(uploadDue('off', null, now)).toBe(false);
|
|
expect(uploadDue('off', '2026-01-01T00:00:00Z', now)).toBe(false);
|
|
});
|
|
it('uploads after every set on "daily"', () => {
|
|
expect(uploadDue('daily', now.toISOString(), now)).toBe(true);
|
|
});
|
|
it('uploads weekly once the last copy is ~a week old', () => {
|
|
expect(uploadDue('weekly', null, now)).toBe(true);
|
|
expect(uploadDue('weekly', '2026-07-10T03:00:00Z', now)).toBe(false);
|
|
expect(uploadDue('weekly', '2026-07-05T02:00:00Z', now)).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe('remote bundle roundtrip', () => {
|
|
let dir: string;
|
|
let restoreDir: string;
|
|
const dav = createDavServer();
|
|
let baseUrl: string;
|
|
const backupId = '20260712-030000';
|
|
|
|
const target = (): { baseUrl: string; username: string; password: string; folder: string } => ({
|
|
baseUrl,
|
|
username: 'tester',
|
|
password: 'secret',
|
|
folder: 'dorfteich-backups',
|
|
});
|
|
|
|
beforeEach(async () => {
|
|
dir = await mkdtemp(join(tmpdir(), 'dorfteich-remote-'));
|
|
restoreDir = await mkdtemp(join(tmpdir(), 'dorfteich-remote-restore-'));
|
|
baseUrl = await dav.start();
|
|
dav.setFailWith(undefined);
|
|
dav.files.clear();
|
|
await writeFile(join(dir, dumpFileName(backupId)), 'dump-bytes');
|
|
await writeFile(join(dir, archiveFileName(backupId)), 'archive-bytes');
|
|
});
|
|
|
|
afterEach(async () => {
|
|
await dav.stop();
|
|
await rm(dir, { recursive: true, force: true });
|
|
await rm(restoreDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it('builds a self-contained bundle with a manifest', async () => {
|
|
const bundle = await buildBundle(dir, backupId, new Date('2026-07-12T03:00:05Z'));
|
|
expect(bundle.sizeBytes).toBeGreaterThan(0);
|
|
await extractArchiveTo(bundle.path, restoreDir);
|
|
const manifest = JSON.parse(await readFile(join(restoreDir, 'manifest.json'), 'utf8'));
|
|
expect(manifest).toMatchObject({ schemaVersion: 1, backupId });
|
|
expect(await readFile(join(restoreDir, dumpFileName(backupId)), 'utf8')).toBe('dump-bytes');
|
|
expect(await readFile(join(restoreDir, archiveFileName(backupId)), 'utf8')).toBe(
|
|
'archive-bytes',
|
|
);
|
|
});
|
|
|
|
it('uploads the bundle, creates the folder, and cleans up locally', async () => {
|
|
const result = await uploadSet({
|
|
env: noMailEnv,
|
|
backupsDir: dir,
|
|
target: target(),
|
|
remoteRetentionDays: 30,
|
|
backupId,
|
|
previous: undefined,
|
|
now: () => new Date('2026-07-12T03:00:10Z'),
|
|
log: silentLog,
|
|
});
|
|
expect(result.lastUpload.outcome).toBe('succeeded');
|
|
expect(result.lastSuccessfulUpload?.backupId).toBe(backupId);
|
|
const uploaded = [...dav.files.keys()];
|
|
expect(uploaded).toEqual([
|
|
`/remote.php/dav/files/tester/dorfteich-backups/dorfteich-backup-${backupId}.tar.gz`,
|
|
]);
|
|
// Bundle and manifest are derived data — gone after the upload.
|
|
expect(existsSync(join(dir, `dorfteich-backup-${backupId}.tar.gz`))).toBe(false);
|
|
expect(existsSync(join(dir, 'manifest.json'))).toBe(false);
|
|
});
|
|
|
|
it('reports a failed upload without throwing and keeps the previous success', async () => {
|
|
dav.setFailWith(401);
|
|
const previous = {
|
|
lastUpload: {
|
|
backupId: '20260711-030000',
|
|
finishedAt: '2026-07-11T03:01:00Z',
|
|
outcome: 'succeeded' as const,
|
|
sizeBytes: 10,
|
|
},
|
|
lastSuccessfulUpload: {
|
|
backupId: '20260711-030000',
|
|
finishedAt: '2026-07-11T03:01:00Z',
|
|
sizeBytes: 10,
|
|
},
|
|
};
|
|
const result = await uploadSet({
|
|
env: noMailEnv,
|
|
backupsDir: dir,
|
|
target: target(),
|
|
remoteRetentionDays: 30,
|
|
backupId,
|
|
previous,
|
|
now: () => new Date('2026-07-12T03:00:10Z'),
|
|
log: silentLog,
|
|
});
|
|
expect(result.lastUpload.outcome).toBe('failed');
|
|
expect(result.lastUpload.error).toContain('authentication failed');
|
|
expect(result.lastSuccessfulUpload).toEqual(previous.lastSuccessfulUpload);
|
|
});
|
|
|
|
it('prunes expired remote bundles but never the newest one', async () => {
|
|
const folder = '/remote.php/dav/files/tester/dorfteich-backups';
|
|
dav.collections.add(folder);
|
|
dav.files.set(`${folder}/dorfteich-backup-20260101-030000.tar.gz`, Buffer.from('old'));
|
|
dav.files.set(`${folder}/dorfteich-backup-20260102-030000.tar.gz`, Buffer.from('old2'));
|
|
dav.files.set(`${folder}/unrelated-file.txt`, Buffer.from('keep'));
|
|
await pruneRemote(target(), 30, new Date('2026-07-12T03:00:00Z'), silentLog);
|
|
expect([...dav.files.keys()].sort()).toEqual([
|
|
`${folder}/dorfteich-backup-20260102-030000.tar.gz`,
|
|
`${folder}/unrelated-file.txt`,
|
|
]);
|
|
});
|
|
|
|
it('downloads and unpacks a remote set, verifying the manifest', async () => {
|
|
await uploadSet({
|
|
env: noMailEnv,
|
|
backupsDir: dir,
|
|
target: target(),
|
|
remoteRetentionDays: 30,
|
|
backupId,
|
|
previous: undefined,
|
|
now: () => new Date(),
|
|
log: silentLog,
|
|
});
|
|
await fetchRemoteSet({ backupsDir: restoreDir, target: target(), backupId, log: silentLog });
|
|
expect(await readFile(join(restoreDir, dumpFileName(backupId)), 'utf8')).toBe('dump-bytes');
|
|
expect(await readFile(join(restoreDir, archiveFileName(backupId)), 'utf8')).toBe(
|
|
'archive-bytes',
|
|
);
|
|
});
|
|
|
|
it('fails a download of a set that does not exist remotely', async () => {
|
|
await expect(
|
|
fetchRemoteSet({
|
|
backupsDir: restoreDir,
|
|
target: target(),
|
|
backupId: '20250101-000000',
|
|
log: silentLog,
|
|
}),
|
|
).rejects.toThrow(/download/);
|
|
});
|
|
});
|