#288: reset schema before pg_restore — partitioned tables broke --clean
Some checks failed
CI / Lint, typecheck, test (pull_request) Successful in 6m48s
CI / Build container images (pull_request) Successful in 1m46s
CI / Auth e2e pack (pull_request) Successful in 8m22s
CI / Import/export fidelity gate (pull_request) Successful in 1m8s
CD / Deploy to Test (push) Blocked by required conditions
CD / Smoke tests against Test (push) Blocked by required conditions
CD / Promote to Int (push) Blocked by required conditions
CI / Auth e2e pack (push) Blocked by required conditions
CI / Import/export fidelity gate (push) Blocked by required conditions
CI / Build container images (push) Blocked by required conditions
CD / Build and push images (push) Has been cancelled
CI / Lint, typecheck, test (push) Has been cancelled
Some checks failed
CI / Lint, typecheck, test (pull_request) Successful in 6m48s
CI / Build container images (pull_request) Successful in 1m46s
CI / Auth e2e pack (pull_request) Successful in 8m22s
CI / Import/export fidelity gate (pull_request) Successful in 1m8s
CD / Deploy to Test (push) Blocked by required conditions
CD / Smoke tests against Test (push) Blocked by required conditions
CD / Promote to Int (push) Blocked by required conditions
CI / Auth e2e pack (push) Blocked by required conditions
CI / Import/export fidelity gate (push) Blocked by required conditions
CI / Build container images (push) Blocked by required conditions
CD / Build and push images (push) Has been cancelled
CI / Lint, typecheck, test (push) Has been cancelled
Since #224 read_events is partitioned; the dump carries per-partition primary keys as own entries, and pg_restore --clean emitted DROP CONSTRAINT against inherited constraints, which PostgreSQL refuses. The restore then reported FAILED although the content was restored. Dropping and recreating the public schema first makes every --clean drop a no-op and the restore faithful: objects created after the backup no longer survive. Verified in the isolated environment of #220 (set 20260731-132200, exit 0, readyz green). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AUtYMxwTCMHG9mVHnwbFg8
This commit is contained in:
parent
a758c9d78b
commit
4f6596e8a2
40
apps/backup/src/pg.test.ts
Normal file
40
apps/backup/src/pg.test.ts
Normal file
@ -0,0 +1,40 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { pgEnvFromUrl, pgRestoreArgs, schemaResetArgs, schemaResetSql } from './pg.js';
|
||||
|
||||
describe('pgEnvFromUrl', () => {
|
||||
it('maps the URL onto libpq env vars, never argv', () => {
|
||||
expect(pgEnvFromUrl('postgresql://user:p%40ss@db:5433/dorfteich')).toEqual({
|
||||
PGHOST: 'db',
|
||||
PGPORT: '5433',
|
||||
PGUSER: 'user',
|
||||
PGPASSWORD: 'p@ss',
|
||||
PGDATABASE: 'dorfteich',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('restore command plan (issue #288)', () => {
|
||||
it('resets the schema first — --clean alone cannot drop inherited partition PKs', () => {
|
||||
expect(schemaResetSql).toBe('DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;');
|
||||
expect(schemaResetArgs('dorfteich')).toEqual([
|
||||
'--dbname',
|
||||
'dorfteich',
|
||||
'--set',
|
||||
'ON_ERROR_STOP=1',
|
||||
'--command',
|
||||
schemaResetSql,
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps --clean --if-exists as no-op belt and braces on the empty schema', () => {
|
||||
expect(pgRestoreArgs('dorfteich', '/backups/x.dump')).toEqual([
|
||||
'--clean',
|
||||
'--if-exists',
|
||||
'--no-owner',
|
||||
'--dbname',
|
||||
'dorfteich',
|
||||
'/backups/x.dump',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@ -39,16 +39,32 @@ export async function pgDump(databaseUrl: string, outFile: string): Promise<void
|
||||
}
|
||||
|
||||
/**
|
||||
* Restores a custom-format dump into the live database. `--clean
|
||||
* --if-exists` drops recreated objects first, so the restore lands on a
|
||||
* database that may still hold newer state (the runbook stops the app
|
||||
* services, not the db container).
|
||||
* Run before `pg_restore`: dropping and recreating the schema makes every
|
||||
* `--clean` drop a no-op and the restore faithful — objects created after
|
||||
* the backup do not survive it. Relying on `--clean` alone fails on
|
||||
* partitioned tables: the dump carries per-partition primary keys as own
|
||||
* entries, and the matching drops hit *inherited* constraints on the live
|
||||
* database, which PostgreSQL refuses (issue #288). The schema holds no
|
||||
* extension objects, so the CASCADE is bounded to our own objects.
|
||||
*/
|
||||
export const schemaResetSql = 'DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;';
|
||||
|
||||
export function schemaResetArgs(database: string): string[] {
|
||||
return ['--dbname', database, '--set', 'ON_ERROR_STOP=1', '--command', schemaResetSql];
|
||||
}
|
||||
|
||||
export function pgRestoreArgs(database: string, dumpFile: string): string[] {
|
||||
return ['--clean', '--if-exists', '--no-owner', '--dbname', database, dumpFile];
|
||||
}
|
||||
|
||||
/**
|
||||
* Restores a custom-format dump into the live database: schema reset (see
|
||||
* above), then `pg_restore`. `--clean --if-exists` stays as belt and braces
|
||||
* — against the empty schema all its drops are no-ops (the runbook stops
|
||||
* the app services, not the db container).
|
||||
*/
|
||||
export async function pgRestore(databaseUrl: string, dumpFile: string): Promise<void> {
|
||||
const database = pgEnvFromUrl(databaseUrl).PGDATABASE ?? '';
|
||||
await runPgTool(
|
||||
'pg_restore',
|
||||
['--clean', '--if-exists', '--no-owner', '--dbname', database, dumpFile],
|
||||
databaseUrl,
|
||||
);
|
||||
await runPgTool('psql', schemaResetArgs(database), databaseUrl);
|
||||
await runPgTool('pg_restore', pgRestoreArgs(database, dumpFile), databaseUrl);
|
||||
}
|
||||
|
||||
@ -79,7 +79,7 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack.
|
||||
- **Restore runbook** (also the Prod-relocation procedure) — automated by
|
||||
`deploy/backup/restore.sh <backup-id>`, run from the stage directory:
|
||||
1. stop the app services (`web`, `api`, `collab`; the db stays up),
|
||||
2. restore DB: `pg_restore --clean --if-exists` into the `db` container,
|
||||
2. restore DB: schema reset + `pg_restore` into the `db` container (#288),
|
||||
3. restore volume: unpack the matching uploads/plugins archive,
|
||||
4. `docker compose up -d`, verify `/readyz`, spot-check a page + a file.
|
||||
- **Drills** (issue #87): monthly automated restore of the latest backup
|
||||
|
||||
@ -17,7 +17,8 @@ On the stage host, from the stage directory (`/srv/DOCKER/dorfteich-<stage>/`):
|
||||
id in `status.json` → `lastSuccess.backupId`.
|
||||
2. **Run the automated runbook:** `./restore.sh <backup-id>`
|
||||
(`deploy/backup/restore.sh`). It stops `web`/`api`/`collab` (the db stays
|
||||
up), replays the dump with `pg_restore --clean --if-exists` and unpacks
|
||||
up), resets the `public` schema and replays the dump with `pg_restore`
|
||||
(#288 — objects created after the backup do not survive) and unpacks
|
||||
the volume archive through the backup sidecar image, starts the stack,
|
||||
and polls `/readyz`.
|
||||
3. **Verify:** `/readyz` fully green, spot-check one page and one uploaded
|
||||
|
||||
Loading…
Reference in New Issue
Block a user