#288: reset schema before pg_restore — partitioned tables broke --clean
Some checks failed
CI / Lint, typecheck, test (pull_request) Successful in 6m48s
CI / Build container images (pull_request) Successful in 1m46s
CI / Auth e2e pack (pull_request) Successful in 8m22s
CI / Import/export fidelity gate (pull_request) Successful in 1m8s
CD / Deploy to Test (push) Blocked by required conditions
CD / Smoke tests against Test (push) Blocked by required conditions
CD / Promote to Int (push) Blocked by required conditions
CI / Auth e2e pack (push) Blocked by required conditions
CI / Import/export fidelity gate (push) Blocked by required conditions
CI / Build container images (push) Blocked by required conditions
CD / Build and push images (push) Has been cancelled
CI / Lint, typecheck, test (push) Has been cancelled
Some checks failed
CI / Lint, typecheck, test (pull_request) Successful in 6m48s
CI / Build container images (pull_request) Successful in 1m46s
CI / Auth e2e pack (pull_request) Successful in 8m22s
CI / Import/export fidelity gate (pull_request) Successful in 1m8s
CD / Deploy to Test (push) Blocked by required conditions
CD / Smoke tests against Test (push) Blocked by required conditions
CD / Promote to Int (push) Blocked by required conditions
CI / Auth e2e pack (push) Blocked by required conditions
CI / Import/export fidelity gate (push) Blocked by required conditions
CI / Build container images (push) Blocked by required conditions
CD / Build and push images (push) Has been cancelled
CI / Lint, typecheck, test (push) Has been cancelled
Since #224 read_events is partitioned; the dump carries per-partition primary keys as own entries, and pg_restore --clean emitted DROP CONSTRAINT against inherited constraints, which PostgreSQL refuses. The restore then reported FAILED although the content was restored. Dropping and recreating the public schema first makes every --clean drop a no-op and the restore faithful: objects created after the backup no longer survive. Verified in the isolated environment of #220 (set 20260731-132200, exit 0, readyz green). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AUtYMxwTCMHG9mVHnwbFg8
This commit is contained in:
parent
a758c9d78b
commit
4f6596e8a2
40
apps/backup/src/pg.test.ts
Normal file
40
apps/backup/src/pg.test.ts
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { pgEnvFromUrl, pgRestoreArgs, schemaResetArgs, schemaResetSql } from './pg.js';
|
||||||
|
|
||||||
|
describe('pgEnvFromUrl', () => {
|
||||||
|
it('maps the URL onto libpq env vars, never argv', () => {
|
||||||
|
expect(pgEnvFromUrl('postgresql://user:p%40ss@db:5433/dorfteich')).toEqual({
|
||||||
|
PGHOST: 'db',
|
||||||
|
PGPORT: '5433',
|
||||||
|
PGUSER: 'user',
|
||||||
|
PGPASSWORD: 'p@ss',
|
||||||
|
PGDATABASE: 'dorfteich',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('restore command plan (issue #288)', () => {
|
||||||
|
it('resets the schema first — --clean alone cannot drop inherited partition PKs', () => {
|
||||||
|
expect(schemaResetSql).toBe('DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;');
|
||||||
|
expect(schemaResetArgs('dorfteich')).toEqual([
|
||||||
|
'--dbname',
|
||||||
|
'dorfteich',
|
||||||
|
'--set',
|
||||||
|
'ON_ERROR_STOP=1',
|
||||||
|
'--command',
|
||||||
|
schemaResetSql,
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps --clean --if-exists as no-op belt and braces on the empty schema', () => {
|
||||||
|
expect(pgRestoreArgs('dorfteich', '/backups/x.dump')).toEqual([
|
||||||
|
'--clean',
|
||||||
|
'--if-exists',
|
||||||
|
'--no-owner',
|
||||||
|
'--dbname',
|
||||||
|
'dorfteich',
|
||||||
|
'/backups/x.dump',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -39,16 +39,32 @@ export async function pgDump(databaseUrl: string, outFile: string): Promise<void
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Restores a custom-format dump into the live database. `--clean
|
* Run before `pg_restore`: dropping and recreating the schema makes every
|
||||||
* --if-exists` drops recreated objects first, so the restore lands on a
|
* `--clean` drop a no-op and the restore faithful — objects created after
|
||||||
* database that may still hold newer state (the runbook stops the app
|
* the backup do not survive it. Relying on `--clean` alone fails on
|
||||||
* services, not the db container).
|
* partitioned tables: the dump carries per-partition primary keys as own
|
||||||
|
* entries, and the matching drops hit *inherited* constraints on the live
|
||||||
|
* database, which PostgreSQL refuses (issue #288). The schema holds no
|
||||||
|
* extension objects, so the CASCADE is bounded to our own objects.
|
||||||
|
*/
|
||||||
|
export const schemaResetSql = 'DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public;';
|
||||||
|
|
||||||
|
export function schemaResetArgs(database: string): string[] {
|
||||||
|
return ['--dbname', database, '--set', 'ON_ERROR_STOP=1', '--command', schemaResetSql];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function pgRestoreArgs(database: string, dumpFile: string): string[] {
|
||||||
|
return ['--clean', '--if-exists', '--no-owner', '--dbname', database, dumpFile];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Restores a custom-format dump into the live database: schema reset (see
|
||||||
|
* above), then `pg_restore`. `--clean --if-exists` stays as belt and braces
|
||||||
|
* — against the empty schema all its drops are no-ops (the runbook stops
|
||||||
|
* the app services, not the db container).
|
||||||
*/
|
*/
|
||||||
export async function pgRestore(databaseUrl: string, dumpFile: string): Promise<void> {
|
export async function pgRestore(databaseUrl: string, dumpFile: string): Promise<void> {
|
||||||
const database = pgEnvFromUrl(databaseUrl).PGDATABASE ?? '';
|
const database = pgEnvFromUrl(databaseUrl).PGDATABASE ?? '';
|
||||||
await runPgTool(
|
await runPgTool('psql', schemaResetArgs(database), databaseUrl);
|
||||||
'pg_restore',
|
await runPgTool('pg_restore', pgRestoreArgs(database, dumpFile), databaseUrl);
|
||||||
['--clean', '--if-exists', '--no-owner', '--dbname', database, dumpFile],
|
|
||||||
databaseUrl,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -79,7 +79,7 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack.
|
|||||||
- **Restore runbook** (also the Prod-relocation procedure) — automated by
|
- **Restore runbook** (also the Prod-relocation procedure) — automated by
|
||||||
`deploy/backup/restore.sh <backup-id>`, run from the stage directory:
|
`deploy/backup/restore.sh <backup-id>`, run from the stage directory:
|
||||||
1. stop the app services (`web`, `api`, `collab`; the db stays up),
|
1. stop the app services (`web`, `api`, `collab`; the db stays up),
|
||||||
2. restore DB: `pg_restore --clean --if-exists` into the `db` container,
|
2. restore DB: schema reset + `pg_restore` into the `db` container (#288),
|
||||||
3. restore volume: unpack the matching uploads/plugins archive,
|
3. restore volume: unpack the matching uploads/plugins archive,
|
||||||
4. `docker compose up -d`, verify `/readyz`, spot-check a page + a file.
|
4. `docker compose up -d`, verify `/readyz`, spot-check a page + a file.
|
||||||
- **Drills** (issue #87): monthly automated restore of the latest backup
|
- **Drills** (issue #87): monthly automated restore of the latest backup
|
||||||
|
|||||||
@ -17,7 +17,8 @@ On the stage host, from the stage directory (`/srv/DOCKER/dorfteich-<stage>/`):
|
|||||||
id in `status.json` → `lastSuccess.backupId`.
|
id in `status.json` → `lastSuccess.backupId`.
|
||||||
2. **Run the automated runbook:** `./restore.sh <backup-id>`
|
2. **Run the automated runbook:** `./restore.sh <backup-id>`
|
||||||
(`deploy/backup/restore.sh`). It stops `web`/`api`/`collab` (the db stays
|
(`deploy/backup/restore.sh`). It stops `web`/`api`/`collab` (the db stays
|
||||||
up), replays the dump with `pg_restore --clean --if-exists` and unpacks
|
up), resets the `public` schema and replays the dump with `pg_restore`
|
||||||
|
(#288 — objects created after the backup do not survive) and unpacks
|
||||||
the volume archive through the backup sidecar image, starts the stack,
|
the volume archive through the backup sidecar image, starts the stack,
|
||||||
and polls `/readyz`.
|
and polls `/readyz`.
|
||||||
3. **Verify:** `/readyz` fully green, spot-check one page and one uploaded
|
3. **Verify:** `/readyz` fully green, spot-check one page and one uploaded
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user