From 0ef96147e0d0516dca655921c2e244819f454a6f Mon Sep 17 00:00:00 2001 From: Claude Fable 5 Date: Sat, 11 Jul 2026 19:02:59 +0200 Subject: [PATCH] Extend readyz with backup freshness and a degraded status level (#85) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readyz now enumerates database/migrations (hard failures, HTTP 503), converter/renderer, and a new backup check that reads the sidecar's status.json from the read-only backups mount and warns when the last successful backup is older than 26 h. Warning-level checks surface as overall status "degraded" while staying HTTP 200 — monitors alert on the body keyword, Docker healthchecks keep using the liveness endpoints so a degraded instance is never restart-looped. The status.json shape moved to @dorfteich/shared as the contract between the sidecar and its readers (#85/#86); deploy/monitoring.md defines the Uptime-Kuma monitor set per stage. The api image also pre-creates /data/backups node-owned so the shared backups volume stays writable for the sidecar regardless of which container initializes it, and the sidecar's scheduler survives runs that cannot even record their status. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1 --- apps/api/Dockerfile | 8 +- apps/api/src/health/backup-freshness.test.ts | 158 +++++++++++++++++++ apps/api/src/health/backup-freshness.ts | 60 +++++++ apps/api/src/health/health.controller.ts | 6 +- apps/api/src/health/readiness.service.ts | 28 ++-- apps/backup/src/index.ts | 20 ++- apps/backup/src/status.ts | 42 ++--- deploy/compose/docker-compose.yml | 4 + deploy/monitoring.md | 58 +++++++ docs/architecture/operations.md | 31 ++-- packages/shared/src/backup-status.ts | 42 +++++ packages/shared/src/env.ts | 6 + packages/shared/src/index.ts | 1 + 13 files changed, 400 insertions(+), 64 deletions(-) create mode 100644 apps/api/src/health/backup-freshness.test.ts create mode 100644 apps/api/src/health/backup-freshness.ts create mode 100644 deploy/monitoring.md create mode 100644 packages/shared/src/backup-status.ts diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index a8f0a22..ace9431 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -28,7 +28,7 @@ ARG APP_VERSION=0.0.0-dev # Default the data dirs to the writable, node-owned locations created below, so # the image works out of the box even where compose does not set them; compose # still mounts named volumes here for persistence (UPLOADS_DIR/PLUGINS_DIR). -ENV NODE_ENV=production APP_VERSION=${APP_VERSION} UPLOADS_DIR=/data/uploads PLUGINS_DIR=/data/plugins SECRETS_FILE=/data/secrets/secrets.env +ENV NODE_ENV=production APP_VERSION=${APP_VERSION} UPLOADS_DIR=/data/uploads PLUGINS_DIR=/data/plugins SECRETS_FILE=/data/secrets/secrets.env BACKUPS_DIR=/data/backups WORKDIR /app COPY --from=build --chown=node:node /out /app # Generate the Prisma client for this image's platform. @@ -36,8 +36,10 @@ RUN node node_modules/prisma/build/index.js generate # A fresh named volume mounted at /data/uploads or /data/plugins is created # root-owned; pre-creating them here (Docker copies an image directory's # ownership into a new volume on first mount) lets the non-root `node` user -# write to them. -RUN mkdir -p /data/uploads /data/plugins /data/secrets && chown -R node:node /data/uploads /data/plugins /data/secrets +# write to them. /data/backups is mounted read-only here, but pre-creating it +# node-owned keeps the shared `backups` volume writable for the backup +# sidecar even when the api container is the one that initializes it. +RUN mkdir -p /data/uploads /data/plugins /data/secrets /data/backups && chown -R node:node /data/uploads /data/plugins /data/secrets /data/backups USER node EXPOSE 3000 HEALTHCHECK --interval=30s --timeout=3s --retries=3 \ diff --git a/apps/api/src/health/backup-freshness.test.ts b/apps/api/src/health/backup-freshness.test.ts new file mode 100644 index 0000000..8b1b806 --- /dev/null +++ b/apps/api/src/health/backup-freshness.test.ts @@ -0,0 +1,158 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { BACKUP_STATUS_FILE, type BackupStatus } from '@dorfteich/shared'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { backupFreshnessCheck } from './backup-freshness'; +import { ReadinessService } from './readiness.service'; + +import type { AppConfig } from '../config/app-config.service'; +import type { PrismaService } from '../prisma/prisma.service'; + +const NOW = new Date('2026-07-12T12:00:00Z'); +const HOUR = 3_600_000; + +let dir: string; + +function writeStatus(overrides: Partial): void { + const finishedAt = new Date(NOW.getTime() - 3 * HOUR).toISOString(); + const status: BackupStatus = { + schemaVersion: 1, + updatedAt: finishedAt, + retentionDays: 7, + lastRun: { + backupId: '20260712-090000', + startedAt: finishedAt, + finishedAt, + durationMs: 1000, + outcome: 'succeeded', + sizes: { dumpBytes: 1, archiveBytes: 1 }, + }, + lastSuccess: { + backupId: '20260712-090000', + finishedAt, + sizes: { dumpBytes: 1, archiveBytes: 1 }, + }, + ...overrides, + }; + writeFileSync(join(dir, BACKUP_STATUS_FILE), JSON.stringify(status)); +} + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'dorfteich-freshness-')); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe('backupFreshnessCheck', () => { + it('warns when no status was ever recorded', () => { + const check = backupFreshnessCheck(join(dir, 'nowhere'), NOW); + expect(check).toMatchObject({ name: 'backup', status: 'warn' }); + expect(check.detail).toContain('no backup status'); + }); + + it('warns on an unreadable status file', () => { + writeFileSync(join(dir, BACKUP_STATUS_FILE), '{torn'); + expect(backupFreshnessCheck(dir, NOW).status).toBe('warn'); + }); + + it('warns when no backup ever succeeded, with the last error', () => { + writeStatus({ + lastSuccess: null, + lastRun: { + backupId: '20260712-090000', + startedAt: NOW.toISOString(), + finishedAt: NOW.toISOString(), + durationMs: 5, + outcome: 'failed', + error: 'pg_dump failed: connection refused', + }, + }); + const check = backupFreshnessCheck(dir, NOW); + expect(check.status).toBe('warn'); + expect(check.detail).toContain('connection refused'); + }); + + it('warns when the last success is older than 26 h', () => { + const old = new Date(NOW.getTime() - 30 * HOUR).toISOString(); + writeStatus({ + lastSuccess: { + backupId: '20260711-060000', + finishedAt: old, + sizes: { dumpBytes: 1, archiveBytes: 1 }, + }, + }); + const check = backupFreshnessCheck(dir, NOW); + expect(check.status).toBe('warn'); + expect(check.detail).toContain('30 h old (max 26 h)'); + }); + + it('is ok on a fresh success', () => { + writeStatus({}); + expect(backupFreshnessCheck(dir, NOW)).toEqual({ name: 'backup', status: 'ok' }); + }); + + it('stays ok but reports a failed run newer than the fresh success', () => { + writeStatus({ + lastRun: { + backupId: '20260712-110000', + startedAt: NOW.toISOString(), + finishedAt: NOW.toISOString(), + durationMs: 5, + outcome: 'failed', + error: 'disk full', + }, + }); + const check = backupFreshnessCheck(dir, NOW); + expect(check.status).toBe('ok'); + expect(check.detail).toContain('disk full'); + }); +}); + +describe('ReadinessService report (issue #85 degraded semantics)', () => { + // Fakes: the database probe is the only prisma call the report makes. + const prismaUp = { $queryRaw: async () => [{ pending: 0n }] } as unknown as PrismaService; + const prismaDown = { + $queryRaw: async () => { + throw new Error('connection refused'); + }, + } as unknown as PrismaService; + + function configWith(backupsDir: string): AppConfig { + return { + env: { + // Unreachable sidecars: converter/renderer report warn — irrelevant + // here, the assertions pin the database/migrations/backup checks. + PANDOC_URL: 'http://127.0.0.1:59998', + GOTENBERG_URL: 'http://127.0.0.1:59998', + BACKUPS_DIR: backupsDir, + }, + } as AppConfig; + } + + it('reports degraded (not unready) on a stale backup with healthy hard checks', async () => { + const old = new Date(Date.now() - 40 * HOUR).toISOString(); + writeStatus({ + lastSuccess: { + backupId: '20260710-030000', + finishedAt: old, + sizes: { dumpBytes: 1, archiveBytes: 1 }, + }, + }); + const report = await new ReadinessService(prismaUp, configWith(dir)).report(); + + expect(report.status).toBe('degraded'); + const byName = Object.fromEntries(report.checks.map((c) => [c.name, c.status])); + expect(byName).toMatchObject({ database: 'ok', migrations: 'ok', backup: 'warn' }); + }); + + it('reports unready only on hard failures', async () => { + const report = await new ReadinessService(prismaDown, configWith(dir)).report(); + expect(report.status).toBe('unready'); + expect(report.checks.find((c) => c.name === 'database')?.status).toBe('failed'); + }); +}); diff --git a/apps/api/src/health/backup-freshness.ts b/apps/api/src/health/backup-freshness.ts new file mode 100644 index 0000000..1cc4cd0 --- /dev/null +++ b/apps/api/src/health/backup-freshness.ts @@ -0,0 +1,60 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { + BACKUP_FRESH_MAX_AGE_HOURS, + BACKUP_STATUS_FILE, + type BackupStatus, +} from '@dorfteich/shared'; + +import type { ReadinessCheck } from './readiness.service'; + +/** + * Backup freshness for readyz (issue #85, operations.md §Health): reads the + * sidecar's `status.json` from the shared backups volume. Always + * warning-level — a stale or missing backup degrades the instance so + * monitors alert, but the api keeps serving (503 is reserved for the + * database/migrations hard failures). + */ +export function backupFreshnessCheck(backupsDir: string, now: Date): ReadinessCheck { + const path = join(backupsDir, BACKUP_STATUS_FILE); + if (!existsSync(path)) { + return { name: 'backup', status: 'warn', detail: 'no backup status recorded yet' }; + } + + let status: BackupStatus; + try { + status = JSON.parse(readFileSync(path, 'utf8')) as BackupStatus; + } catch { + return { name: 'backup', status: 'warn', detail: 'backup status file is unreadable' }; + } + + if (!status.lastSuccess) { + const error = status.lastRun?.error; + return { + name: 'backup', + status: 'warn', + detail: error ? `no successful backup yet — last run: ${error}` : 'no successful backup yet', + }; + } + + const ageHours = (now.getTime() - new Date(status.lastSuccess.finishedAt).getTime()) / 3_600_000; + if (!Number.isFinite(ageHours) || ageHours > BACKUP_FRESH_MAX_AGE_HOURS) { + return { + name: 'backup', + status: 'warn', + detail: `last successful backup ${status.lastSuccess.backupId} is ${Math.round(ageHours)} h old (max ${BACKUP_FRESH_MAX_AGE_HOURS} h)`, + }; + } + + // Fresh success; still surface a failed newer run so operators see it + // before the freshness window runs out. + if (status.lastRun.outcome === 'failed') { + return { + name: 'backup', + status: 'ok', + detail: `fresh, but the last run failed: ${status.lastRun.error ?? 'unknown error'}`, + }; + } + return { name: 'backup', status: 'ok' }; +} diff --git a/apps/api/src/health/health.controller.ts b/apps/api/src/health/health.controller.ts index 5fe1363..045c9dc 100644 --- a/apps/api/src/health/health.controller.ts +++ b/apps/api/src/health/health.controller.ts @@ -25,13 +25,15 @@ export class HealthController { /** * Readiness: the api can do real work. Used by uptime monitoring. * The report body is sent as-is with 200/503 (not through the exception - * filter) so monitors always see which check failed. + * filter) so monitors always see which check failed. `degraded` stays + * HTTP 200 — 503 is reserved for hard failures (issue #85); monitors + * catch degradation via a keyword check on the body (deploy/monitoring.md). */ @Get('readyz') async readyz(@Res() res: Response): Promise { const report = await this.readiness.report(); res - .status(report.status === 'ok' ? HttpStatus.OK : HttpStatus.SERVICE_UNAVAILABLE) + .status(report.status === 'unready' ? HttpStatus.SERVICE_UNAVAILABLE : HttpStatus.OK) .json(report); } } diff --git a/apps/api/src/health/readiness.service.ts b/apps/api/src/health/readiness.service.ts index eaf190b..6d8939f 100644 --- a/apps/api/src/health/readiness.service.ts +++ b/apps/api/src/health/readiness.service.ts @@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common'; import { AppConfig } from '../config/app-config.service'; import { PrismaService } from '../prisma/prisma.service'; +import { backupFreshnessCheck } from './backup-freshness'; export interface ReadinessCheck { /** `warn` reports a degraded-but-serving dependency: the instance still @@ -13,7 +14,13 @@ export interface ReadinessCheck { } export interface ReadinessReport { - status: 'ok' | 'unready'; + /** + * `degraded` = at least one warning-level check (converter/renderer down, + * backup stale) while the instance still serves; monitors alert on it via + * the body, but only `unready` turns into HTTP 503 (issue #85, + * operations.md §Health). + */ + status: 'ok' | 'degraded' | 'unready'; checks: ReadinessCheck[]; } @@ -30,10 +37,10 @@ export class ReadinessService { /** * Readiness = the api can do real work: database reachable and all - * migrations applied. Further checks (converters, backup freshness) are - * added by later stories (issues #62, #85) — each as one more entry in - * the checks array, never as a separate endpoint. Converter reachability - * is warning-level: import/export degrades, but the instance stays ready. + * migrations applied — those two are the hard failures behind HTTP 503. + * Everything else (converter, renderer, backup freshness) is + * warning-level: the feature degrades, the instance stays ready, and the + * overall status says `degraded` so monitors can alert on the body. */ async report(): Promise { const checks: ReadinessCheck[] = [ @@ -41,11 +48,14 @@ export class ReadinessService { await this.migrationsApplied(), await this.converterReachable(), await this.rendererReachable(), + backupFreshnessCheck(this.config.env.BACKUPS_DIR, new Date()), ]; - return { - status: checks.some((c) => c.status === 'failed') ? 'unready' : 'ok', - checks, - }; + const status = checks.some((c) => c.status === 'failed') + ? 'unready' + : checks.some((c) => c.status === 'warn') + ? 'degraded' + : 'ok'; + return { status, checks }; } private async converterReachable(): Promise { diff --git a/apps/backup/src/index.ts b/apps/backup/src/index.ts index 7eae44d..be8a5f1 100644 --- a/apps/backup/src/index.ts +++ b/apps/backup/src/index.ts @@ -30,16 +30,30 @@ const deps: RunnerDeps = { log, }; +/** + * runBackup handles run errors itself (failed status + alert); this guard + * catches what it cannot — e.g. an unwritable backups volume, where even + * status.json fails. Logging instead of crashing keeps the container out + * of a restart loop; the missed run shows up through the freshness check. + */ +async function guardedRun(): Promise<'succeeded' | 'failed'> { + try { + return (await runBackup(deps)).lastRun.outcome; + } catch (error) { + log.error({ error: String(error) }, 'backup run could not even record its status'); + return 'failed'; + } +} + if (process.env.BACKUP_RUN_ONCE === '1') { - const status = await runBackup(deps); - process.exit(status.lastRun.outcome === 'succeeded' ? 0 : 1); + process.exit((await guardedRun()) === 'succeeded' ? 0 : 1); } log.info( { time: env.BACKUP_TIME, retentionDays: env.BACKUP_RETENTION_DAYS, dir: env.BACKUPS_DIR }, 'backup sidecar started', ); -const schedule = scheduleDaily(env.BACKUP_TIME, async () => void (await runBackup(deps)), log); +const schedule = scheduleDaily(env.BACKUP_TIME, async () => void (await guardedRun()), log); for (const signal of ['SIGTERM', 'SIGINT'] as const) { process.on(signal, () => { diff --git a/apps/backup/src/status.ts b/apps/backup/src/status.ts index 07438bc..b613e9d 100644 --- a/apps/backup/src/status.ts +++ b/apps/backup/src/status.ts @@ -2,44 +2,20 @@ import { existsSync, readFileSync } from 'node:fs'; import { rename, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; +import { BACKUP_STATUS_FILE, type BackupStatus } from '@dorfteich/shared'; + /** - * `status.json` on the backups volume is the machine-readable outcome of the - * last run (ADR 0015): the api's backup-freshness readiness check (#85) and - * the admin panel's backup card (#86) consume it. Keep the shape additive — - * bump `schemaVersion` on breaking changes. + * File I/O for `status.json` on the backups volume. The shape itself is a + * shared contract (`@dorfteich/shared` backup-status.ts) because the api's + * readiness check (#85) and admin panel (#86) read what this sidecar writes. */ -export const STATUS_FILE = 'status.json'; - -export interface BackupSizes { - dumpBytes: number; - archiveBytes: number; -} - -export interface BackupRun { - backupId: string; - startedAt: string; - finishedAt: string; - durationMs: number; - outcome: 'succeeded' | 'failed'; - /** Present on failure: the first error the run hit, as a plain string. */ - error?: string; - /** Present on success. */ - sizes?: BackupSizes; -} - -export interface BackupStatus { - schemaVersion: 1; - updatedAt: string; - retentionDays: number; - lastRun: BackupRun; - /** Carried across failed runs so freshness checks see the real gap. */ - lastSuccess: { backupId: string; finishedAt: string; sizes: BackupSizes } | null; -} +export type { BackupRun, BackupSizes, BackupStatus } from '@dorfteich/shared'; +export { BACKUP_STATUS_FILE as STATUS_FILE } from '@dorfteich/shared'; /** Reads the previous status; a missing or torn file is simply "no status". */ export function readStatus(backupsDir: string): BackupStatus | null { - const path = join(backupsDir, STATUS_FILE); + const path = join(backupsDir, BACKUP_STATUS_FILE); if (!existsSync(path)) return null; try { return JSON.parse(readFileSync(path, 'utf8')) as BackupStatus; @@ -50,7 +26,7 @@ export function readStatus(backupsDir: string): BackupStatus | null { /** Atomic write (staging + rename) so readers never see a torn file. */ export async function writeStatus(backupsDir: string, status: BackupStatus): Promise { - const path = join(backupsDir, STATUS_FILE); + const path = join(backupsDir, BACKUP_STATUS_FILE); const staging = `${path}.tmp-${process.pid}`; await writeFile(staging, JSON.stringify(status, null, 2) + '\n'); await rename(staging, path); diff --git a/deploy/compose/docker-compose.yml b/deploy/compose/docker-compose.yml index 8773f91..a1f7d2a 100644 --- a/deploy/compose/docker-compose.yml +++ b/deploy/compose/docker-compose.yml @@ -77,6 +77,9 @@ services: # Matches the `plugins` volume mount below (ADR 0008, issue #71). A Site # Admin drops ZIPs into its `_dropzone/` subfolder; the watcher installs them. PLUGINS_DIR: /data/plugins + # Read-only view of the backup sidecar's volume — the api only consumes + # its status.json (readyz freshness #85, admin backup card #86). + BACKUPS_DIR: /data/backups # Internal pandoc-server sidecar for import/export (ADR 0009, issue #62). PANDOC_URL: http://pandoc:3030 # Internal Gotenberg sidecar for PDF export (ADR 0009, issue #67). @@ -88,6 +91,7 @@ services: - uploads:/data/uploads - plugins:/data/plugins - secrets:/data/secrets + - backups:/data/backups:ro depends_on: db: condition: service_healthy diff --git a/deploy/monitoring.md b/deploy/monitoring.md new file mode 100644 index 0000000..a78ba07 --- /dev/null +++ b/deploy/monitoring.md @@ -0,0 +1,58 @@ +# Uptime monitoring (issue #85) + +Pragmatic monitoring per operations.md: no metrics stack — the health +endpoints are the single integration point, watched by the operator's +existing **Uptime-Kuma** instance. + +## Semantics: down vs. degraded + +`GET /api/v1/readyz` returns: + +| HTTP | `status` | Meaning | Reaction | +| ---- | ---------- | ------------------------------------------------------------------- | --------------------------------------------- | +| 200 | `ok` | all checks green | — | +| 200 | `degraded` | a warning-level check: converter/renderer down, backup stale (26 h) | alert, fix without urgency — users are served | +| 503 | `unready` | hard failure: database unreachable or migrations pending | page — the instance cannot serve | + +Checks enumerated in the body: `database`, `migrations` (hard), +`converter`, `renderer`, `backup` (warning-level; `backup` reads the +sidecar's `status.json` and warns when the last successful backup is +older than 26 h — ADR 0015). + +**Degraded never restarts containers**: the Docker healthchecks use the +liveness endpoints only (api `/api/v1/healthz`, web `/healthz`, collab +`/healthz`), never `readyz`. The collab `/healthz` includes its database +probe (issue #33) — Compose marks the container `unhealthy` then, but +`restart: unless-stopped` only restarts on process exit, so an unhealthy +container is visible in `docker compose ps`, not restart-looped. + +## Monitor set (per stage) + +Four monitors per stage in Uptime-Kuma; suggested interval 60 s, retries 2. + +| # | Monitor | Type | Target (Test example) | Alert condition | +| --- | ---------------------- | ----------------- | ------------------------------------------------- | -------------------------------------- | +| 1 | ` web` | HTTP(s) | `https://test.dorfteich.cloud/healthz` | non-2xx | +| 2 | ` api ready` | HTTP(s) | `https://test.dorfteich.cloud/api/v1/readyz` | non-2xx (= unready/down) | +| 3 | ` api degraded` | HTTP(s) – Keyword | same URL, keyword `"status":"ok"` must be present | body says `degraded` while HTTP is 200 | +| 4 | ` collab` | WebSocket | `wss://test.dorfteich.cloud/collab` | connect failure | + +Monitor 3 is what catches `degraded` (stale backup, dead sidecars) — +monitor 2 alone only sees hard failures. If the Kuma version has no +WebSocket monitor type, an HTTP monitor on +`https:///collab/healthz` (200, includes the DB probe) is the +fallback for monitor 4. + +Stages: `test.dorfteich.cloud`, `int.dorfteich.cloud` (web-check-only is +acceptable per operations.md — at minimum monitors 1–2, no paging), +`dorfteich.online` (Prod, full set + notification; set up and verified at +go-live, checklist item in #89). + +Setting up the monitors is an operator action in Uptime-Kuma (owner: +Stefan); this file is the definition they follow. + +## Related + +- operations.md §Health & monitoring (endpoint semantics) +- ADR 0015 (backup freshness feeds the `backup` check) +- deploy/backup/restore.sh (what to do when the backup check warns) diff --git a/docs/architecture/operations.md b/docs/architecture/operations.md index 039dbba..27ecf26 100644 --- a/docs/architecture/operations.md +++ b/docs/architecture/operations.md @@ -6,21 +6,24 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack. ## Health & monitoring - **Health endpoints**: `api` exposes `/healthz` (liveness: process up) and - `/readyz` (readiness: DB reachable, migrations applied, converters - reachable, backup freshness < 26 h). `collab` exposes `/healthz` - (process + DB). `web` serves a static `/healthz`. -- **Docker healthchecks** on every service (compose `healthcheck:`), so - `docker compose ps` and restarts reflect real state; - `restart: unless-stopped` everywhere. -- **External uptime monitoring**: the operator's existing Uptime-Kuma - monitors `https://dorfteich.online/healthz` (web), `/api/v1/healthz`, and - a WebSocket check on `/collab`, with notification on failure. Test/Int - get web-check-only monitors (no paging). + `/readyz` (readiness: DB reachable + migrations applied = hard failures + → 503; converter/renderer reachability and backup freshness < 26 h are + warning-level → overall `status: degraded`, still HTTP 200). `collab` + exposes `/healthz` (process + DB). `web` serves a static `/healthz`. +- **Docker healthchecks** use the liveness endpoints only — never `readyz` — + so a degraded instance is not restart-looped (`restart: unless-stopped` + restarts on process exit, an `unhealthy` mark just shows in + `docker compose ps`). +- **External uptime monitoring**: the operator's existing Uptime-Kuma runs + the monitor set defined in `deploy/monitoring.md` (web healthz, api + readyz for down, a keyword monitor on the readyz body for degraded, and + a collab WebSocket check), with notification on failure for Prod. + Test/Int get reduced monitors (no paging). - **Backup alerting**: the backup sidecar writes - `backups/status.json` after every run; `readyz` degrades when the last - successful backup is older than 26 h, which surfaces through Uptime-Kuma - without extra tooling. Additionally the sidecar sends a failure e-mail - via the instance SMTP. + `backups/status.json` after every run; `readyz` reads it (issue #85) and + degrades when the last successful backup is older than 26 h, which + surfaces through Uptime-Kuma without extra tooling. Additionally the + sidecar sends a failure e-mail via the instance SMTP. ## Logging diff --git a/packages/shared/src/backup-status.ts b/packages/shared/src/backup-status.ts new file mode 100644 index 0000000..eea69f3 --- /dev/null +++ b/packages/shared/src/backup-status.ts @@ -0,0 +1,42 @@ +/** + * The contract of the backup sidecar's `status.json` (ADR 0015, issue #83): + * the sidecar writes it after every run, the api reads it for the readyz + * backup-freshness check (issue #85) and the admin panel's backup card + * (issue #86). Keep the shape additive — bump `schemaVersion` on breaking + * changes. + */ + +export const BACKUP_STATUS_FILE = 'status.json'; + +/** + * Freshness bound for readiness (operations.md §Health): the nightly cadence + * plus a two-hour grace window. An older (or missing) last success degrades + * readyz — it never hard-fails it. + */ +export const BACKUP_FRESH_MAX_AGE_HOURS = 26; + +export interface BackupSizes { + dumpBytes: number; + archiveBytes: number; +} + +export interface BackupRun { + backupId: string; + startedAt: string; + finishedAt: string; + durationMs: number; + outcome: 'succeeded' | 'failed'; + /** Present on failure: the first error the run hit, as a plain string. */ + error?: string; + /** Present on success. */ + sizes?: BackupSizes; +} + +export interface BackupStatus { + schemaVersion: 1; + updatedAt: string; + retentionDays: number; + lastRun: BackupRun; + /** Carried across failed runs so freshness checks see the real gap. */ + lastSuccess: { backupId: string; finishedAt: string; sizes: BackupSizes } | null; +} diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts index ceb7c80..64c1957 100644 --- a/packages/shared/src/env.ts +++ b/packages/shared/src/env.ts @@ -96,6 +96,12 @@ export const apiEnvSchema = z.object({ * here; the relative default serves native dev/test runs. */ PLUGINS_DIR: z.string().min(1).default('./data/plugins'), + /** + * The backup sidecar's volume with the restore sets and `status.json` + * (ADR 0015). The api mounts it read-only and only consumes the status + * file (readyz freshness, issue #85; admin backup card, issue #86). + */ + BACKUPS_DIR: z.string().min(1).default('./data/backups'), /** * Env-backed secret store (security.md §Secrets, issue #80): a mode-600 * dotenv-style file on a persistent volume where the setup wizard writes diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index f8895a6..ec6fa42 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1,6 +1,7 @@ export * from './admin-users'; export * from './api-error'; export * from './auth'; +export * from './backup-status'; export * from './collab-token'; export * from './editor-schema'; export * from './env';