Extend readyz with backup freshness and a degraded status level (#85)
All checks were successful
CI / Lint, typecheck, test (push) Successful in 3m11s
CI / Build container images (push) Has been skipped
CD / Build and push images (push) Successful in 3m44s
CD / Deploy to Test (push) Successful in 9s
CD / Smoke tests against Test (push) Successful in 1m10s
CD / Promote to Int (push) Successful in 10s
CI / Auth e2e pack (push) Successful in 5m13s
CI / Import/export fidelity gate (push) Successful in 46s
All checks were successful
CI / Lint, typecheck, test (push) Successful in 3m11s
CI / Build container images (push) Has been skipped
CD / Build and push images (push) Successful in 3m44s
CD / Deploy to Test (push) Successful in 9s
CD / Smoke tests against Test (push) Successful in 1m10s
CD / Promote to Int (push) Successful in 10s
CI / Auth e2e pack (push) Successful in 5m13s
CI / Import/export fidelity gate (push) Successful in 46s
readyz now enumerates database/migrations (hard failures, HTTP 503), converter/renderer, and a new backup check that reads the sidecar's status.json from the read-only backups mount and warns when the last successful backup is older than 26 h. Warning-level checks surface as overall status "degraded" while staying HTTP 200 — monitors alert on the body keyword, Docker healthchecks keep using the liveness endpoints so a degraded instance is never restart-looped. The status.json shape moved to @dorfteich/shared as the contract between the sidecar and its readers (#85/#86); deploy/monitoring.md defines the Uptime-Kuma monitor set per stage. The api image also pre-creates /data/backups node-owned so the shared backups volume stays writable for the sidecar regardless of which container initializes it, and the sidecar's scheduler survives runs that cannot even record their status. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1
This commit is contained in:
parent
8dbff86537
commit
0ef96147e0
@ -28,7 +28,7 @@ ARG APP_VERSION=0.0.0-dev
|
|||||||
# Default the data dirs to the writable, node-owned locations created below, so
|
# Default the data dirs to the writable, node-owned locations created below, so
|
||||||
# the image works out of the box even where compose does not set them; compose
|
# the image works out of the box even where compose does not set them; compose
|
||||||
# still mounts named volumes here for persistence (UPLOADS_DIR/PLUGINS_DIR).
|
# still mounts named volumes here for persistence (UPLOADS_DIR/PLUGINS_DIR).
|
||||||
ENV NODE_ENV=production APP_VERSION=${APP_VERSION} UPLOADS_DIR=/data/uploads PLUGINS_DIR=/data/plugins SECRETS_FILE=/data/secrets/secrets.env
|
ENV NODE_ENV=production APP_VERSION=${APP_VERSION} UPLOADS_DIR=/data/uploads PLUGINS_DIR=/data/plugins SECRETS_FILE=/data/secrets/secrets.env BACKUPS_DIR=/data/backups
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY --from=build --chown=node:node /out /app
|
COPY --from=build --chown=node:node /out /app
|
||||||
# Generate the Prisma client for this image's platform.
|
# Generate the Prisma client for this image's platform.
|
||||||
@ -36,8 +36,10 @@ RUN node node_modules/prisma/build/index.js generate
|
|||||||
# A fresh named volume mounted at /data/uploads or /data/plugins is created
|
# A fresh named volume mounted at /data/uploads or /data/plugins is created
|
||||||
# root-owned; pre-creating them here (Docker copies an image directory's
|
# root-owned; pre-creating them here (Docker copies an image directory's
|
||||||
# ownership into a new volume on first mount) lets the non-root `node` user
|
# ownership into a new volume on first mount) lets the non-root `node` user
|
||||||
# write to them.
|
# write to them. /data/backups is mounted read-only here, but pre-creating it
|
||||||
RUN mkdir -p /data/uploads /data/plugins /data/secrets && chown -R node:node /data/uploads /data/plugins /data/secrets
|
# node-owned keeps the shared `backups` volume writable for the backup
|
||||||
|
# sidecar even when the api container is the one that initializes it.
|
||||||
|
RUN mkdir -p /data/uploads /data/plugins /data/secrets /data/backups && chown -R node:node /data/uploads /data/plugins /data/secrets /data/backups
|
||||||
USER node
|
USER node
|
||||||
EXPOSE 3000
|
EXPOSE 3000
|
||||||
HEALTHCHECK --interval=30s --timeout=3s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=3s --retries=3 \
|
||||||
|
|||||||
158
apps/api/src/health/backup-freshness.test.ts
Normal file
158
apps/api/src/health/backup-freshness.test.ts
Normal file
@ -0,0 +1,158 @@
|
|||||||
|
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
|
||||||
|
import { BACKUP_STATUS_FILE, type BackupStatus } from '@dorfteich/shared';
|
||||||
|
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { backupFreshnessCheck } from './backup-freshness';
|
||||||
|
import { ReadinessService } from './readiness.service';
|
||||||
|
|
||||||
|
import type { AppConfig } from '../config/app-config.service';
|
||||||
|
import type { PrismaService } from '../prisma/prisma.service';
|
||||||
|
|
||||||
|
const NOW = new Date('2026-07-12T12:00:00Z');
|
||||||
|
const HOUR = 3_600_000;
|
||||||
|
|
||||||
|
let dir: string;
|
||||||
|
|
||||||
|
function writeStatus(overrides: Partial<BackupStatus>): void {
|
||||||
|
const finishedAt = new Date(NOW.getTime() - 3 * HOUR).toISOString();
|
||||||
|
const status: BackupStatus = {
|
||||||
|
schemaVersion: 1,
|
||||||
|
updatedAt: finishedAt,
|
||||||
|
retentionDays: 7,
|
||||||
|
lastRun: {
|
||||||
|
backupId: '20260712-090000',
|
||||||
|
startedAt: finishedAt,
|
||||||
|
finishedAt,
|
||||||
|
durationMs: 1000,
|
||||||
|
outcome: 'succeeded',
|
||||||
|
sizes: { dumpBytes: 1, archiveBytes: 1 },
|
||||||
|
},
|
||||||
|
lastSuccess: {
|
||||||
|
backupId: '20260712-090000',
|
||||||
|
finishedAt,
|
||||||
|
sizes: { dumpBytes: 1, archiveBytes: 1 },
|
||||||
|
},
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
writeFileSync(join(dir, BACKUP_STATUS_FILE), JSON.stringify(status));
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
dir = mkdtempSync(join(tmpdir(), 'dorfteich-freshness-'));
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
rmSync(dir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('backupFreshnessCheck', () => {
|
||||||
|
it('warns when no status was ever recorded', () => {
|
||||||
|
const check = backupFreshnessCheck(join(dir, 'nowhere'), NOW);
|
||||||
|
expect(check).toMatchObject({ name: 'backup', status: 'warn' });
|
||||||
|
expect(check.detail).toContain('no backup status');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('warns on an unreadable status file', () => {
|
||||||
|
writeFileSync(join(dir, BACKUP_STATUS_FILE), '{torn');
|
||||||
|
expect(backupFreshnessCheck(dir, NOW).status).toBe('warn');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('warns when no backup ever succeeded, with the last error', () => {
|
||||||
|
writeStatus({
|
||||||
|
lastSuccess: null,
|
||||||
|
lastRun: {
|
||||||
|
backupId: '20260712-090000',
|
||||||
|
startedAt: NOW.toISOString(),
|
||||||
|
finishedAt: NOW.toISOString(),
|
||||||
|
durationMs: 5,
|
||||||
|
outcome: 'failed',
|
||||||
|
error: 'pg_dump failed: connection refused',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const check = backupFreshnessCheck(dir, NOW);
|
||||||
|
expect(check.status).toBe('warn');
|
||||||
|
expect(check.detail).toContain('connection refused');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('warns when the last success is older than 26 h', () => {
|
||||||
|
const old = new Date(NOW.getTime() - 30 * HOUR).toISOString();
|
||||||
|
writeStatus({
|
||||||
|
lastSuccess: {
|
||||||
|
backupId: '20260711-060000',
|
||||||
|
finishedAt: old,
|
||||||
|
sizes: { dumpBytes: 1, archiveBytes: 1 },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const check = backupFreshnessCheck(dir, NOW);
|
||||||
|
expect(check.status).toBe('warn');
|
||||||
|
expect(check.detail).toContain('30 h old (max 26 h)');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is ok on a fresh success', () => {
|
||||||
|
writeStatus({});
|
||||||
|
expect(backupFreshnessCheck(dir, NOW)).toEqual({ name: 'backup', status: 'ok' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('stays ok but reports a failed run newer than the fresh success', () => {
|
||||||
|
writeStatus({
|
||||||
|
lastRun: {
|
||||||
|
backupId: '20260712-110000',
|
||||||
|
startedAt: NOW.toISOString(),
|
||||||
|
finishedAt: NOW.toISOString(),
|
||||||
|
durationMs: 5,
|
||||||
|
outcome: 'failed',
|
||||||
|
error: 'disk full',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const check = backupFreshnessCheck(dir, NOW);
|
||||||
|
expect(check.status).toBe('ok');
|
||||||
|
expect(check.detail).toContain('disk full');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('ReadinessService report (issue #85 degraded semantics)', () => {
|
||||||
|
// Fakes: the database probe is the only prisma call the report makes.
|
||||||
|
const prismaUp = { $queryRaw: async () => [{ pending: 0n }] } as unknown as PrismaService;
|
||||||
|
const prismaDown = {
|
||||||
|
$queryRaw: async () => {
|
||||||
|
throw new Error('connection refused');
|
||||||
|
},
|
||||||
|
} as unknown as PrismaService;
|
||||||
|
|
||||||
|
function configWith(backupsDir: string): AppConfig {
|
||||||
|
return {
|
||||||
|
env: {
|
||||||
|
// Unreachable sidecars: converter/renderer report warn — irrelevant
|
||||||
|
// here, the assertions pin the database/migrations/backup checks.
|
||||||
|
PANDOC_URL: 'http://127.0.0.1:59998',
|
||||||
|
GOTENBERG_URL: 'http://127.0.0.1:59998',
|
||||||
|
BACKUPS_DIR: backupsDir,
|
||||||
|
},
|
||||||
|
} as AppConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
it('reports degraded (not unready) on a stale backup with healthy hard checks', async () => {
|
||||||
|
const old = new Date(Date.now() - 40 * HOUR).toISOString();
|
||||||
|
writeStatus({
|
||||||
|
lastSuccess: {
|
||||||
|
backupId: '20260710-030000',
|
||||||
|
finishedAt: old,
|
||||||
|
sizes: { dumpBytes: 1, archiveBytes: 1 },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const report = await new ReadinessService(prismaUp, configWith(dir)).report();
|
||||||
|
|
||||||
|
expect(report.status).toBe('degraded');
|
||||||
|
const byName = Object.fromEntries(report.checks.map((c) => [c.name, c.status]));
|
||||||
|
expect(byName).toMatchObject({ database: 'ok', migrations: 'ok', backup: 'warn' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reports unready only on hard failures', async () => {
|
||||||
|
const report = await new ReadinessService(prismaDown, configWith(dir)).report();
|
||||||
|
expect(report.status).toBe('unready');
|
||||||
|
expect(report.checks.find((c) => c.name === 'database')?.status).toBe('failed');
|
||||||
|
});
|
||||||
|
});
|
||||||
60
apps/api/src/health/backup-freshness.ts
Normal file
60
apps/api/src/health/backup-freshness.ts
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
import { existsSync, readFileSync } from 'node:fs';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
|
||||||
|
import {
|
||||||
|
BACKUP_FRESH_MAX_AGE_HOURS,
|
||||||
|
BACKUP_STATUS_FILE,
|
||||||
|
type BackupStatus,
|
||||||
|
} from '@dorfteich/shared';
|
||||||
|
|
||||||
|
import type { ReadinessCheck } from './readiness.service';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Backup freshness for readyz (issue #85, operations.md §Health): reads the
|
||||||
|
* sidecar's `status.json` from the shared backups volume. Always
|
||||||
|
* warning-level — a stale or missing backup degrades the instance so
|
||||||
|
* monitors alert, but the api keeps serving (503 is reserved for the
|
||||||
|
* database/migrations hard failures).
|
||||||
|
*/
|
||||||
|
export function backupFreshnessCheck(backupsDir: string, now: Date): ReadinessCheck {
|
||||||
|
const path = join(backupsDir, BACKUP_STATUS_FILE);
|
||||||
|
if (!existsSync(path)) {
|
||||||
|
return { name: 'backup', status: 'warn', detail: 'no backup status recorded yet' };
|
||||||
|
}
|
||||||
|
|
||||||
|
let status: BackupStatus;
|
||||||
|
try {
|
||||||
|
status = JSON.parse(readFileSync(path, 'utf8')) as BackupStatus;
|
||||||
|
} catch {
|
||||||
|
return { name: 'backup', status: 'warn', detail: 'backup status file is unreadable' };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!status.lastSuccess) {
|
||||||
|
const error = status.lastRun?.error;
|
||||||
|
return {
|
||||||
|
name: 'backup',
|
||||||
|
status: 'warn',
|
||||||
|
detail: error ? `no successful backup yet — last run: ${error}` : 'no successful backup yet',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const ageHours = (now.getTime() - new Date(status.lastSuccess.finishedAt).getTime()) / 3_600_000;
|
||||||
|
if (!Number.isFinite(ageHours) || ageHours > BACKUP_FRESH_MAX_AGE_HOURS) {
|
||||||
|
return {
|
||||||
|
name: 'backup',
|
||||||
|
status: 'warn',
|
||||||
|
detail: `last successful backup ${status.lastSuccess.backupId} is ${Math.round(ageHours)} h old (max ${BACKUP_FRESH_MAX_AGE_HOURS} h)`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fresh success; still surface a failed newer run so operators see it
|
||||||
|
// before the freshness window runs out.
|
||||||
|
if (status.lastRun.outcome === 'failed') {
|
||||||
|
return {
|
||||||
|
name: 'backup',
|
||||||
|
status: 'ok',
|
||||||
|
detail: `fresh, but the last run failed: ${status.lastRun.error ?? 'unknown error'}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { name: 'backup', status: 'ok' };
|
||||||
|
}
|
||||||
@ -25,13 +25,15 @@ export class HealthController {
|
|||||||
/**
|
/**
|
||||||
* Readiness: the api can do real work. Used by uptime monitoring.
|
* Readiness: the api can do real work. Used by uptime monitoring.
|
||||||
* The report body is sent as-is with 200/503 (not through the exception
|
* The report body is sent as-is with 200/503 (not through the exception
|
||||||
* filter) so monitors always see which check failed.
|
* filter) so monitors always see which check failed. `degraded` stays
|
||||||
|
* HTTP 200 — 503 is reserved for hard failures (issue #85); monitors
|
||||||
|
* catch degradation via a keyword check on the body (deploy/monitoring.md).
|
||||||
*/
|
*/
|
||||||
@Get('readyz')
|
@Get('readyz')
|
||||||
async readyz(@Res() res: Response): Promise<void> {
|
async readyz(@Res() res: Response): Promise<void> {
|
||||||
const report = await this.readiness.report();
|
const report = await this.readiness.report();
|
||||||
res
|
res
|
||||||
.status(report.status === 'ok' ? HttpStatus.OK : HttpStatus.SERVICE_UNAVAILABLE)
|
.status(report.status === 'unready' ? HttpStatus.SERVICE_UNAVAILABLE : HttpStatus.OK)
|
||||||
.json(report);
|
.json(report);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
|
|||||||
|
|
||||||
import { AppConfig } from '../config/app-config.service';
|
import { AppConfig } from '../config/app-config.service';
|
||||||
import { PrismaService } from '../prisma/prisma.service';
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
|
import { backupFreshnessCheck } from './backup-freshness';
|
||||||
|
|
||||||
export interface ReadinessCheck {
|
export interface ReadinessCheck {
|
||||||
/** `warn` reports a degraded-but-serving dependency: the instance still
|
/** `warn` reports a degraded-but-serving dependency: the instance still
|
||||||
@ -13,7 +14,13 @@ export interface ReadinessCheck {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface ReadinessReport {
|
export interface ReadinessReport {
|
||||||
status: 'ok' | 'unready';
|
/**
|
||||||
|
* `degraded` = at least one warning-level check (converter/renderer down,
|
||||||
|
* backup stale) while the instance still serves; monitors alert on it via
|
||||||
|
* the body, but only `unready` turns into HTTP 503 (issue #85,
|
||||||
|
* operations.md §Health).
|
||||||
|
*/
|
||||||
|
status: 'ok' | 'degraded' | 'unready';
|
||||||
checks: ReadinessCheck[];
|
checks: ReadinessCheck[];
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -30,10 +37,10 @@ export class ReadinessService {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Readiness = the api can do real work: database reachable and all
|
* Readiness = the api can do real work: database reachable and all
|
||||||
* migrations applied. Further checks (converters, backup freshness) are
|
* migrations applied — those two are the hard failures behind HTTP 503.
|
||||||
* added by later stories (issues #62, #85) — each as one more entry in
|
* Everything else (converter, renderer, backup freshness) is
|
||||||
* the checks array, never as a separate endpoint. Converter reachability
|
* warning-level: the feature degrades, the instance stays ready, and the
|
||||||
* is warning-level: import/export degrades, but the instance stays ready.
|
* overall status says `degraded` so monitors can alert on the body.
|
||||||
*/
|
*/
|
||||||
async report(): Promise<ReadinessReport> {
|
async report(): Promise<ReadinessReport> {
|
||||||
const checks: ReadinessCheck[] = [
|
const checks: ReadinessCheck[] = [
|
||||||
@ -41,11 +48,14 @@ export class ReadinessService {
|
|||||||
await this.migrationsApplied(),
|
await this.migrationsApplied(),
|
||||||
await this.converterReachable(),
|
await this.converterReachable(),
|
||||||
await this.rendererReachable(),
|
await this.rendererReachable(),
|
||||||
|
backupFreshnessCheck(this.config.env.BACKUPS_DIR, new Date()),
|
||||||
];
|
];
|
||||||
return {
|
const status = checks.some((c) => c.status === 'failed')
|
||||||
status: checks.some((c) => c.status === 'failed') ? 'unready' : 'ok',
|
? 'unready'
|
||||||
checks,
|
: checks.some((c) => c.status === 'warn')
|
||||||
};
|
? 'degraded'
|
||||||
|
: 'ok';
|
||||||
|
return { status, checks };
|
||||||
}
|
}
|
||||||
|
|
||||||
private async converterReachable(): Promise<ReadinessCheck> {
|
private async converterReachable(): Promise<ReadinessCheck> {
|
||||||
|
|||||||
@ -30,16 +30,30 @@ const deps: RunnerDeps = {
|
|||||||
log,
|
log,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* runBackup handles run errors itself (failed status + alert); this guard
|
||||||
|
* catches what it cannot — e.g. an unwritable backups volume, where even
|
||||||
|
* status.json fails. Logging instead of crashing keeps the container out
|
||||||
|
* of a restart loop; the missed run shows up through the freshness check.
|
||||||
|
*/
|
||||||
|
async function guardedRun(): Promise<'succeeded' | 'failed'> {
|
||||||
|
try {
|
||||||
|
return (await runBackup(deps)).lastRun.outcome;
|
||||||
|
} catch (error) {
|
||||||
|
log.error({ error: String(error) }, 'backup run could not even record its status');
|
||||||
|
return 'failed';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (process.env.BACKUP_RUN_ONCE === '1') {
|
if (process.env.BACKUP_RUN_ONCE === '1') {
|
||||||
const status = await runBackup(deps);
|
process.exit((await guardedRun()) === 'succeeded' ? 0 : 1);
|
||||||
process.exit(status.lastRun.outcome === 'succeeded' ? 0 : 1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
log.info(
|
log.info(
|
||||||
{ time: env.BACKUP_TIME, retentionDays: env.BACKUP_RETENTION_DAYS, dir: env.BACKUPS_DIR },
|
{ time: env.BACKUP_TIME, retentionDays: env.BACKUP_RETENTION_DAYS, dir: env.BACKUPS_DIR },
|
||||||
'backup sidecar started',
|
'backup sidecar started',
|
||||||
);
|
);
|
||||||
const schedule = scheduleDaily(env.BACKUP_TIME, async () => void (await runBackup(deps)), log);
|
const schedule = scheduleDaily(env.BACKUP_TIME, async () => void (await guardedRun()), log);
|
||||||
|
|
||||||
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
|
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
|
||||||
process.on(signal, () => {
|
process.on(signal, () => {
|
||||||
|
|||||||
@ -2,44 +2,20 @@ import { existsSync, readFileSync } from 'node:fs';
|
|||||||
import { rename, writeFile } from 'node:fs/promises';
|
import { rename, writeFile } from 'node:fs/promises';
|
||||||
import { join } from 'node:path';
|
import { join } from 'node:path';
|
||||||
|
|
||||||
|
import { BACKUP_STATUS_FILE, type BackupStatus } from '@dorfteich/shared';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* `status.json` on the backups volume is the machine-readable outcome of the
|
* File I/O for `status.json` on the backups volume. The shape itself is a
|
||||||
* last run (ADR 0015): the api's backup-freshness readiness check (#85) and
|
* shared contract (`@dorfteich/shared` backup-status.ts) because the api's
|
||||||
* the admin panel's backup card (#86) consume it. Keep the shape additive —
|
* readiness check (#85) and admin panel (#86) read what this sidecar writes.
|
||||||
* bump `schemaVersion` on breaking changes.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export const STATUS_FILE = 'status.json';
|
export type { BackupRun, BackupSizes, BackupStatus } from '@dorfteich/shared';
|
||||||
|
export { BACKUP_STATUS_FILE as STATUS_FILE } from '@dorfteich/shared';
|
||||||
export interface BackupSizes {
|
|
||||||
dumpBytes: number;
|
|
||||||
archiveBytes: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface BackupRun {
|
|
||||||
backupId: string;
|
|
||||||
startedAt: string;
|
|
||||||
finishedAt: string;
|
|
||||||
durationMs: number;
|
|
||||||
outcome: 'succeeded' | 'failed';
|
|
||||||
/** Present on failure: the first error the run hit, as a plain string. */
|
|
||||||
error?: string;
|
|
||||||
/** Present on success. */
|
|
||||||
sizes?: BackupSizes;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface BackupStatus {
|
|
||||||
schemaVersion: 1;
|
|
||||||
updatedAt: string;
|
|
||||||
retentionDays: number;
|
|
||||||
lastRun: BackupRun;
|
|
||||||
/** Carried across failed runs so freshness checks see the real gap. */
|
|
||||||
lastSuccess: { backupId: string; finishedAt: string; sizes: BackupSizes } | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Reads the previous status; a missing or torn file is simply "no status". */
|
/** Reads the previous status; a missing or torn file is simply "no status". */
|
||||||
export function readStatus(backupsDir: string): BackupStatus | null {
|
export function readStatus(backupsDir: string): BackupStatus | null {
|
||||||
const path = join(backupsDir, STATUS_FILE);
|
const path = join(backupsDir, BACKUP_STATUS_FILE);
|
||||||
if (!existsSync(path)) return null;
|
if (!existsSync(path)) return null;
|
||||||
try {
|
try {
|
||||||
return JSON.parse(readFileSync(path, 'utf8')) as BackupStatus;
|
return JSON.parse(readFileSync(path, 'utf8')) as BackupStatus;
|
||||||
@ -50,7 +26,7 @@ export function readStatus(backupsDir: string): BackupStatus | null {
|
|||||||
|
|
||||||
/** Atomic write (staging + rename) so readers never see a torn file. */
|
/** Atomic write (staging + rename) so readers never see a torn file. */
|
||||||
export async function writeStatus(backupsDir: string, status: BackupStatus): Promise<void> {
|
export async function writeStatus(backupsDir: string, status: BackupStatus): Promise<void> {
|
||||||
const path = join(backupsDir, STATUS_FILE);
|
const path = join(backupsDir, BACKUP_STATUS_FILE);
|
||||||
const staging = `${path}.tmp-${process.pid}`;
|
const staging = `${path}.tmp-${process.pid}`;
|
||||||
await writeFile(staging, JSON.stringify(status, null, 2) + '\n');
|
await writeFile(staging, JSON.stringify(status, null, 2) + '\n');
|
||||||
await rename(staging, path);
|
await rename(staging, path);
|
||||||
|
|||||||
@ -77,6 +77,9 @@ services:
|
|||||||
# Matches the `plugins` volume mount below (ADR 0008, issue #71). A Site
|
# Matches the `plugins` volume mount below (ADR 0008, issue #71). A Site
|
||||||
# Admin drops ZIPs into its `_dropzone/` subfolder; the watcher installs them.
|
# Admin drops ZIPs into its `_dropzone/` subfolder; the watcher installs them.
|
||||||
PLUGINS_DIR: /data/plugins
|
PLUGINS_DIR: /data/plugins
|
||||||
|
# Read-only view of the backup sidecar's volume — the api only consumes
|
||||||
|
# its status.json (readyz freshness #85, admin backup card #86).
|
||||||
|
BACKUPS_DIR: /data/backups
|
||||||
# Internal pandoc-server sidecar for import/export (ADR 0009, issue #62).
|
# Internal pandoc-server sidecar for import/export (ADR 0009, issue #62).
|
||||||
PANDOC_URL: http://pandoc:3030
|
PANDOC_URL: http://pandoc:3030
|
||||||
# Internal Gotenberg sidecar for PDF export (ADR 0009, issue #67).
|
# Internal Gotenberg sidecar for PDF export (ADR 0009, issue #67).
|
||||||
@ -88,6 +91,7 @@ services:
|
|||||||
- uploads:/data/uploads
|
- uploads:/data/uploads
|
||||||
- plugins:/data/plugins
|
- plugins:/data/plugins
|
||||||
- secrets:/data/secrets
|
- secrets:/data/secrets
|
||||||
|
- backups:/data/backups:ro
|
||||||
depends_on:
|
depends_on:
|
||||||
db:
|
db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|||||||
58
deploy/monitoring.md
Normal file
58
deploy/monitoring.md
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
# Uptime monitoring (issue #85)
|
||||||
|
|
||||||
|
Pragmatic monitoring per operations.md: no metrics stack — the health
|
||||||
|
endpoints are the single integration point, watched by the operator's
|
||||||
|
existing **Uptime-Kuma** instance.
|
||||||
|
|
||||||
|
## Semantics: down vs. degraded
|
||||||
|
|
||||||
|
`GET /api/v1/readyz` returns:
|
||||||
|
|
||||||
|
| HTTP | `status` | Meaning | Reaction |
|
||||||
|
| ---- | ---------- | ------------------------------------------------------------------- | --------------------------------------------- |
|
||||||
|
| 200 | `ok` | all checks green | — |
|
||||||
|
| 200 | `degraded` | a warning-level check: converter/renderer down, backup stale (26 h) | alert, fix without urgency — users are served |
|
||||||
|
| 503 | `unready` | hard failure: database unreachable or migrations pending | page — the instance cannot serve |
|
||||||
|
|
||||||
|
Checks enumerated in the body: `database`, `migrations` (hard),
|
||||||
|
`converter`, `renderer`, `backup` (warning-level; `backup` reads the
|
||||||
|
sidecar's `status.json` and warns when the last successful backup is
|
||||||
|
older than 26 h — ADR 0015).
|
||||||
|
|
||||||
|
**Degraded never restarts containers**: the Docker healthchecks use the
|
||||||
|
liveness endpoints only (api `/api/v1/healthz`, web `/healthz`, collab
|
||||||
|
`/healthz`), never `readyz`. The collab `/healthz` includes its database
|
||||||
|
probe (issue #33) — Compose marks the container `unhealthy` then, but
|
||||||
|
`restart: unless-stopped` only restarts on process exit, so an unhealthy
|
||||||
|
container is visible in `docker compose ps`, not restart-looped.
|
||||||
|
|
||||||
|
## Monitor set (per stage)
|
||||||
|
|
||||||
|
Four monitors per stage in Uptime-Kuma; suggested interval 60 s, retries 2.
|
||||||
|
|
||||||
|
| # | Monitor | Type | Target (Test example) | Alert condition |
|
||||||
|
| --- | ---------------------- | ----------------- | ------------------------------------------------- | -------------------------------------- |
|
||||||
|
| 1 | `<stage> web` | HTTP(s) | `https://test.dorfteich.cloud/healthz` | non-2xx |
|
||||||
|
| 2 | `<stage> api ready` | HTTP(s) | `https://test.dorfteich.cloud/api/v1/readyz` | non-2xx (= unready/down) |
|
||||||
|
| 3 | `<stage> api degraded` | HTTP(s) – Keyword | same URL, keyword `"status":"ok"` must be present | body says `degraded` while HTTP is 200 |
|
||||||
|
| 4 | `<stage> collab` | WebSocket | `wss://test.dorfteich.cloud/collab` | connect failure |
|
||||||
|
|
||||||
|
Monitor 3 is what catches `degraded` (stale backup, dead sidecars) —
|
||||||
|
monitor 2 alone only sees hard failures. If the Kuma version has no
|
||||||
|
WebSocket monitor type, an HTTP monitor on
|
||||||
|
`https://<stage>/collab/healthz` (200, includes the DB probe) is the
|
||||||
|
fallback for monitor 4.
|
||||||
|
|
||||||
|
Stages: `test.dorfteich.cloud`, `int.dorfteich.cloud` (web-check-only is
|
||||||
|
acceptable per operations.md — at minimum monitors 1–2, no paging),
|
||||||
|
`dorfteich.online` (Prod, full set + notification; set up and verified at
|
||||||
|
go-live, checklist item in #89).
|
||||||
|
|
||||||
|
Setting up the monitors is an operator action in Uptime-Kuma (owner:
|
||||||
|
Stefan); this file is the definition they follow.
|
||||||
|
|
||||||
|
## Related
|
||||||
|
|
||||||
|
- operations.md §Health & monitoring (endpoint semantics)
|
||||||
|
- ADR 0015 (backup freshness feeds the `backup` check)
|
||||||
|
- deploy/backup/restore.sh (what to do when the backup check warns)
|
||||||
@ -6,21 +6,24 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack.
|
|||||||
## Health & monitoring
|
## Health & monitoring
|
||||||
|
|
||||||
- **Health endpoints**: `api` exposes `/healthz` (liveness: process up) and
|
- **Health endpoints**: `api` exposes `/healthz` (liveness: process up) and
|
||||||
`/readyz` (readiness: DB reachable, migrations applied, converters
|
`/readyz` (readiness: DB reachable + migrations applied = hard failures
|
||||||
reachable, backup freshness < 26 h). `collab` exposes `/healthz`
|
→ 503; converter/renderer reachability and backup freshness < 26 h are
|
||||||
(process + DB). `web` serves a static `/healthz`.
|
warning-level → overall `status: degraded`, still HTTP 200). `collab`
|
||||||
- **Docker healthchecks** on every service (compose `healthcheck:`), so
|
exposes `/healthz` (process + DB). `web` serves a static `/healthz`.
|
||||||
`docker compose ps` and restarts reflect real state;
|
- **Docker healthchecks** use the liveness endpoints only — never `readyz` —
|
||||||
`restart: unless-stopped` everywhere.
|
so a degraded instance is not restart-looped (`restart: unless-stopped`
|
||||||
- **External uptime monitoring**: the operator's existing Uptime-Kuma
|
restarts on process exit, an `unhealthy` mark just shows in
|
||||||
monitors `https://dorfteich.online/healthz` (web), `/api/v1/healthz`, and
|
`docker compose ps`).
|
||||||
a WebSocket check on `/collab`, with notification on failure. Test/Int
|
- **External uptime monitoring**: the operator's existing Uptime-Kuma runs
|
||||||
get web-check-only monitors (no paging).
|
the monitor set defined in `deploy/monitoring.md` (web healthz, api
|
||||||
|
readyz for down, a keyword monitor on the readyz body for degraded, and
|
||||||
|
a collab WebSocket check), with notification on failure for Prod.
|
||||||
|
Test/Int get reduced monitors (no paging).
|
||||||
- **Backup alerting**: the backup sidecar writes
|
- **Backup alerting**: the backup sidecar writes
|
||||||
`backups/status.json` after every run; `readyz` degrades when the last
|
`backups/status.json` after every run; `readyz` reads it (issue #85) and
|
||||||
successful backup is older than 26 h, which surfaces through Uptime-Kuma
|
degrades when the last successful backup is older than 26 h, which
|
||||||
without extra tooling. Additionally the sidecar sends a failure e-mail
|
surfaces through Uptime-Kuma without extra tooling. Additionally the
|
||||||
via the instance SMTP.
|
sidecar sends a failure e-mail via the instance SMTP.
|
||||||
|
|
||||||
## Logging
|
## Logging
|
||||||
|
|
||||||
|
|||||||
42
packages/shared/src/backup-status.ts
Normal file
42
packages/shared/src/backup-status.ts
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
/**
|
||||||
|
* The contract of the backup sidecar's `status.json` (ADR 0015, issue #83):
|
||||||
|
* the sidecar writes it after every run, the api reads it for the readyz
|
||||||
|
* backup-freshness check (issue #85) and the admin panel's backup card
|
||||||
|
* (issue #86). Keep the shape additive — bump `schemaVersion` on breaking
|
||||||
|
* changes.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const BACKUP_STATUS_FILE = 'status.json';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Freshness bound for readiness (operations.md §Health): the nightly cadence
|
||||||
|
* plus a two-hour grace window. An older (or missing) last success degrades
|
||||||
|
* readyz — it never hard-fails it.
|
||||||
|
*/
|
||||||
|
export const BACKUP_FRESH_MAX_AGE_HOURS = 26;
|
||||||
|
|
||||||
|
export interface BackupSizes {
|
||||||
|
dumpBytes: number;
|
||||||
|
archiveBytes: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BackupRun {
|
||||||
|
backupId: string;
|
||||||
|
startedAt: string;
|
||||||
|
finishedAt: string;
|
||||||
|
durationMs: number;
|
||||||
|
outcome: 'succeeded' | 'failed';
|
||||||
|
/** Present on failure: the first error the run hit, as a plain string. */
|
||||||
|
error?: string;
|
||||||
|
/** Present on success. */
|
||||||
|
sizes?: BackupSizes;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BackupStatus {
|
||||||
|
schemaVersion: 1;
|
||||||
|
updatedAt: string;
|
||||||
|
retentionDays: number;
|
||||||
|
lastRun: BackupRun;
|
||||||
|
/** Carried across failed runs so freshness checks see the real gap. */
|
||||||
|
lastSuccess: { backupId: string; finishedAt: string; sizes: BackupSizes } | null;
|
||||||
|
}
|
||||||
@ -96,6 +96,12 @@ export const apiEnvSchema = z.object({
|
|||||||
* here; the relative default serves native dev/test runs.
|
* here; the relative default serves native dev/test runs.
|
||||||
*/
|
*/
|
||||||
PLUGINS_DIR: z.string().min(1).default('./data/plugins'),
|
PLUGINS_DIR: z.string().min(1).default('./data/plugins'),
|
||||||
|
/**
|
||||||
|
* The backup sidecar's volume with the restore sets and `status.json`
|
||||||
|
* (ADR 0015). The api mounts it read-only and only consumes the status
|
||||||
|
* file (readyz freshness, issue #85; admin backup card, issue #86).
|
||||||
|
*/
|
||||||
|
BACKUPS_DIR: z.string().min(1).default('./data/backups'),
|
||||||
/**
|
/**
|
||||||
* Env-backed secret store (security.md §Secrets, issue #80): a mode-600
|
* Env-backed secret store (security.md §Secrets, issue #80): a mode-600
|
||||||
* dotenv-style file on a persistent volume where the setup wizard writes
|
* dotenv-style file on a persistent volume where the setup wizard writes
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
export * from './admin-users';
|
export * from './admin-users';
|
||||||
export * from './api-error';
|
export * from './api-error';
|
||||||
export * from './auth';
|
export * from './auth';
|
||||||
|
export * from './backup-status';
|
||||||
export * from './collab-token';
|
export * from './collab-token';
|
||||||
export * from './editor-schema';
|
export * from './editor-schema';
|
||||||
export * from './env';
|
export * from './env';
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user