From 2bdb0ec2cf78f5d425f0eef1e7347935772a7ae4 Mon Sep 17 00:00:00 2001 From: Claude Fable 5 Date: Fri, 31 Jul 2026 12:21:45 +0200 Subject: [PATCH] =?UTF-8?q?#224:=20read-trail=20storage=20=E2=80=94=20part?= =?UTF-8?q?itioning,=20retention,=20admin=20query=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert read_events to monthly RANGE partitions on occurred_at, with a DEFAULT partition as safety net: a lagging maintenance job must never turn the trail's hard-failure semantics into an outage for classified reads. The dedup unique pair (#223) moves to per-partition indexes (PostgreSQL cannot carry it on the parent); a bucket spanning a month boundary may record one duplicate — over-recording is acceptable, gaps are not. New daily job read-trail-maintenance (job-count fence 9 -> 10) creates months ahead — each with its dedup index — and applies the trail's own retention readTrail.retentionDays (default 365, deliberately independent of audit.retentionDays): whole expired months are DROPped without scanning, remainders deleted by range, every run audited as read_trail.pruned (catalogue v1.2; the fence regex now admits an underscore namespace). Site-Admin query path GET /admin/system/read-events answers "who read page X" and "what did user Y read" within a period — API-only by design, documented. Growth measured and documented in data-model.md: ~1 MB per 1000 events including indexes. Tests: retention pruning + audited deletion + admin queries on the shared database; the partitioned shape, per-partition P2002 dedup, months-ahead creation and DROP-based pruning against a fresh database built by the real migration chain. Refs #224. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AUtYMxwTCMHG9mVHnwbFg8 --- .../migration.sql | 76 +++++ apps/api/prisma/schema.prisma | 8 +- apps/api/src/admin/system-admin.controller.ts | 12 + apps/api/src/admin/system-admin.service.ts | 63 ++++ apps/api/src/audit/audit-actions.ts | 1 + apps/api/src/audit/audit-catalogue.test.ts | 5 +- .../read-trail-maintenance.service.ts | 139 ++++++++ .../read-trail-storage.e2e.db.test.ts | 308 ++++++++++++++++++ apps/api/src/read-trail/read-trail.module.ts | 31 +- .../src/settings/instance-settings.service.ts | 7 + apps/web/e2e/system.spec.ts | 5 +- .../adr/0023-read-access-audit-trail.md | 17 + docs/architecture/audit-events.md | 15 +- docs/architecture/data-model.md | 34 ++ docs/architecture/operations.md | 1 + docs/vs-nfd/20-massnahmenplan.md | 2 +- docs/vs-nfd/50-haertungsleitfaden.md | 1 + packages/shared/src/system.ts | 43 +++ 18 files changed, 750 insertions(+), 18 deletions(-) create mode 100644 apps/api/prisma/migrations/20260731170000_read_events_partitioned/migration.sql create mode 100644 apps/api/src/read-trail/read-trail-maintenance.service.ts create mode 100644 apps/api/src/read-trail/read-trail-storage.e2e.db.test.ts diff --git a/apps/api/prisma/migrations/20260731170000_read_events_partitioned/migration.sql b/apps/api/prisma/migrations/20260731170000_read_events_partitioned/migration.sql new file mode 100644 index 0000000..1cc68ab --- /dev/null +++ b/apps/api/prisma/migrations/20260731170000_read_events_partitioned/migration.sql @@ -0,0 +1,76 @@ +-- #224 (ADR 0023): convert read_events to monthly RANGE partitions on +-- occurred_at. Volume grows unbounded with use; retention then DROPs whole +-- expired partitions instead of scanning deletes. The primary key gains the +-- partition column (PostgreSQL requirement); the dedup unique pair +-- (dedup_key, window_bucket) moves to PER-PARTITION unique indexes — a +-- partitioned parent cannot carry it without the partition key. A bucket +-- spanning a month boundary can therefore record one duplicate; documented +-- in ADR 0023, over-recording is acceptable, gaps are not. +-- +-- A DEFAULT partition catches rows outside every maintained range, so a +-- lagging maintenance job can never make classified reads fail (the trail's +-- hard-failure semantics would otherwise turn an ops miss into an outage). + +ALTER TABLE "read_events" RENAME TO "read_events_old"; +ALTER INDEX "read_events_pkey" RENAME TO "read_events_old_pkey"; +ALTER INDEX "read_events_dedup_key_window_bucket_key" RENAME TO "read_events_old_dedup_key"; +ALTER INDEX "read_events_page_id_occurred_at_idx" RENAME TO "read_events_old_page_idx"; +ALTER INDEX "read_events_actor_id_occurred_at_idx" RENAME TO "read_events_old_actor_idx"; +ALTER INDEX "read_events_occurred_at_idx" RENAME TO "read_events_old_at_idx"; + +CREATE TABLE "read_events" ( + "id" TEXT NOT NULL, + "occurred_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "actor_id" TEXT, + "session_key" TEXT NOT NULL, + "page_id" TEXT, + "pond_id" TEXT NOT NULL, + "channel" TEXT NOT NULL, + "classification" TEXT NOT NULL, + "details" JSONB, + "dedup_key" TEXT NOT NULL, + "window_bucket" BIGINT NOT NULL, + "window_seconds" INTEGER NOT NULL, + + CONSTRAINT "read_events_pkey" PRIMARY KEY ("id", "occurred_at") +) PARTITION BY RANGE ("occurred_at"); + +-- Non-unique parent indexes propagate to every partition automatically. +CREATE INDEX "read_events_page_id_occurred_at_idx" ON "read_events"("page_id", "occurred_at"); +CREATE INDEX "read_events_actor_id_occurred_at_idx" ON "read_events"("actor_id", "occurred_at"); +CREATE INDEX "read_events_occurred_at_idx" ON "read_events"("occurred_at"); + +-- The safety-net partition, plus the current and the next month — the daily +-- maintenance job (read-trail-maintenance) keeps creating months ahead and +-- adds the same per-partition dedup index to each new one. +CREATE TABLE "read_events_default" PARTITION OF "read_events" DEFAULT; +CREATE UNIQUE INDEX "read_events_default_dedup_key" + ON "read_events_default"("dedup_key", "window_bucket"); + +DO $$ +DECLARE + m DATE; + part TEXT; +BEGIN + FOR i IN 0..1 LOOP + m := date_trunc('month', now())::date + (i || ' month')::interval; + part := 'read_events_y' || to_char(m, 'YYYY') || 'm' || to_char(m, 'MM'); + EXECUTE format( + 'CREATE TABLE %I PARTITION OF "read_events" FOR VALUES FROM (%L) TO (%L)', + part, m, m + interval '1 month'); + EXECUTE format( + 'CREATE UNIQUE INDEX %I ON %I ("dedup_key", "window_bucket")', + part || '_dedup_key', part); + END LOOP; +END $$; + +INSERT INTO "read_events" + ("id", "occurred_at", "actor_id", "session_key", "page_id", "pond_id", + "channel", "classification", "details", "dedup_key", "window_bucket", + "window_seconds") +SELECT "id", "occurred_at", "actor_id", "session_key", "page_id", "pond_id", + "channel", "classification", "details", "dedup_key", "window_bucket", + "window_seconds" + FROM "read_events_old"; + +DROP TABLE "read_events_old"; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 228ce7b..61b07a0 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -96,8 +96,13 @@ model AuditEntry { /// volume, purpose and legal basis all differ. Deliberately WITHOUT foreign /// keys: evidence must survive a page purge and a hard user deletion — the /// ids stay as recorded (pseudonymous uuids), history is never rewritten. +/// +/// In migrated databases the table is RANGE-partitioned by `occurred_at` +/// (monthly, issue #224) — hence the composite id. The dedup unique pair +/// lives per partition there (a partitioned parent cannot carry it without +/// the partition key); `db push` test databases get it on the plain table. model ReadEvent { - id String @id @default(uuid()) + id String @default(uuid()) occurredAt DateTime @default(now()) @map("occurred_at") /// Null = anonymous reader (public grant); `sessionKey` still names the /// browsing session, so the anonymous marker is explicit, not an accident. @@ -124,6 +129,7 @@ model ReadEvent { /// represents up to this many seconds, so the evidence is not overread. windowSeconds Int @map("window_seconds") + @@id([id, occurredAt]) @@unique([dedupKey, windowBucket]) @@index([pageId, occurredAt]) @@index([actorId, occurredAt]) diff --git a/apps/api/src/admin/system-admin.controller.ts b/apps/api/src/admin/system-admin.controller.ts index 09e40ce..362d92a 100644 --- a/apps/api/src/admin/system-admin.controller.ts +++ b/apps/api/src/admin/system-admin.controller.ts @@ -1,9 +1,12 @@ import { Controller, Get, Param, Post, Query, Req, UseGuards } from '@nestjs/common'; import { auditListQuerySchema, + readEventListQuerySchema, type AuditListQuery, type AuditListView, type JobTriggerResult, + type ReadEventListQuery, + type ReadEventListView, type StorageOverviewView, type SystemBackupView, type SystemJobView, @@ -45,6 +48,15 @@ export class SystemAdminController { return this.system.auditLog(query); } + /** Read-access trail queries (issue #224): "who read page X", "what did + * user Y read" — Site-Admin only, like the audit viewer above. */ + @Get('read-events') + async readEvents( + @Query(new ZodValidationPipe(readEventListQuerySchema)) query: ReadEventListQuery, + ): Promise { + return this.system.readEvents(query); + } + @Get('storage') async storage(): Promise { return this.system.storage(); diff --git a/apps/api/src/admin/system-admin.service.ts b/apps/api/src/admin/system-admin.service.ts index 95b55b8..6ff46f1 100644 --- a/apps/api/src/admin/system-admin.service.ts +++ b/apps/api/src/admin/system-admin.service.ts @@ -7,10 +7,13 @@ import { AUDIT_PAGE_SIZE, BACKUP_FRESH_MAX_AGE_HOURS, BACKUP_STATUS_FILE, + READ_EVENT_PAGE_SIZE, type AuditListQuery, type AuditListView, type BackupStatus, type JobTriggerResult, + type ReadEventListQuery, + type ReadEventListView, type StorageOverviewView, type SystemBackupView, type SystemJobView, @@ -160,6 +163,66 @@ export class SystemAdminService { }; } + /** + * The Site-Admin query path over the read-access trail (issue #224, + * ADR 0023) — evidence nobody can read is not evidence. Answers "who read + * page X" and "what did user Y read" within a period. API-only by design + * (no panel yet): the trail is an examiner's tool, not a daily screen — + * documented in data-model.md §read_events. + */ + async readEvents(query: ReadEventListQuery): Promise { + const where: Prisma.ReadEventWhereInput = {}; + if (query.pageId) where.pageId = query.pageId; + if (query.actor) { + const actor = await this.prisma.user.findUnique({ where: { username: query.actor } }); + // An unknown username matches nothing rather than everything. + where.actorId = actor?.id ?? '00000000-0000-0000-0000-000000000000'; + } + if (query.channel) where.channel = query.channel; + if (query.from || query.to) { + where.occurredAt = { + ...(query.from ? { gte: query.from } : {}), + ...(query.to ? { lte: query.to } : {}), + }; + } + + const total = await this.prisma.readEvent.count({ where }); + const pageCount = Math.max(1, Math.ceil(total / READ_EVENT_PAGE_SIZE)); + const page = Math.min(query.page, pageCount); + const events = await this.prisma.readEvent.findMany({ + where, + orderBy: { occurredAt: 'desc' }, + skip: (page - 1) * READ_EVENT_PAGE_SIZE, + take: READ_EVENT_PAGE_SIZE, + }); + // No FK on actor_id (evidence outlives accounts) — resolve what still + // exists in one query, show the bare id otherwise. + const actorIds = [...new Set(events.map((e) => e.actorId).filter((id): id is string => !!id))]; + const actors = actorIds.length + ? await this.prisma.user.findMany({ + where: { id: { in: actorIds } }, + select: { id: true, username: true, displayName: true }, + }) + : []; + const actorById = new Map(actors.map((a) => [a.id, a])); + return { + entries: events.map((event) => ({ + id: event.id, + occurredAt: event.occurredAt.toISOString(), + actor: event.actorId ? (actorById.get(event.actorId) ?? null) : null, + pageId: event.pageId, + pondId: event.pondId, + channel: event.channel, + classification: event.classification, + windowSeconds: event.windowSeconds, + details: (event.details as Record | null) ?? null, + })), + page, + pageCount, + total, + }; + } + async storage(): Promise { const usages = await this.prisma.pondUsage.findMany({ where: { pond: { deletedAt: null } }, diff --git a/apps/api/src/audit/audit-actions.ts b/apps/api/src/audit/audit-actions.ts index 49e80da..5156293 100644 --- a/apps/api/src/audit/audit-actions.ts +++ b/apps/api/src/audit/audit-actions.ts @@ -40,6 +40,7 @@ export const AUDIT_EVENTS = { 'pond.purged': { severity: 'notice' }, 'quota.override_cleared': { severity: 'notice' }, 'quota.override_set': { severity: 'notice' }, + 'read_trail.pruned': { severity: 'info' }, 'settings.changed': { severity: 'notice' }, 'setup.admin_created': { severity: 'notice' }, 'setup.completed': { severity: 'info' }, diff --git a/apps/api/src/audit/audit-catalogue.test.ts b/apps/api/src/audit/audit-catalogue.test.ts index 04ef16e..61d23cf 100644 --- a/apps/api/src/audit/audit-catalogue.test.ts +++ b/apps/api/src/audit/audit-catalogue.test.ts @@ -18,11 +18,12 @@ import { AUDIT_EVENTS } from './audit-actions'; const doc = readFileSync(join(__dirname, '../../../../docs/architecture/audit-events.md'), 'utf8'); /** Event rows are `| \`ns.event\` | trigger | severity | …` — the dot in the - * id keeps field-set rows (`msg`, `severity`, …) out of the match. */ + * id keeps field-set rows (`msg`, `severity`, …) out of the match. The + * namespace may carry an underscore since `read_trail.*` (issue #224). */ function documentedEvents(): Map { const events = new Map(); for (const line of doc.split('\n')) { - const id = /^\| `([a-z]+\.[a-z_]+)` +\|/.exec(line)?.[1]; + const id = /^\| `([a-z_]+\.[a-z_]+)` +\|/.exec(line)?.[1]; if (!id) continue; const cells = line.split('|').map((cell) => cell.trim()); // cells[0] is the empty string before the leading pipe. diff --git a/apps/api/src/read-trail/read-trail-maintenance.service.ts b/apps/api/src/read-trail/read-trail-maintenance.service.ts new file mode 100644 index 0000000..71b0f69 --- /dev/null +++ b/apps/api/src/read-trail/read-trail-maintenance.service.ts @@ -0,0 +1,139 @@ +import { Injectable } from '@nestjs/common'; +import { PinoLogger } from 'nestjs-pino'; + +import { AuditService } from '../audit/audit.service'; +import { ClockService } from '../common/clock.service'; +import { PrismaService } from '../prisma/prisma.service'; +import { InstanceSettingsService } from '../settings/instance-settings.service'; + +const MS_PER_DAY = 24 * 60 * 60 * 1000; + +/** How many months ahead of "now" a partition must exist. Two keeps a + * multi-week job outage from ever reaching an uncovered month (the DEFAULT + * partition would still catch it — reads never fail on a missing month). */ +const MONTHS_AHEAD = 2; + +/** + * Read-trail storage maintenance (issue #224, ADR 0023), one daily job with + * two duties: + * + * 1. **Partition upkeep** — `read_events` is RANGE-partitioned by month + * (migration `20260731170000`); this creates the next {@link MONTHS_AHEAD} + * monthly partitions, each with its per-partition dedup unique index + * (#223 — the partitioned parent cannot carry it). A `db push` database + * (tests) has a plain table; partition work skips itself there. + * 2. **Retention** — events older than `readTrail.retentionDays` are + * removed: whole months by dropping their partition (no scan), the + * remainder (default partition, plain tables) by a ranged delete. The + * deletion is audited (`read_trail.pruned`) so a gap in the evidence is + * always explainable — same principle as `audit.pruned` (#196), but a + * deliberately separate period. + */ +@Injectable() +export class ReadTrailMaintenanceService { + constructor( + private readonly prisma: PrismaService, + private readonly settings: InstanceSettingsService, + private readonly audit: AuditService, + private readonly clock: ClockService, + private readonly logger: PinoLogger, + ) { + this.logger.setContext(ReadTrailMaintenanceService.name); + } + + async run(): Promise { + await this.ensurePartitions(); + await this.pruneExpired(); + } + + /** True when read_events is a partitioned parent (relkind `p`). */ + private async isPartitioned(): Promise { + const rows = await this.prisma.$queryRaw<{ relkind: string }[]>` + SELECT relkind::text FROM pg_class + WHERE relname = 'read_events' AND relnamespace = 'public'::regnamespace`; + return rows[0]?.relkind === 'p'; + } + + /** `read_events_y2026m08` for 2026-08. */ + private partitionName(month: Date): string { + const y = month.getUTCFullYear(); + const m = String(month.getUTCMonth() + 1).padStart(2, '0'); + return `read_events_y${y}m${m}`; + } + + private monthStart(base: Date, offsetMonths: number): Date { + return new Date(Date.UTC(base.getUTCFullYear(), base.getUTCMonth() + offsetMonths, 1)); + } + + async ensurePartitions(): Promise { + if (!(await this.isPartitioned())) { + this.logger.debug('read_events is not partitioned here; skipping partition upkeep'); + return; + } + const now = this.clock.now(); + for (let offset = 0; offset <= MONTHS_AHEAD; offset += 1) { + const from = this.monthStart(now, offset); + const to = this.monthStart(now, offset + 1); + const name = this.partitionName(from); + try { + await this.prisma.$executeRawUnsafe( + `CREATE TABLE IF NOT EXISTS "${name}" PARTITION OF "read_events" + FOR VALUES FROM ('${from.toISOString()}') TO ('${to.toISOString()}')`, + ); + await this.prisma.$executeRawUnsafe( + `CREATE UNIQUE INDEX IF NOT EXISTS "${name}_dedup_key" + ON "${name}" ("dedup_key", "window_bucket")`, + ); + } catch (error) { + // Most likely: the DEFAULT partition already holds rows of this month + // (the job lagged past a month boundary). Nothing is lost — those + // rows live in the default partition and age out through the ranged + // delete below; the month just cannot get its own partition anymore. + this.logger.warn({ partition: name, err: error }, 'read-trail partition not created'); + } + } + } + + async pruneExpired(): Promise { + const retentionDays = await this.settings.get('readTrail.retentionDays'); + const cutoff = new Date(this.clock.now().getTime() - retentionDays * MS_PER_DAY); + let dropped = 0; + + if (await this.isPartitioned()) { + // Whole months strictly before the cutoff month go by DROP — no scan, + // and the dropped range is exact (every row in them is < cutoff). + const partitions = await this.prisma.$queryRaw<{ relname: string }[]>` + SELECT c.relname::text + FROM pg_inherits i + JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = 'read_events'::regclass + AND c.relname ~ '^read_events_y[0-9]{4}m[0-9]{2}$'`; + const cutoffMonth = this.monthStart(cutoff, 0); + for (const { relname } of partitions) { + const match = /^read_events_y(\d{4})m(\d{2})$/.exec(relname); + if (!match) continue; + const monthEnd = new Date(Date.UTC(Number(match[1]), Number(match[2]), 1)); + if (monthEnd.getTime() > cutoffMonth.getTime()) continue; + const counted = await this.prisma.$queryRawUnsafe<{ count: bigint }[]>( + `SELECT count(*)::bigint AS count FROM "${relname}"`, + ); + dropped += Number(counted[0]?.count ?? 0n); + await this.prisma.$executeRawUnsafe(`DROP TABLE "${relname}"`); + } + } + + // The remainder: rows before the cutoff inside surviving partitions, + // the default partition, or a plain (test) table. + const deleted = await this.prisma.readEvent.deleteMany({ + where: { occurredAt: { lt: cutoff } }, + }); + const count = dropped + deleted.count; + if (count > 0) { + await this.audit.record({ + action: 'read_trail.pruned', + details: { count, cutoff: cutoff.toISOString(), retentionDays }, + }); + } + return count; + } +} diff --git a/apps/api/src/read-trail/read-trail-storage.e2e.db.test.ts b/apps/api/src/read-trail/read-trail-storage.e2e.db.test.ts new file mode 100644 index 0000000..bf26b73 --- /dev/null +++ b/apps/api/src/read-trail/read-trail-storage.e2e.db.test.ts @@ -0,0 +1,308 @@ +import { execFileSync } from 'node:child_process'; +import { join } from 'node:path'; + +import { INestApplication } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; +import { PinoLogger } from 'nestjs-pino'; +import request from 'supertest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; + +import { ClockService } from '../common/clock.service'; +import { InstanceSettingsService } from '../settings/instance-settings.service'; +import { createTestApp, sessionCookieOf } from '../testing/test-app'; +import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db'; +import { UsersService } from '../users/users.service'; + +import { ReadTrailMaintenanceService } from './read-trail-maintenance.service'; + +const API_ROOT = join(__dirname, '..', '..'); +const MS_PER_DAY = 24 * 60 * 60 * 1000; + +/** + * Read-trail storage (issue #224, ADR 0023): its own retention period with + * an audited deletion, the Site-Admin query path, and — against a freshly + * migrated database, where the real DDL applies — the monthly RANGE + * partitioning with per-partition dedup indexes and DROP-based pruning. + */ +describe.skipIf(!hasTestDb)('read-trail storage (e2e, issue #224)', () => { + const suffix = uniqueSuffix(); + + describe('retention and admin query path (shared database)', () => { + let app: INestApplication; + let prisma: PrismaClient; + const password = 'lesetrail speicher 123'; + + let adminId: string; + let adminCookie: string; + let readerCookie: string; + // Plain text ids — read_events has no FKs by design (#222). + const pondId = `pond-${suffix}`; + + const api = () => request(app.getHttpServer()); + + async function makeUser(handle: string, siteAdmin: boolean) { + const users = app.get(UsersService); + const user = await users.createUser({ + username: `${handle}-${suffix}`, + email: `${handle}-${suffix}@example.test`, + displayName: handle, + password, + locale: 'en', + }); + await users.markEmailVerified(user.id); + if (siteAdmin) { + await prisma.user.update({ where: { id: user.id }, data: { isSiteAdmin: true } }); + } + const cookie = sessionCookieOf( + await api() + .post('/api/v1/auth/login') + .send({ usernameOrEmail: `${handle}-${suffix}`, password }) + .expect(200), + ); + return { id: user.id, cookie }; + } + + function eventRow(overrides: Record) { + return { + actorId: null, + sessionKey: 'anon', + pageId: null, + pondId, + channel: 'page_view', + classification: 'vs_nfd', + dedupKey: `k-${suffix}-${Math.random().toString(36).slice(2)}`, + windowBucket: BigInt(Math.floor(Math.random() * 1_000_000_000)), + windowSeconds: 300, + ...overrides, + }; + } + + beforeAll(async () => { + prisma = createTestPrisma(); + await prisma.rateLimit.deleteMany({}); + app = await createTestApp(); + const admin = await makeUser('storage-admin', true); + adminId = admin.id; + adminCookie = admin.cookie; + const reader = await makeUser('storage-reader', false); + readerCookie = reader.cookie; + }); + + afterAll(async () => { + await prisma.readEvent.deleteMany({ where: { pondId } }); + await prisma.auditEntry.deleteMany({ where: { action: 'read_trail.pruned' } }); + await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.retentionDays' } }); + await prisma.user.deleteMany({ where: { username: { contains: suffix } } }); + await prisma.$disconnect(); + await app.close(); + }); + + it('prunes only events past its own period and audits the deletion', async () => { + await app.get(InstanceSettingsService).set('readTrail.retentionDays', 30, adminId); + const now = Date.now(); + await prisma.readEvent.createMany({ + data: [ + eventRow({ occurredAt: new Date(now - 40 * MS_PER_DAY) }), + eventRow({ occurredAt: new Date(now - 31 * MS_PER_DAY) }), + eventRow({ occurredAt: new Date(now - 5 * MS_PER_DAY) }), + ], + }); + + const maintenance = app.get(ReadTrailMaintenanceService); + const pruned = await maintenance.pruneExpired(); + expect(pruned).toBeGreaterThanOrEqual(2); + + const remaining = await prisma.readEvent.findMany({ where: { pondId } }); + expect(remaining).toHaveLength(1); + + const audit = await prisma.auditEntry.findFirst({ + where: { action: 'read_trail.pruned' }, + orderBy: { at: 'desc' }, + }); + expect(audit).not.toBeNull(); + expect(audit!.details).toMatchObject({ retentionDays: 30 }); + }); + + it('answers "who read page X" and "what did user Y read" for Site Admins only', async () => { + const pageId = `page-${suffix}`; + await prisma.readEvent.createMany({ + data: [ + eventRow({ pageId, actorId: adminId, channel: 'export' }), + eventRow({ pageId: `other-${suffix}`, actorId: null }), + ], + }); + + const byPage = await api() + .get(`/api/v1/admin/system/read-events?pageId=${pageId}`) + .set('Cookie', adminCookie) + .expect(200); + expect(byPage.body.total).toBe(1); + expect(byPage.body.entries[0]).toMatchObject({ + pageId, + channel: 'export', + windowSeconds: 300, + }); + expect(byPage.body.entries[0].actor).toMatchObject({ id: adminId }); + + const byActor = await api() + .get(`/api/v1/admin/system/read-events?actor=storage-admin-${suffix}`) + .set('Cookie', adminCookie) + .expect(200); + expect(byActor.body.total).toBe(1); + + // An unknown username matches nothing rather than everything. + const unknown = await api() + .get(`/api/v1/admin/system/read-events?actor=nobody-${suffix}`) + .set('Cookie', adminCookie) + .expect(200); + expect(unknown.body.total).toBe(0); + + await api() + .get(`/api/v1/admin/system/read-events?pageId=${pageId}`) + .set('Cookie', readerCookie) + .expect(403); + }); + }); + + describe('partitioned shape (fresh database, real migrations)', () => { + const baseUrl = process.env.TEST_DATABASE_URL!; + const dbName = `dorfteich_trail_${suffix}`; + let fresh: PrismaClient; + let maintenance: ReadTrailMaintenanceService; + const auditRecord = vi.fn().mockResolvedValue(undefined); + let retentionDays = 365; + + function freshUrl(): string { + const url = new URL(baseUrl); + url.pathname = `/${dbName}`; + return url.toString(); + } + + beforeAll(async () => { + const admin = new PrismaClient({ datasourceUrl: baseUrl }); + try { + await admin.$executeRawUnsafe(`CREATE DATABASE "${dbName}"`); + } finally { + await admin.$disconnect(); + } + execFileSync( + process.execPath, + [join(API_ROOT, 'node_modules', 'prisma', 'build', 'index.js'), 'migrate', 'deploy'], + { env: { ...process.env, DATABASE_URL: freshUrl() }, stdio: 'pipe', cwd: API_ROOT }, + ); + fresh = new PrismaClient({ datasourceUrl: freshUrl() }); + const settingsStub = { + get: async () => retentionDays, + } as unknown as InstanceSettingsService; + const auditStub = { record: auditRecord } as never; + const loggerStub = { + setContext: () => undefined, + debug: () => undefined, + info: () => undefined, + warn: () => undefined, + error: () => undefined, + } as unknown as PinoLogger; + maintenance = new ReadTrailMaintenanceService( + fresh as never, + settingsStub, + auditStub, + new ClockService(), + loggerStub, + ); + }, 120_000); + + afterAll(async () => { + await fresh.$disconnect(); + const admin = new PrismaClient({ datasourceUrl: baseUrl }); + try { + await admin.$executeRawUnsafe(`DROP DATABASE IF EXISTS "${dbName}" WITH (FORCE)`); + } finally { + await admin.$disconnect(); + } + }); + + it('migrates read_events to a partitioned parent with current-month coverage', async () => { + const kind = await fresh.$queryRaw<{ relkind: string }[]>` + SELECT relkind::text FROM pg_class WHERE relname = 'read_events'`; + expect(kind[0]!.relkind).toBe('p'); + + const partitions = await fresh.$queryRaw<{ relname: string }[]>` + SELECT c.relname::text FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = 'read_events'::regclass ORDER BY c.relname`; + const names = partitions.map((p) => p.relname); + expect(names).toContain('read_events_default'); + expect(names.some((n) => /^read_events_y\d{4}m\d{2}$/.test(n))).toBe(true); + }); + + it('keeps the dedup unique pair enforced per partition (P2002 on the duplicate)', async () => { + const row = { + actorId: null, + sessionKey: 'anon', + pageId: null, + pondId: 'pond-part', + channel: 'page_view', + classification: 'vs_nfd', + dedupKey: `dup-${suffix}`, + windowBucket: 42n, + windowSeconds: 300, + }; + await fresh.readEvent.create({ data: row }); + await expect(fresh.readEvent.create({ data: { ...row } })).rejects.toMatchObject({ + code: 'P2002', + }); + }); + + it('creates months ahead and prunes whole expired partitions by DROP, audited', async () => { + await maintenance.ensurePartitions(); + const next = new Date(); + const nextMonth = new Date(Date.UTC(next.getUTCFullYear(), next.getUTCMonth() + 2, 1)); + const nextName = `read_events_y${nextMonth.getUTCFullYear()}m${String( + nextMonth.getUTCMonth() + 1, + ).padStart(2, '0')}`; + const created = await fresh.$queryRawUnsafe<{ relname: string }[]>( + `SELECT relname::text FROM pg_class WHERE relname = '${nextName}'`, + ); + expect(created).toHaveLength(1); + // ...and each new partition carries its own dedup unique index. + const index = await fresh.$queryRawUnsafe<{ indexname: string }[]>( + `SELECT indexname::text FROM pg_indexes WHERE tablename = '${nextName}' + AND indexname = '${nextName}_dedup_key'`, + ); + expect(index).toHaveLength(1); + + // An old month: partition + one event well past retention. + await fresh.$executeRawUnsafe( + `CREATE TABLE "read_events_y2020m01" PARTITION OF "read_events" + FOR VALUES FROM ('2020-01-01') TO ('2020-02-01')`, + ); + await fresh.readEvent.create({ + data: { + occurredAt: new Date('2020-01-15T12:00:00Z'), + actorId: null, + sessionKey: 'anon', + pageId: null, + pondId: 'pond-part', + channel: 'export', + classification: 'vs_nfd', + dedupKey: `old-${suffix}`, + windowBucket: 1n, + windowSeconds: 300, + }, + }); + + const pruned = await maintenance.pruneExpired(); + expect(pruned).toBeGreaterThanOrEqual(1); + const gone = await fresh.$queryRawUnsafe<{ relname: string }[]>( + `SELECT relname::text FROM pg_class WHERE relname = 'read_events_y2020m01'`, + ); + expect(gone).toHaveLength(0); + expect(auditRecord).toHaveBeenCalledWith( + expect.objectContaining({ action: 'read_trail.pruned' }), + ); + // The current month's events survive. + const kept = await fresh.readEvent.count({ where: { dedupKey: `dup-${suffix}` } }); + expect(kept).toBe(1); + retentionDays = 365; // restore for any later use + }); + }); +}); diff --git a/apps/api/src/read-trail/read-trail.module.ts b/apps/api/src/read-trail/read-trail.module.ts index 61a12aa..3f812de 100644 --- a/apps/api/src/read-trail/read-trail.module.ts +++ b/apps/api/src/read-trail/read-trail.module.ts @@ -1,10 +1,16 @@ -import { Global, Module } from '@nestjs/common'; +import { Global, Module, OnModuleInit } from '@nestjs/common'; import { CommonModule } from '../common/common.module'; +import { SchedulerModule } from '../scheduler/scheduler.module'; +import { SchedulerService } from '../scheduler/scheduler.service'; import { SettingsModule } from '../settings/settings.module'; +import { ReadTrailMaintenanceService } from './read-trail-maintenance.service'; import { ReadTrailService } from './read-trail.service'; +/** Daily, per operations.md's maintenance-jobs table (issue #224). */ +const READ_TRAIL_MAINTENANCE_CADENCE_SECONDS = 24 * 60 * 60; + /** * Global like AuditModule and for the same reason: the read-access trail * (issue #222, ADR 0023) cuts across every module that serves page content — @@ -12,8 +18,23 @@ import { ReadTrailService } from './read-trail.service'; */ @Global() @Module({ - imports: [CommonModule, SettingsModule], - providers: [ReadTrailService], - exports: [ReadTrailService], + imports: [CommonModule, SchedulerModule, SettingsModule], + providers: [ReadTrailService, ReadTrailMaintenanceService], + exports: [ReadTrailService, ReadTrailMaintenanceService], }) -export class ReadTrailModule {} +export class ReadTrailModule implements OnModuleInit { + constructor( + private readonly scheduler: SchedulerService, + private readonly maintenance: ReadTrailMaintenanceService, + ) {} + + onModuleInit(): void { + this.scheduler.register({ + name: 'read-trail-maintenance', + cadenceSeconds: READ_TRAIL_MAINTENANCE_CADENCE_SECONDS, + run: async () => { + await this.maintenance.run(); + }, + }); + } +} diff --git a/apps/api/src/settings/instance-settings.service.ts b/apps/api/src/settings/instance-settings.service.ts index 30b7be0..11a84ec 100644 --- a/apps/api/src/settings/instance-settings.service.ts +++ b/apps/api/src/settings/instance-settings.service.ts @@ -56,6 +56,13 @@ export const INSTANCE_SETTINGS = { // 5 minutes keeps a live Yjs session (collab tokens every 60 s) at a // bounded ~12 events/hour/page while still evidencing distinct visits. 'readTrail.dedupWindowMinutes': z.number().int().min(1).default(5), + // Read-trail retention (issue #224): days a read event is kept before the + // daily maintenance job removes it — deliberately independent of + // `audit.retentionDays` (#196), because volume, purpose and legal basis + // differ. The deletion itself is audited (`read_trail.pruned`), so a gap + // is always explainable. One year mirrors the audit default; shortening + // it is an operator decision under the purpose limitation (#225). + 'readTrail.retentionDays': z.number().int().min(1).default(365), // Default VS-NfD classification for newly created pages (ADR 0022, // issue #204). An instance operated inside a classified environment sets // this to `vs_nfd` so nothing starts unmarked; inheritance from the diff --git a/apps/web/e2e/system.spec.ts b/apps/web/e2e/system.spec.ts index 203ca7e..2d3dd9a 100644 --- a/apps/web/e2e/system.spec.ts +++ b/apps/web/e2e/system.spec.ts @@ -20,8 +20,9 @@ test('lists maintenance jobs and triggers one manually', async ({ browser }) => // Keep in sync with the scheduler registrations: trash-purge, // version-thinning, page-compaction, data-export-purge, // notification-digest, orphan-file-sweep (#194), audit-retention (#196), - // conversion-payload-prune (#233), mail-outbox-retention (#234). - await expect(jobsTable.locator('tbody tr')).toHaveCount(9); + // conversion-payload-prune (#233), mail-outbox-retention (#234), + // read-trail-maintenance (#224). + await expect(jobsTable.locator('tbody tr')).toHaveCount(10); const firstRow = jobsTable.locator('tbody tr').first(); await firstRow.getByRole('button').click(); diff --git a/docs/architecture/adr/0023-read-access-audit-trail.md b/docs/architecture/adr/0023-read-access-audit-trail.md index a5681d7..38979c5 100644 --- a/docs/architecture/adr/0023-read-access-audit-trail.md +++ b/docs/architecture/adr/0023-read-access-audit-trail.md @@ -89,6 +89,23 @@ traffic per open document. anonymous browsers apart would require fingerprinting (IP/UA), which the purpose limitation (#225) rules out. +## Decisions taken in #224 + +- **Monthly RANGE partitions** on `occurred_at`, plus a DEFAULT partition + as a safety net: a lagging maintenance job must never make classified + reads fail (the trail write is hard-failing — an ops miss must not + become an outage). Retention DROPs whole expired months without + scanning; the pruning run is audited (`read_trail.pruned`) under the + trail's own `readTrail.retentionDays` (default 365). +- **The dedup unique pair lives per partition** (PostgreSQL cannot carry + it on the parent without the partition key). A dedup bucket spanning a + month boundary can therefore record one duplicate — over-recording is + acceptable, gaps are not. +- **Query path is API-only** (`GET /admin/system/read-events`, Site + Admin): the trail is an examiner's tool, not a daily screen. Evidence + nobody can read is not evidence — the path exists; a panel does not. +- **Growth measured**: ~1 MB per 1000 events including indexes. + ## Consequences - The scope limit is the feature's strongest argument in the works-council diff --git a/docs/architecture/audit-events.md b/docs/architecture/audit-events.md index bd2f5a8..8e2e695 100644 --- a/docs/architecture/audit-events.md +++ b/docs/architecture/audit-events.md @@ -1,7 +1,7 @@ # Audit event catalogue -**Catalogue version 1.1 (2026-07-31; 1.1 adds `page.classification_*`, -issue #205).** +**Catalogue version 1.2 (2026-07-31; 1.2 adds `read_trail.pruned`, +issue #224; 1.1 added `page.classification_*`, issue #205).** This is the operator-facing contract for the audit trail: every event id the application can emit, with its trigger, severity, actor/target @@ -103,11 +103,12 @@ failure), `warning` = feeds detection (suspicious or destructive), ### Content integrity & lifecycle (`file.*`, `pond.*`, `audit.*`) -| Id | Trigger | Severity | Actor | Target | Fields | -| ----------------------- | ------------------------------------------------------------------- | -------- | ---------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | -| `file.integrity_failed` | Attachment download hash mismatch — fail-closed (issue #199) | critical | `null` (any downloader; detection) | `attachment` | `pondId`, `expected` (stored sha256), `actual` (computed sha256) | -| `pond.purged` | Pond irreversibly destroyed (manual or trash retention, issue #193) | notice | admin, `null` when retention-run | `pond` | `trigger` (`manual` \| `retention`) plus per-object-type deletion counts (e.g. `pages`, `attachments`, … — informational, keys may grow) | -| `audit.pruned` | Audit retention deleted rows past the period (issue #196) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` | +| Id | Trigger | Severity | Actor | Target | Fields | +| ----------------------- | -------------------------------------------------------------------- | -------- | ---------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `file.integrity_failed` | Attachment download hash mismatch — fail-closed (issue #199) | critical | `null` (any downloader; detection) | `attachment` | `pondId`, `expected` (stored sha256), `actual` (computed sha256) | +| `pond.purged` | Pond irreversibly destroyed (manual or trash retention, issue #193) | notice | admin, `null` when retention-run | `pond` | `trigger` (`manual` \| `retention`) plus per-object-type deletion counts (e.g. `pages`, `attachments`, … — informational, keys may grow) | +| `audit.pruned` | Audit retention deleted rows past the period (issue #196) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` | +| `read_trail.pruned` | Read-trail retention removed events past its own period (issue #224) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` | ### Public API (`api.*`) diff --git a/docs/architecture/data-model.md b/docs/architecture/data-model.md index 7ad29d3..ed6a8b9 100644 --- a/docs/architecture/data-model.md +++ b/docs/architecture/data-model.md @@ -186,6 +186,40 @@ Cached counters per pond: `storage_bytes_used`, `editor_count`, - `jobs`: conversion jobs for import/export (ADR 0009) and maintenance jobs (compaction, trash purge, quota reconciliation) with status + timestamps. +## Read-access trail (`read_events`, issues #222–#224, ADR 0023) + +One row per read of a `classification = vs_nfd` page per channel and dedup +window — separate from `audit_log` because volume, purpose and legal basis +all differ. + +| Column | Notes | +| ---------------------------- | ------------------------------------------------------------------------------------ | +| `id`, `occurred_at` | composite PK (the partition key must be part of it) | +| `actor_id` | reader, null = anonymous; **no FK** — evidence outlives accounts | +| `session_key` | `session:` / `token:` / `job:` / `anon` | +| `page_id`, `pond_id` | plain ids, **no FK** — evidence outlives purges | +| `channel` | `page_view` / `no_js_shell` / `public_api` / `attachment` / `export` / `collab_join` | +| `classification` | at read time; reclassification never rewrites history | +| `dedup_key`, `window_bucket` | unique pair per partition — race-free dedup (#223) | +| `window_seconds` | the row represents up to this many seconds of access | +| `details` | small context (export format, attachment id) — never content | + +Storage shape (issue #224): RANGE-partitioned by month on `occurred_at`, +with a DEFAULT partition as safety net (a lagging maintenance job must +never make classified reads fail — the trail write is hard-failing by +design). The daily `read-trail-maintenance` job creates months ahead +(each with its per-partition dedup index) and applies the trail's own +retention `readTrail.retentionDays` (default 365): whole expired months +are DROPped without scanning, remainders deleted by range; every pruning +run is audited as `read_trail.pruned`. Growth, measured on PG 17: +**~1 MB per 1000 events** including indexes (≈970 bytes/row) — a +100-person instance at hundreds of classified reads/day stays in the tens +of MB per year. + +Query path: `GET /admin/system/read-events` (Site Admin, filters +`pageId`, `actor`, `channel`, `from`/`to`) — deliberately API-only, no +panel: the trail is an examiner's tool, not a daily screen. + ## Comments & notifications (later milestone) - `comments`: `page_id`, `author_id`, `body` (Markdown), `anchor` diff --git a/docs/architecture/operations.md b/docs/architecture/operations.md index 015b80c..b84a217 100644 --- a/docs/architecture/operations.md +++ b/docs/architecture/operations.md @@ -109,6 +109,7 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack. | conversion payload prune | daily | null finished conversion jobs' raw bytes (#233) | | mail outbox retention | daily | delete sent/failed outbox rows past period (#234) | | mail outbox retry | every minute | e-mail delivery with backoff | +| read-trail maintenance | daily | partition months ahead + prune `read_events` (#224) | Job outcomes are visible in the Site Admin UI (last run, status) — that panel is the operator's single glance for instance health. diff --git a/docs/vs-nfd/20-massnahmenplan.md b/docs/vs-nfd/20-massnahmenplan.md index ff4c835..f252195 100644 --- a/docs/vs-nfd/20-massnahmenplan.md +++ b/docs/vs-nfd/20-massnahmenplan.md @@ -136,7 +136,7 @@ Protokolliert werden Lesezugriffe **ausschließlich** auf Seiten mit Attachment-Download, Export, No-JS-Shell, Collab-WS-Join · 4 AT · #222 - [x] Dedup-Fenster (eine Sitzung + eine Seite innerhalb N Minuten = ein Ereignis), sonst erzeugt Yjs-Sync eine Ereignisflut · 2 AT · #223 -- [ ] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224 +- [x] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224 - [ ] Abschaltbar, Zweckbindung dokumentiert · 1–2 AT · #225 Vorteil über den Aufwand hinaus: Die Zweckbindung ist sauber begründbar diff --git a/docs/vs-nfd/50-haertungsleitfaden.md b/docs/vs-nfd/50-haertungsleitfaden.md index 3491c33..6ee05a7 100644 --- a/docs/vs-nfd/50-haertungsleitfaden.md +++ b/docs/vs-nfd/50-haertungsleitfaden.md @@ -40,6 +40,7 @@ Settings-Cache ist in-process (operations.md). | `backup.nextcloud.enabled` | `false` | `false` | „Backup nur lokal": kein Anwendungs-Upload von Restore-Sets zu Drittdiensten. Fernspiegel regelt ausschließlich die Deploy-Allowlist (1.2). | | `trash.retentionDays`, `audit.retentionDays`, `conversion.payloadRetentionDays`, `mail.outboxRetentionDays` | Defaults (30/365/30/30) | ebd. | Aufbewahrung bewusst begrenzt; Verkürzung nach Betreiber-Löschkonzept zulässig (Betriebshandbuch §5). | | `readTrail.dedupWindowMinutes` | Default (5) | `5` | Dedup-Fenster des Lesetrails (#223): je (Sitzung, Seite, Kanal) ein Ereignis pro Fenster — begrenzt die Ereignisflut einer Live-Sitzung auf ~12/h. Kleiner = feineres Protokoll und mehr Zeilen; Änderung mit dem Zweckbindungs-Dokument (#225) abstimmen. | +| `readTrail.retentionDays` | Default (365) | `365` | eigene Aufbewahrung des Lesetrails (#224), bewusst getrennt von `audit.retentionDays`; Löschläufe sind selbst auditiert (`read_trail.pruned`). Dauer mit der Zweckbindung (#225) und dem Betreiber-Löschkonzept abstimmen. | | `legal.imprint`, `legal.privacyPolicy` | befüllt | leer | Betreiberpflicht; leere Seiten zeigen einen Warnbanner. | ### 1.2 Deploy-Konfiguration (`.env` / Compose — nur Plattformzugriff, bewusst nicht per Admin-UI) diff --git a/packages/shared/src/system.ts b/packages/shared/src/system.ts index 182f085..ba10e07 100644 --- a/packages/shared/src/system.ts +++ b/packages/shared/src/system.ts @@ -164,6 +164,49 @@ export interface AuditListView { total: number; } +/** + * Site-Admin query path over the read-access trail (issue #224, ADR 0023) — + * evidence nobody can read is not evidence. Answers the two expected + * questions: "who read page X" (pageId) and "what did user Y read" (actor), + * both within a period. + */ +export const READ_EVENT_PAGE_SIZE = 50; + +export const readEventListQuerySchema = z.object({ + /** Page id the events belong to. */ + pageId: z.string().trim().min(1).optional(), + /** Exact username of the reading user. */ + actor: z.string().trim().min(1).optional(), + channel: z.string().trim().min(1).optional(), + from: z.coerce.date().optional(), + to: z.coerce.date().optional(), + page: z.coerce.number().int().min(1).default(1), +}); + +export type ReadEventListQuery = z.infer; + +export interface ReadEventView { + id: string; + occurredAt: string; + actor: AuditActorView | null; + /** Null for pond-level attachments (no page); the details name the file. */ + pageId: string | null; + pondId: string; + channel: string; + classification: string; + /** Seconds the event's dedup window spans (#223) — the row represents up + * to this much access time, not a single request. */ + windowSeconds: number; + details: Record | null; +} + +export interface ReadEventListView { + entries: ReadEventView[]; + page: number; + pageCount: number; + total: number; +} + export interface StoragePondView { pondId: string; name: string;