From fd4fd60c991b2e744d1e1beb2d8140548ad52a05 Mon Sep 17 00:00:00 2001 From: Claude Fable 5 Date: Fri, 31 Jul 2026 12:13:21 +0200 Subject: [PATCH 1/3] #223: dedup window for the read trail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One event per (session, page, channel) within an aligned window of readTrail.dedupWindowMinutes (default 5): buckets are floor(epoch / windowSeconds), and a unique (dedup_key, window_bucket) pair collapses concurrent duplicates race-free at insert time — the first access in a window is always recorded, a later duplicate lands on the unique violation and is skipped quietly (a skipped duplicate is not a gap; only real write failures still abort the read). Each row carries windowSeconds, so the evidence states it represents a window, never a request count. Reconnects within a window stay one event; a new session records again even for the same user; channels never collapse into each other; the page-less attachment key uses the documented `-` placeholder. Load evidence: 30 collab-token renewals inside one window produce exactly one event (test), bounding a live editing session at ~12 events/hour/page. Window semantics documented in ADR 0023, the VS-NfD security documentation (#228) and as a hardening-guide line for the new setting (care rule: same PR). Refs #223. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AUtYMxwTCMHG9mVHnwbFg8 --- .../migration.sql | 32 ++++ apps/api/prisma/schema.prisma | 9 + .../read-trail-dedup.e2e.db.test.ts | 177 ++++++++++++++++++ apps/api/src/read-trail/read-trail.module.ts | 4 + apps/api/src/read-trail/read-trail.service.ts | 51 +++-- .../src/settings/instance-settings.service.ts | 5 + .../adr/0023-read-access-audit-trail.md | 22 +++ docs/vs-nfd/20-massnahmenplan.md | 2 +- docs/vs-nfd/50-haertungsleitfaden.md | 1 + docs/vs-nfd/60-sicherheitsdokumentation.md | 9 +- 10 files changed, 298 insertions(+), 14 deletions(-) create mode 100644 apps/api/prisma/migrations/20260731160000_read_events_dedup/migration.sql create mode 100644 apps/api/src/read-trail/read-trail-dedup.e2e.db.test.ts diff --git a/apps/api/prisma/migrations/20260731160000_read_events_dedup/migration.sql b/apps/api/prisma/migrations/20260731160000_read_events_dedup/migration.sql new file mode 100644 index 0000000..732e770 --- /dev/null +++ b/apps/api/prisma/migrations/20260731160000_read_events_dedup/migration.sql @@ -0,0 +1,32 @@ +-- #223 (ADR 0023): dedup window for the read trail. Aligned buckets +-- (floor(epoch / window)) with a unique (dedup_key, window_bucket) pair make +-- concurrent duplicates collapse race-free at insert time. +ALTER TABLE "read_events" + ADD COLUMN "dedup_key" TEXT, + ADD COLUMN "window_bucket" BIGINT, + ADD COLUMN "window_seconds" INTEGER; + +-- Backfill rows written between the #222 and #223 deploys under the default +-- 5-minute window, then apply the window's own semantics retroactively: +-- within one (key, bucket) pair only the FIRST event is the evidence row — +-- exactly what the window would have recorded had it existed. +UPDATE "read_events" + SET "dedup_key" = "session_key" || ':' || COALESCE("page_id", '-') || ':' || "channel", + "window_bucket" = FLOOR(EXTRACT(EPOCH FROM "occurred_at") / 300)::BIGINT, + "window_seconds" = 300 + WHERE "dedup_key" IS NULL; + +DELETE FROM "read_events" keep + USING "read_events" first + WHERE keep."dedup_key" = first."dedup_key" + AND keep."window_bucket" = first."window_bucket" + AND (first."occurred_at" < keep."occurred_at" + OR (first."occurred_at" = keep."occurred_at" AND first."id" < keep."id")); + +ALTER TABLE "read_events" + ALTER COLUMN "dedup_key" SET NOT NULL, + ALTER COLUMN "window_bucket" SET NOT NULL, + ALTER COLUMN "window_seconds" SET NOT NULL; + +CREATE UNIQUE INDEX "read_events_dedup_key_window_bucket_key" + ON "read_events"("dedup_key", "window_bucket"); diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 2ad741a..228ce7b 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -115,7 +115,16 @@ model ReadEvent { /// rewrite history (ADR 0023). classification String details Json? + /// Dedup window (issue #223): `::` plus the + /// aligned bucket `floor(epoch / windowSeconds)`. The unique pair makes + /// concurrent duplicate reads collapse race-free (insert or P2002-skip). + dedupKey String @map("dedup_key") + windowBucket BigInt @map("window_bucket") + /// Window length the event was recorded under — the row itself states it + /// represents up to this many seconds, so the evidence is not overread. + windowSeconds Int @map("window_seconds") + @@unique([dedupKey, windowBucket]) @@index([pageId, occurredAt]) @@index([actorId, occurredAt]) @@index([occurredAt]) diff --git a/apps/api/src/read-trail/read-trail-dedup.e2e.db.test.ts b/apps/api/src/read-trail/read-trail-dedup.e2e.db.test.ts new file mode 100644 index 0000000..8b05f79 --- /dev/null +++ b/apps/api/src/read-trail/read-trail-dedup.e2e.db.test.ts @@ -0,0 +1,177 @@ +import { INestApplication } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; +import request from 'supertest'; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; + +import { ClockService } from '../common/clock.service'; +import { createTestApp, sessionCookieOf } from '../testing/test-app'; +import { createTestPrisma, grantOwnerAdmin, hasTestDb, uniqueSuffix } from '../testing/test-db'; +import { UsersService } from '../users/users.service'; + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +/** + * The read-trail dedup window (issue #223, ADR 0023): one event per + * (session, page, channel) within an aligned `readTrail.dedupWindowMinutes` + * window — repeats and reconnect-style re-requests collapse, a new session + * or a new window does not, and the recorded row states the window length + * it represents. + */ +describe.skipIf(!hasTestDb)('read-trail dedup window (e2e, issue #223)', () => { + let app: INestApplication; + let prisma: PrismaClient; + const suffix = uniqueSuffix(); + const password = 'dedup fenster zeugen 123'; + + let ownerId: string; + let ownerCookie: string; + let pondId: string; + let classifiedId: string; + + const api = () => request(app.getHttpServer()); + const login = async () => + sessionCookieOf( + await api() + .post('/api/v1/auth/login') + .send({ usernameOrEmail: `dedup-owner-${suffix}`, password }) + .expect(200), + ); + const events = () => + prisma.readEvent.findMany({ where: { pondId }, orderBy: { occurredAt: 'asc' } }); + + beforeAll(async () => { + prisma = createTestPrisma(); + await prisma.rateLimit.deleteMany({}); + app = await createTestApp(); + const users = app.get(UsersService); + + const owner = await users.createUser({ + username: `dedup-owner-${suffix}`, + email: `dedup-owner-${suffix}@example.test`, + displayName: 'Dedup Owner', + password, + locale: 'en', + }); + await users.markEmailVerified(owner.id); + ownerId = owner.id; + ownerCookie = await login(); + + const pond = await prisma.pond.create({ + data: { slug: `dedup-pond-${suffix}`, name: 'Dedup Pond', type: 'SHARED', ownerId }, + }); + pondId = pond.id; + await grantOwnerAdmin(prisma, pondId, ownerId); + + const page = await prisma.page.create({ + data: { + pondId, + slug: `classified-${suffix}`, + title: 'Classified', + classification: 'VS_NFD', + createdBy: ownerId, + sortKey: 'a0', + ydocState: new Uint8Array(), + contentCache: { + create: { plainText: 'x', markdown: 'x', html: '

x

', outline: [] }, + }, + }, + }); + classifiedId = page.id; + }); + + afterEach(async () => { + await prisma.readEvent.deleteMany({ where: { pondId } }); + await prisma.rateLimit.deleteMany({}); + }); + + afterAll(async () => { + await prisma.readEvent.deleteMany({ where: { pondId } }); + await prisma.attachment.deleteMany({ where: { pondId } }); + await prisma.roleGrant.deleteMany({ where: { pondId } }); + await prisma.pageContentCache.deleteMany({ where: { page: { pondId } } }); + await prisma.page.deleteMany({ where: { pondId } }); + await prisma.pond.deleteMany({ where: { id: pondId } }); + await prisma.user.deleteMany({ where: { id: ownerId } }); + await prisma.$disconnect(); + await app.close(); + }); + + it('collapses repeated reads of one page in one session+channel to a single event that names its window', async () => { + await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200); + await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200); + await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200); + const rows = await events(); + expect(rows).toHaveLength(1); + // The row itself states that it represents a window, not a request. + expect(rows[0]!.windowSeconds).toBe(5 * 60); + expect(rows[0]!.dedupKey).toBe(`${rows[0]!.sessionKey}:${classifiedId}:page_view`); + }); + + it('keeps channels apart: the same session reading and joining collab yields one event each', async () => { + await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200); + await api() + .get(`/api/v1/pages/${classifiedId}/collab-token`) + .set('Cookie', ownerCookie) + .expect(200); + const rows = await events(); + expect(rows.map((r) => r.channel).sort()).toEqual(['collab_join', 'page_view']); + }); + + it('a new session records again even for the same user — a reconnect within the session does not', async () => { + await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200); + // Same session, later request ("reconnect"): deduped. + await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200); + expect(await events()).toHaveLength(1); + + const secondCookie = await login(); + await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', secondCookie).expect(200); + const rows = await events(); + expect(rows).toHaveLength(2); + expect(new Set(rows.map((r) => r.sessionKey)).size).toBe(2); + expect(rows.every((r) => r.actorId === ownerId)).toBe(true); + }); + + it('bounds a live editing session: 30 collab-token renewals in one window are one event', async () => { + for (let i = 0; i < 30; i += 1) { + await api() + .get(`/api/v1/pages/${classifiedId}/collab-token`) + .set('Cookie', ownerCookie) + .expect(200); + } + const rows = await events(); + expect(rows).toHaveLength(1); + expect(rows[0]!.channel).toBe('collab_join'); + }); + + it('opens a new window when the clock moves past the bucket boundary', async () => { + await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200); + const clock = app.get(ClockService); + const later = new Date(Date.now() + 10 * 60 * 1000); + const spy = vi.spyOn(clock, 'now').mockReturnValue(later); + try { + await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200); + } finally { + spy.mockRestore(); + } + expect(await events()).toHaveLength(2); + }); + + it('dedups page-less attachment downloads on the placeholder page key', async () => { + // A pond-level upload has no page; its effective classification falls + // back to the pond maximum (#212) and the dedup key carries `-`. + const uploaded = await api() + .post(`/api/v1/ponds/${pondId}/files`) + .set('Cookie', ownerCookie) + .attach('file', Buffer.concat([PNG_SIGNATURE, Buffer.from('img')]), 'a.png') + .expect(201); + await prisma.readEvent.deleteMany({ where: { pondId } }); + + await api().get(`/api/v1/media/${uploaded.body.id}`).set('Cookie', ownerCookie).expect(200); + await api().get(`/api/v1/media/${uploaded.body.id}`).set('Cookie', ownerCookie).expect(200); + const rows = await events(); + expect(rows).toHaveLength(1); + expect(rows[0]!.channel).toBe('attachment'); + expect(rows[0]!.pageId).toBeNull(); + expect(rows[0]!.dedupKey).toBe(`${rows[0]!.sessionKey}:-:attachment`); + }); +}); diff --git a/apps/api/src/read-trail/read-trail.module.ts b/apps/api/src/read-trail/read-trail.module.ts index 85520d3..61a12aa 100644 --- a/apps/api/src/read-trail/read-trail.module.ts +++ b/apps/api/src/read-trail/read-trail.module.ts @@ -1,5 +1,8 @@ import { Global, Module } from '@nestjs/common'; +import { CommonModule } from '../common/common.module'; +import { SettingsModule } from '../settings/settings.module'; + import { ReadTrailService } from './read-trail.service'; /** @@ -9,6 +12,7 @@ import { ReadTrailService } from './read-trail.service'; */ @Global() @Module({ + imports: [CommonModule, SettingsModule], providers: [ReadTrailService], exports: [ReadTrailService], }) diff --git a/apps/api/src/read-trail/read-trail.service.ts b/apps/api/src/read-trail/read-trail.service.ts index 1189145..de0845f 100644 --- a/apps/api/src/read-trail/read-trail.service.ts +++ b/apps/api/src/read-trail/read-trail.service.ts @@ -2,7 +2,9 @@ import { Injectable } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { PinoLogger } from 'nestjs-pino'; +import { ClockService } from '../common/clock.service'; import { PrismaService } from '../prisma/prisma.service'; +import { InstanceSettingsService } from '../settings/instance-settings.service'; /** * Every read surface classified content can leave through (issue #222, @@ -61,11 +63,22 @@ export interface ReadEventInput extends ReadActor { * swallowed. A lost event is a gap in evidence, so a failed write aborts * the read with the ordinary 500 — the reader retries, the evidence stays * complete (decision recorded in ADR 0023 and security.md). + * + * Dedup window (issue #223): one event per (session, page, channel) within + * an aligned window of `readTrail.dedupWindowMinutes` (default 5). Buckets + * are `floor(epoch / windowSeconds)`, and the unique + * (dedupKey, windowBucket) pair collapses concurrent duplicates race-free: + * the first insert wins, every later one lands on P2002 and is skipped — + * a skipped DUPLICATE is not a gap, so it must not abort the read. The row + * carries `windowSeconds`, so each event states that it represents up to + * that many seconds of access, not a single request. */ @Injectable() export class ReadTrailService { constructor( private readonly prisma: PrismaService, + private readonly settings: InstanceSettingsService, + private readonly clock: ClockService, private readonly logger: PinoLogger, ) { this.logger.setContext(ReadTrailService.name); @@ -73,21 +86,35 @@ export class ReadTrailService { async record(event: ReadEventInput): Promise { const { actorId, sessionKey, pageId, pondId, channel, details } = event; - await this.prisma.readEvent.create({ - data: { - actorId, - sessionKey, - pageId, - pondId, - channel, - classification: 'vs_nfd', - details: details ? (details as Prisma.InputJsonObject) : undefined, - }, - }); + const windowSeconds = (await this.settings.get('readTrail.dedupWindowMinutes')) * 60; + const dedupKey = `${sessionKey}:${pageId ?? '-'}:${channel}`; + const windowBucket = BigInt(Math.floor(this.clock.now().getTime() / 1000 / windowSeconds)); + try { + await this.prisma.readEvent.create({ + data: { + actorId, + sessionKey, + pageId, + pondId, + channel, + classification: 'vs_nfd', + details: details ? (details as Prisma.InputJsonObject) : undefined, + dedupKey, + windowBucket, + windowSeconds, + }, + }); + } catch (error) { + const isDuplicate = + error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002'; + if (!isDuplicate) throw error; + this.logger.debug({ dedupKey, channel }, 'read_trail: deduped within window'); + return; + } // The stdout line mirrors the row (SIEM forwarding beyond this is out of // scope, #224); it fires only after the row is safely persisted. this.logger.info( - { actor: actorId, sessionKey, pageId, pondId, channel }, + { actor: actorId, sessionKey, pageId, pondId, channel, windowSeconds }, 'read_trail: classified page read', ); } diff --git a/apps/api/src/settings/instance-settings.service.ts b/apps/api/src/settings/instance-settings.service.ts index d9aa7f7..30b7be0 100644 --- a/apps/api/src/settings/instance-settings.service.ts +++ b/apps/api/src/settings/instance-settings.service.ts @@ -51,6 +51,11 @@ export const INSTANCE_SETTINGS = { // bounded. PENDING rows — including failed-but-retryable ones — are // never touched; the retry loop owns them. 'mail.outboxRetentionDays': z.number().int().min(1).default(30), + // Read-trail dedup window (issue #223, ADR 0023): one event per + // (session, page, channel) within an aligned window of this many minutes. + // 5 minutes keeps a live Yjs session (collab tokens every 60 s) at a + // bounded ~12 events/hour/page while still evidencing distinct visits. + 'readTrail.dedupWindowMinutes': z.number().int().min(1).default(5), // Default VS-NfD classification for newly created pages (ADR 0022, // issue #204). An instance operated inside a classified environment sets // this to `vs_nfd` so nothing starts unmarked; inheritance from the diff --git a/docs/architecture/adr/0023-read-access-audit-trail.md b/docs/architecture/adr/0023-read-access-audit-trail.md index 38c0147..a5681d7 100644 --- a/docs/architecture/adr/0023-read-access-audit-trail.md +++ b/docs/architecture/adr/0023-read-access-audit-trail.md @@ -67,6 +67,28 @@ traffic per open document. also the MCP `read_page` path), `job:` (background builds such as the account data export), `anon` (anonymous reader on a public grant). +## Decisions taken in #223 + +- **Aligned dedup windows.** One event per (session, page, channel) within + an aligned window of `readTrail.dedupWindowMinutes` (default 5): + buckets are `floor(epoch / windowSeconds)`, and a unique + (dedupKey, windowBucket) pair collapses concurrent duplicates race-free + at insert time. Sliding windows (measured from the first event) were + rejected — they need a read-before-write and lose the race-safety of the + plain unique insert. Consequence: two reads just across a bucket + boundary yield two events; over-recording is acceptable, gaps are not. +- **A skipped duplicate is not a gap** — the unique-violation path returns + quietly (debug log), only real write failures abort the read. +- **Each row states its window** (`windowSeconds`), so the evidence reads + as "accessed at least once in these N minutes", never as a request + count. What the trail can prove about a live session: the per-minute + collab-token renewals collapse to one `collab_join` event per window — + presence during the window, not activity within it. +- **Anonymous readers share one `anon` key**: all anonymous reads of a page + through one channel inside a window are one event. Deliberate — telling + anonymous browsers apart would require fingerprinting (IP/UA), which the + purpose limitation (#225) rules out. + ## Consequences - The scope limit is the feature's strongest argument in the works-council diff --git a/docs/vs-nfd/20-massnahmenplan.md b/docs/vs-nfd/20-massnahmenplan.md index 9d01deb..ff4c835 100644 --- a/docs/vs-nfd/20-massnahmenplan.md +++ b/docs/vs-nfd/20-massnahmenplan.md @@ -134,7 +134,7 @@ Protokolliert werden Lesezugriffe **ausschließlich** auf Seiten mit - [x] Instrumentierung der Lesepfade: Seitenansicht, Public-API-GET, Attachment-Download, Export, No-JS-Shell, Collab-WS-Join · 4 AT · #222 -- [ ] Dedup-Fenster (eine Sitzung + eine Seite innerhalb N Minuten = ein +- [x] Dedup-Fenster (eine Sitzung + eine Seite innerhalb N Minuten = ein Ereignis), sonst erzeugt Yjs-Sync eine Ereignisflut · 2 AT · #223 - [ ] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224 - [ ] Abschaltbar, Zweckbindung dokumentiert · 1–2 AT · #225 diff --git a/docs/vs-nfd/50-haertungsleitfaden.md b/docs/vs-nfd/50-haertungsleitfaden.md index 6323c22..3491c33 100644 --- a/docs/vs-nfd/50-haertungsleitfaden.md +++ b/docs/vs-nfd/50-haertungsleitfaden.md @@ -39,6 +39,7 @@ Settings-Cache ist in-process (operations.md). | `upload.allowedExtensions` | nur das dienstlich Nötige (z. B. `pdf`) | Standardliste | jede zusätzliche Endung vergrößert die Menge nicht prüfbarer Binärformate im Bestand. Bilder sind davon unabhängig immer erlaubt (Magic-Byte-geprüft). | | `backup.nextcloud.enabled` | `false` | `false` | „Backup nur lokal": kein Anwendungs-Upload von Restore-Sets zu Drittdiensten. Fernspiegel regelt ausschließlich die Deploy-Allowlist (1.2). | | `trash.retentionDays`, `audit.retentionDays`, `conversion.payloadRetentionDays`, `mail.outboxRetentionDays` | Defaults (30/365/30/30) | ebd. | Aufbewahrung bewusst begrenzt; Verkürzung nach Betreiber-Löschkonzept zulässig (Betriebshandbuch §5). | +| `readTrail.dedupWindowMinutes` | Default (5) | `5` | Dedup-Fenster des Lesetrails (#223): je (Sitzung, Seite, Kanal) ein Ereignis pro Fenster — begrenzt die Ereignisflut einer Live-Sitzung auf ~12/h. Kleiner = feineres Protokoll und mehr Zeilen; Änderung mit dem Zweckbindungs-Dokument (#225) abstimmen. | | `legal.imprint`, `legal.privacyPolicy` | befüllt | leer | Betreiberpflicht; leere Seiten zeigen einen Warnbanner. | ### 1.2 Deploy-Konfiguration (`.env` / Compose — nur Plattformzugriff, bewusst nicht per Admin-UI) diff --git a/docs/vs-nfd/60-sicherheitsdokumentation.md b/docs/vs-nfd/60-sicherheitsdokumentation.md index 0cc7231..bb686f5 100644 --- a/docs/vs-nfd/60-sicherheitsdokumentation.md +++ b/docs/vs-nfd/60-sicherheitsdokumentation.md @@ -181,7 +181,14 @@ Datenträger- und Transportschutz ist Plattformsache ### 3.5 Lesekanäle Jeder Kanal, über den Seiteninhalt die Anwendung verlässt (zugleich die -Instrumentierungsliste für den Lesetrail, #222): +Instrumentierungsliste für den Lesetrail, #222). Der Lesetrail +protokolliert je (Sitzung, Seite, Kanal) **ein Ereignis pro +Dedup-Fenster** (`readTrail.dedupWindowMinutes`, Default 5 Minuten, +#223); jede Zeile trägt die Fensterlänge. Beweiswert damit: „mindestens +ein Zugriff in diesem Fenster" — **nicht** eine Zugriffszählung, und +beim Kanal Collab „Live-Verbindung bestand während des Fensters", nicht +einzelne Sync-Frames. Anonyme Leser teilen sich den Marker `anon` +(keine Fingerprinting-Unterscheidung, Zweckbindung #225): | Kanal | Pfad | Rechteprüfung | Kennzeichnung (M26) | | -------------------------- | -------------------------------- | -------------------------------------- | ----------------------------------- | -- 2.45.2 From 2bdb0ec2cf78f5d425f0eef1e7347935772a7ae4 Mon Sep 17 00:00:00 2001 From: Claude Fable 5 Date: Fri, 31 Jul 2026 12:21:45 +0200 Subject: [PATCH 2/3] =?UTF-8?q?#224:=20read-trail=20storage=20=E2=80=94=20?= =?UTF-8?q?partitioning,=20retention,=20admin=20query=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert read_events to monthly RANGE partitions on occurred_at, with a DEFAULT partition as safety net: a lagging maintenance job must never turn the trail's hard-failure semantics into an outage for classified reads. The dedup unique pair (#223) moves to per-partition indexes (PostgreSQL cannot carry it on the parent); a bucket spanning a month boundary may record one duplicate — over-recording is acceptable, gaps are not. New daily job read-trail-maintenance (job-count fence 9 -> 10) creates months ahead — each with its dedup index — and applies the trail's own retention readTrail.retentionDays (default 365, deliberately independent of audit.retentionDays): whole expired months are DROPped without scanning, remainders deleted by range, every run audited as read_trail.pruned (catalogue v1.2; the fence regex now admits an underscore namespace). Site-Admin query path GET /admin/system/read-events answers "who read page X" and "what did user Y read" within a period — API-only by design, documented. Growth measured and documented in data-model.md: ~1 MB per 1000 events including indexes. Tests: retention pruning + audited deletion + admin queries on the shared database; the partitioned shape, per-partition P2002 dedup, months-ahead creation and DROP-based pruning against a fresh database built by the real migration chain. Refs #224. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AUtYMxwTCMHG9mVHnwbFg8 --- .../migration.sql | 76 +++++ apps/api/prisma/schema.prisma | 8 +- apps/api/src/admin/system-admin.controller.ts | 12 + apps/api/src/admin/system-admin.service.ts | 63 ++++ apps/api/src/audit/audit-actions.ts | 1 + apps/api/src/audit/audit-catalogue.test.ts | 5 +- .../read-trail-maintenance.service.ts | 139 ++++++++ .../read-trail-storage.e2e.db.test.ts | 308 ++++++++++++++++++ apps/api/src/read-trail/read-trail.module.ts | 31 +- .../src/settings/instance-settings.service.ts | 7 + apps/web/e2e/system.spec.ts | 5 +- .../adr/0023-read-access-audit-trail.md | 17 + docs/architecture/audit-events.md | 15 +- docs/architecture/data-model.md | 34 ++ docs/architecture/operations.md | 1 + docs/vs-nfd/20-massnahmenplan.md | 2 +- docs/vs-nfd/50-haertungsleitfaden.md | 1 + packages/shared/src/system.ts | 43 +++ 18 files changed, 750 insertions(+), 18 deletions(-) create mode 100644 apps/api/prisma/migrations/20260731170000_read_events_partitioned/migration.sql create mode 100644 apps/api/src/read-trail/read-trail-maintenance.service.ts create mode 100644 apps/api/src/read-trail/read-trail-storage.e2e.db.test.ts diff --git a/apps/api/prisma/migrations/20260731170000_read_events_partitioned/migration.sql b/apps/api/prisma/migrations/20260731170000_read_events_partitioned/migration.sql new file mode 100644 index 0000000..1cc68ab --- /dev/null +++ b/apps/api/prisma/migrations/20260731170000_read_events_partitioned/migration.sql @@ -0,0 +1,76 @@ +-- #224 (ADR 0023): convert read_events to monthly RANGE partitions on +-- occurred_at. Volume grows unbounded with use; retention then DROPs whole +-- expired partitions instead of scanning deletes. The primary key gains the +-- partition column (PostgreSQL requirement); the dedup unique pair +-- (dedup_key, window_bucket) moves to PER-PARTITION unique indexes — a +-- partitioned parent cannot carry it without the partition key. A bucket +-- spanning a month boundary can therefore record one duplicate; documented +-- in ADR 0023, over-recording is acceptable, gaps are not. +-- +-- A DEFAULT partition catches rows outside every maintained range, so a +-- lagging maintenance job can never make classified reads fail (the trail's +-- hard-failure semantics would otherwise turn an ops miss into an outage). + +ALTER TABLE "read_events" RENAME TO "read_events_old"; +ALTER INDEX "read_events_pkey" RENAME TO "read_events_old_pkey"; +ALTER INDEX "read_events_dedup_key_window_bucket_key" RENAME TO "read_events_old_dedup_key"; +ALTER INDEX "read_events_page_id_occurred_at_idx" RENAME TO "read_events_old_page_idx"; +ALTER INDEX "read_events_actor_id_occurred_at_idx" RENAME TO "read_events_old_actor_idx"; +ALTER INDEX "read_events_occurred_at_idx" RENAME TO "read_events_old_at_idx"; + +CREATE TABLE "read_events" ( + "id" TEXT NOT NULL, + "occurred_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "actor_id" TEXT, + "session_key" TEXT NOT NULL, + "page_id" TEXT, + "pond_id" TEXT NOT NULL, + "channel" TEXT NOT NULL, + "classification" TEXT NOT NULL, + "details" JSONB, + "dedup_key" TEXT NOT NULL, + "window_bucket" BIGINT NOT NULL, + "window_seconds" INTEGER NOT NULL, + + CONSTRAINT "read_events_pkey" PRIMARY KEY ("id", "occurred_at") +) PARTITION BY RANGE ("occurred_at"); + +-- Non-unique parent indexes propagate to every partition automatically. +CREATE INDEX "read_events_page_id_occurred_at_idx" ON "read_events"("page_id", "occurred_at"); +CREATE INDEX "read_events_actor_id_occurred_at_idx" ON "read_events"("actor_id", "occurred_at"); +CREATE INDEX "read_events_occurred_at_idx" ON "read_events"("occurred_at"); + +-- The safety-net partition, plus the current and the next month — the daily +-- maintenance job (read-trail-maintenance) keeps creating months ahead and +-- adds the same per-partition dedup index to each new one. +CREATE TABLE "read_events_default" PARTITION OF "read_events" DEFAULT; +CREATE UNIQUE INDEX "read_events_default_dedup_key" + ON "read_events_default"("dedup_key", "window_bucket"); + +DO $$ +DECLARE + m DATE; + part TEXT; +BEGIN + FOR i IN 0..1 LOOP + m := date_trunc('month', now())::date + (i || ' month')::interval; + part := 'read_events_y' || to_char(m, 'YYYY') || 'm' || to_char(m, 'MM'); + EXECUTE format( + 'CREATE TABLE %I PARTITION OF "read_events" FOR VALUES FROM (%L) TO (%L)', + part, m, m + interval '1 month'); + EXECUTE format( + 'CREATE UNIQUE INDEX %I ON %I ("dedup_key", "window_bucket")', + part || '_dedup_key', part); + END LOOP; +END $$; + +INSERT INTO "read_events" + ("id", "occurred_at", "actor_id", "session_key", "page_id", "pond_id", + "channel", "classification", "details", "dedup_key", "window_bucket", + "window_seconds") +SELECT "id", "occurred_at", "actor_id", "session_key", "page_id", "pond_id", + "channel", "classification", "details", "dedup_key", "window_bucket", + "window_seconds" + FROM "read_events_old"; + +DROP TABLE "read_events_old"; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 228ce7b..61b07a0 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -96,8 +96,13 @@ model AuditEntry { /// volume, purpose and legal basis all differ. Deliberately WITHOUT foreign /// keys: evidence must survive a page purge and a hard user deletion — the /// ids stay as recorded (pseudonymous uuids), history is never rewritten. +/// +/// In migrated databases the table is RANGE-partitioned by `occurred_at` +/// (monthly, issue #224) — hence the composite id. The dedup unique pair +/// lives per partition there (a partitioned parent cannot carry it without +/// the partition key); `db push` test databases get it on the plain table. model ReadEvent { - id String @id @default(uuid()) + id String @default(uuid()) occurredAt DateTime @default(now()) @map("occurred_at") /// Null = anonymous reader (public grant); `sessionKey` still names the /// browsing session, so the anonymous marker is explicit, not an accident. @@ -124,6 +129,7 @@ model ReadEvent { /// represents up to this many seconds, so the evidence is not overread. windowSeconds Int @map("window_seconds") + @@id([id, occurredAt]) @@unique([dedupKey, windowBucket]) @@index([pageId, occurredAt]) @@index([actorId, occurredAt]) diff --git a/apps/api/src/admin/system-admin.controller.ts b/apps/api/src/admin/system-admin.controller.ts index 09e40ce..362d92a 100644 --- a/apps/api/src/admin/system-admin.controller.ts +++ b/apps/api/src/admin/system-admin.controller.ts @@ -1,9 +1,12 @@ import { Controller, Get, Param, Post, Query, Req, UseGuards } from '@nestjs/common'; import { auditListQuerySchema, + readEventListQuerySchema, type AuditListQuery, type AuditListView, type JobTriggerResult, + type ReadEventListQuery, + type ReadEventListView, type StorageOverviewView, type SystemBackupView, type SystemJobView, @@ -45,6 +48,15 @@ export class SystemAdminController { return this.system.auditLog(query); } + /** Read-access trail queries (issue #224): "who read page X", "what did + * user Y read" — Site-Admin only, like the audit viewer above. */ + @Get('read-events') + async readEvents( + @Query(new ZodValidationPipe(readEventListQuerySchema)) query: ReadEventListQuery, + ): Promise { + return this.system.readEvents(query); + } + @Get('storage') async storage(): Promise { return this.system.storage(); diff --git a/apps/api/src/admin/system-admin.service.ts b/apps/api/src/admin/system-admin.service.ts index 95b55b8..6ff46f1 100644 --- a/apps/api/src/admin/system-admin.service.ts +++ b/apps/api/src/admin/system-admin.service.ts @@ -7,10 +7,13 @@ import { AUDIT_PAGE_SIZE, BACKUP_FRESH_MAX_AGE_HOURS, BACKUP_STATUS_FILE, + READ_EVENT_PAGE_SIZE, type AuditListQuery, type AuditListView, type BackupStatus, type JobTriggerResult, + type ReadEventListQuery, + type ReadEventListView, type StorageOverviewView, type SystemBackupView, type SystemJobView, @@ -160,6 +163,66 @@ export class SystemAdminService { }; } + /** + * The Site-Admin query path over the read-access trail (issue #224, + * ADR 0023) — evidence nobody can read is not evidence. Answers "who read + * page X" and "what did user Y read" within a period. API-only by design + * (no panel yet): the trail is an examiner's tool, not a daily screen — + * documented in data-model.md §read_events. + */ + async readEvents(query: ReadEventListQuery): Promise { + const where: Prisma.ReadEventWhereInput = {}; + if (query.pageId) where.pageId = query.pageId; + if (query.actor) { + const actor = await this.prisma.user.findUnique({ where: { username: query.actor } }); + // An unknown username matches nothing rather than everything. + where.actorId = actor?.id ?? '00000000-0000-0000-0000-000000000000'; + } + if (query.channel) where.channel = query.channel; + if (query.from || query.to) { + where.occurredAt = { + ...(query.from ? { gte: query.from } : {}), + ...(query.to ? { lte: query.to } : {}), + }; + } + + const total = await this.prisma.readEvent.count({ where }); + const pageCount = Math.max(1, Math.ceil(total / READ_EVENT_PAGE_SIZE)); + const page = Math.min(query.page, pageCount); + const events = await this.prisma.readEvent.findMany({ + where, + orderBy: { occurredAt: 'desc' }, + skip: (page - 1) * READ_EVENT_PAGE_SIZE, + take: READ_EVENT_PAGE_SIZE, + }); + // No FK on actor_id (evidence outlives accounts) — resolve what still + // exists in one query, show the bare id otherwise. + const actorIds = [...new Set(events.map((e) => e.actorId).filter((id): id is string => !!id))]; + const actors = actorIds.length + ? await this.prisma.user.findMany({ + where: { id: { in: actorIds } }, + select: { id: true, username: true, displayName: true }, + }) + : []; + const actorById = new Map(actors.map((a) => [a.id, a])); + return { + entries: events.map((event) => ({ + id: event.id, + occurredAt: event.occurredAt.toISOString(), + actor: event.actorId ? (actorById.get(event.actorId) ?? null) : null, + pageId: event.pageId, + pondId: event.pondId, + channel: event.channel, + classification: event.classification, + windowSeconds: event.windowSeconds, + details: (event.details as Record | null) ?? null, + })), + page, + pageCount, + total, + }; + } + async storage(): Promise { const usages = await this.prisma.pondUsage.findMany({ where: { pond: { deletedAt: null } }, diff --git a/apps/api/src/audit/audit-actions.ts b/apps/api/src/audit/audit-actions.ts index 49e80da..5156293 100644 --- a/apps/api/src/audit/audit-actions.ts +++ b/apps/api/src/audit/audit-actions.ts @@ -40,6 +40,7 @@ export const AUDIT_EVENTS = { 'pond.purged': { severity: 'notice' }, 'quota.override_cleared': { severity: 'notice' }, 'quota.override_set': { severity: 'notice' }, + 'read_trail.pruned': { severity: 'info' }, 'settings.changed': { severity: 'notice' }, 'setup.admin_created': { severity: 'notice' }, 'setup.completed': { severity: 'info' }, diff --git a/apps/api/src/audit/audit-catalogue.test.ts b/apps/api/src/audit/audit-catalogue.test.ts index 04ef16e..61d23cf 100644 --- a/apps/api/src/audit/audit-catalogue.test.ts +++ b/apps/api/src/audit/audit-catalogue.test.ts @@ -18,11 +18,12 @@ import { AUDIT_EVENTS } from './audit-actions'; const doc = readFileSync(join(__dirname, '../../../../docs/architecture/audit-events.md'), 'utf8'); /** Event rows are `| \`ns.event\` | trigger | severity | …` — the dot in the - * id keeps field-set rows (`msg`, `severity`, …) out of the match. */ + * id keeps field-set rows (`msg`, `severity`, …) out of the match. The + * namespace may carry an underscore since `read_trail.*` (issue #224). */ function documentedEvents(): Map { const events = new Map(); for (const line of doc.split('\n')) { - const id = /^\| `([a-z]+\.[a-z_]+)` +\|/.exec(line)?.[1]; + const id = /^\| `([a-z_]+\.[a-z_]+)` +\|/.exec(line)?.[1]; if (!id) continue; const cells = line.split('|').map((cell) => cell.trim()); // cells[0] is the empty string before the leading pipe. diff --git a/apps/api/src/read-trail/read-trail-maintenance.service.ts b/apps/api/src/read-trail/read-trail-maintenance.service.ts new file mode 100644 index 0000000..71b0f69 --- /dev/null +++ b/apps/api/src/read-trail/read-trail-maintenance.service.ts @@ -0,0 +1,139 @@ +import { Injectable } from '@nestjs/common'; +import { PinoLogger } from 'nestjs-pino'; + +import { AuditService } from '../audit/audit.service'; +import { ClockService } from '../common/clock.service'; +import { PrismaService } from '../prisma/prisma.service'; +import { InstanceSettingsService } from '../settings/instance-settings.service'; + +const MS_PER_DAY = 24 * 60 * 60 * 1000; + +/** How many months ahead of "now" a partition must exist. Two keeps a + * multi-week job outage from ever reaching an uncovered month (the DEFAULT + * partition would still catch it — reads never fail on a missing month). */ +const MONTHS_AHEAD = 2; + +/** + * Read-trail storage maintenance (issue #224, ADR 0023), one daily job with + * two duties: + * + * 1. **Partition upkeep** — `read_events` is RANGE-partitioned by month + * (migration `20260731170000`); this creates the next {@link MONTHS_AHEAD} + * monthly partitions, each with its per-partition dedup unique index + * (#223 — the partitioned parent cannot carry it). A `db push` database + * (tests) has a plain table; partition work skips itself there. + * 2. **Retention** — events older than `readTrail.retentionDays` are + * removed: whole months by dropping their partition (no scan), the + * remainder (default partition, plain tables) by a ranged delete. The + * deletion is audited (`read_trail.pruned`) so a gap in the evidence is + * always explainable — same principle as `audit.pruned` (#196), but a + * deliberately separate period. + */ +@Injectable() +export class ReadTrailMaintenanceService { + constructor( + private readonly prisma: PrismaService, + private readonly settings: InstanceSettingsService, + private readonly audit: AuditService, + private readonly clock: ClockService, + private readonly logger: PinoLogger, + ) { + this.logger.setContext(ReadTrailMaintenanceService.name); + } + + async run(): Promise { + await this.ensurePartitions(); + await this.pruneExpired(); + } + + /** True when read_events is a partitioned parent (relkind `p`). */ + private async isPartitioned(): Promise { + const rows = await this.prisma.$queryRaw<{ relkind: string }[]>` + SELECT relkind::text FROM pg_class + WHERE relname = 'read_events' AND relnamespace = 'public'::regnamespace`; + return rows[0]?.relkind === 'p'; + } + + /** `read_events_y2026m08` for 2026-08. */ + private partitionName(month: Date): string { + const y = month.getUTCFullYear(); + const m = String(month.getUTCMonth() + 1).padStart(2, '0'); + return `read_events_y${y}m${m}`; + } + + private monthStart(base: Date, offsetMonths: number): Date { + return new Date(Date.UTC(base.getUTCFullYear(), base.getUTCMonth() + offsetMonths, 1)); + } + + async ensurePartitions(): Promise { + if (!(await this.isPartitioned())) { + this.logger.debug('read_events is not partitioned here; skipping partition upkeep'); + return; + } + const now = this.clock.now(); + for (let offset = 0; offset <= MONTHS_AHEAD; offset += 1) { + const from = this.monthStart(now, offset); + const to = this.monthStart(now, offset + 1); + const name = this.partitionName(from); + try { + await this.prisma.$executeRawUnsafe( + `CREATE TABLE IF NOT EXISTS "${name}" PARTITION OF "read_events" + FOR VALUES FROM ('${from.toISOString()}') TO ('${to.toISOString()}')`, + ); + await this.prisma.$executeRawUnsafe( + `CREATE UNIQUE INDEX IF NOT EXISTS "${name}_dedup_key" + ON "${name}" ("dedup_key", "window_bucket")`, + ); + } catch (error) { + // Most likely: the DEFAULT partition already holds rows of this month + // (the job lagged past a month boundary). Nothing is lost — those + // rows live in the default partition and age out through the ranged + // delete below; the month just cannot get its own partition anymore. + this.logger.warn({ partition: name, err: error }, 'read-trail partition not created'); + } + } + } + + async pruneExpired(): Promise { + const retentionDays = await this.settings.get('readTrail.retentionDays'); + const cutoff = new Date(this.clock.now().getTime() - retentionDays * MS_PER_DAY); + let dropped = 0; + + if (await this.isPartitioned()) { + // Whole months strictly before the cutoff month go by DROP — no scan, + // and the dropped range is exact (every row in them is < cutoff). + const partitions = await this.prisma.$queryRaw<{ relname: string }[]>` + SELECT c.relname::text + FROM pg_inherits i + JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = 'read_events'::regclass + AND c.relname ~ '^read_events_y[0-9]{4}m[0-9]{2}$'`; + const cutoffMonth = this.monthStart(cutoff, 0); + for (const { relname } of partitions) { + const match = /^read_events_y(\d{4})m(\d{2})$/.exec(relname); + if (!match) continue; + const monthEnd = new Date(Date.UTC(Number(match[1]), Number(match[2]), 1)); + if (monthEnd.getTime() > cutoffMonth.getTime()) continue; + const counted = await this.prisma.$queryRawUnsafe<{ count: bigint }[]>( + `SELECT count(*)::bigint AS count FROM "${relname}"`, + ); + dropped += Number(counted[0]?.count ?? 0n); + await this.prisma.$executeRawUnsafe(`DROP TABLE "${relname}"`); + } + } + + // The remainder: rows before the cutoff inside surviving partitions, + // the default partition, or a plain (test) table. + const deleted = await this.prisma.readEvent.deleteMany({ + where: { occurredAt: { lt: cutoff } }, + }); + const count = dropped + deleted.count; + if (count > 0) { + await this.audit.record({ + action: 'read_trail.pruned', + details: { count, cutoff: cutoff.toISOString(), retentionDays }, + }); + } + return count; + } +} diff --git a/apps/api/src/read-trail/read-trail-storage.e2e.db.test.ts b/apps/api/src/read-trail/read-trail-storage.e2e.db.test.ts new file mode 100644 index 0000000..bf26b73 --- /dev/null +++ b/apps/api/src/read-trail/read-trail-storage.e2e.db.test.ts @@ -0,0 +1,308 @@ +import { execFileSync } from 'node:child_process'; +import { join } from 'node:path'; + +import { INestApplication } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; +import { PinoLogger } from 'nestjs-pino'; +import request from 'supertest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; + +import { ClockService } from '../common/clock.service'; +import { InstanceSettingsService } from '../settings/instance-settings.service'; +import { createTestApp, sessionCookieOf } from '../testing/test-app'; +import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db'; +import { UsersService } from '../users/users.service'; + +import { ReadTrailMaintenanceService } from './read-trail-maintenance.service'; + +const API_ROOT = join(__dirname, '..', '..'); +const MS_PER_DAY = 24 * 60 * 60 * 1000; + +/** + * Read-trail storage (issue #224, ADR 0023): its own retention period with + * an audited deletion, the Site-Admin query path, and — against a freshly + * migrated database, where the real DDL applies — the monthly RANGE + * partitioning with per-partition dedup indexes and DROP-based pruning. + */ +describe.skipIf(!hasTestDb)('read-trail storage (e2e, issue #224)', () => { + const suffix = uniqueSuffix(); + + describe('retention and admin query path (shared database)', () => { + let app: INestApplication; + let prisma: PrismaClient; + const password = 'lesetrail speicher 123'; + + let adminId: string; + let adminCookie: string; + let readerCookie: string; + // Plain text ids — read_events has no FKs by design (#222). + const pondId = `pond-${suffix}`; + + const api = () => request(app.getHttpServer()); + + async function makeUser(handle: string, siteAdmin: boolean) { + const users = app.get(UsersService); + const user = await users.createUser({ + username: `${handle}-${suffix}`, + email: `${handle}-${suffix}@example.test`, + displayName: handle, + password, + locale: 'en', + }); + await users.markEmailVerified(user.id); + if (siteAdmin) { + await prisma.user.update({ where: { id: user.id }, data: { isSiteAdmin: true } }); + } + const cookie = sessionCookieOf( + await api() + .post('/api/v1/auth/login') + .send({ usernameOrEmail: `${handle}-${suffix}`, password }) + .expect(200), + ); + return { id: user.id, cookie }; + } + + function eventRow(overrides: Record) { + return { + actorId: null, + sessionKey: 'anon', + pageId: null, + pondId, + channel: 'page_view', + classification: 'vs_nfd', + dedupKey: `k-${suffix}-${Math.random().toString(36).slice(2)}`, + windowBucket: BigInt(Math.floor(Math.random() * 1_000_000_000)), + windowSeconds: 300, + ...overrides, + }; + } + + beforeAll(async () => { + prisma = createTestPrisma(); + await prisma.rateLimit.deleteMany({}); + app = await createTestApp(); + const admin = await makeUser('storage-admin', true); + adminId = admin.id; + adminCookie = admin.cookie; + const reader = await makeUser('storage-reader', false); + readerCookie = reader.cookie; + }); + + afterAll(async () => { + await prisma.readEvent.deleteMany({ where: { pondId } }); + await prisma.auditEntry.deleteMany({ where: { action: 'read_trail.pruned' } }); + await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.retentionDays' } }); + await prisma.user.deleteMany({ where: { username: { contains: suffix } } }); + await prisma.$disconnect(); + await app.close(); + }); + + it('prunes only events past its own period and audits the deletion', async () => { + await app.get(InstanceSettingsService).set('readTrail.retentionDays', 30, adminId); + const now = Date.now(); + await prisma.readEvent.createMany({ + data: [ + eventRow({ occurredAt: new Date(now - 40 * MS_PER_DAY) }), + eventRow({ occurredAt: new Date(now - 31 * MS_PER_DAY) }), + eventRow({ occurredAt: new Date(now - 5 * MS_PER_DAY) }), + ], + }); + + const maintenance = app.get(ReadTrailMaintenanceService); + const pruned = await maintenance.pruneExpired(); + expect(pruned).toBeGreaterThanOrEqual(2); + + const remaining = await prisma.readEvent.findMany({ where: { pondId } }); + expect(remaining).toHaveLength(1); + + const audit = await prisma.auditEntry.findFirst({ + where: { action: 'read_trail.pruned' }, + orderBy: { at: 'desc' }, + }); + expect(audit).not.toBeNull(); + expect(audit!.details).toMatchObject({ retentionDays: 30 }); + }); + + it('answers "who read page X" and "what did user Y read" for Site Admins only', async () => { + const pageId = `page-${suffix}`; + await prisma.readEvent.createMany({ + data: [ + eventRow({ pageId, actorId: adminId, channel: 'export' }), + eventRow({ pageId: `other-${suffix}`, actorId: null }), + ], + }); + + const byPage = await api() + .get(`/api/v1/admin/system/read-events?pageId=${pageId}`) + .set('Cookie', adminCookie) + .expect(200); + expect(byPage.body.total).toBe(1); + expect(byPage.body.entries[0]).toMatchObject({ + pageId, + channel: 'export', + windowSeconds: 300, + }); + expect(byPage.body.entries[0].actor).toMatchObject({ id: adminId }); + + const byActor = await api() + .get(`/api/v1/admin/system/read-events?actor=storage-admin-${suffix}`) + .set('Cookie', adminCookie) + .expect(200); + expect(byActor.body.total).toBe(1); + + // An unknown username matches nothing rather than everything. + const unknown = await api() + .get(`/api/v1/admin/system/read-events?actor=nobody-${suffix}`) + .set('Cookie', adminCookie) + .expect(200); + expect(unknown.body.total).toBe(0); + + await api() + .get(`/api/v1/admin/system/read-events?pageId=${pageId}`) + .set('Cookie', readerCookie) + .expect(403); + }); + }); + + describe('partitioned shape (fresh database, real migrations)', () => { + const baseUrl = process.env.TEST_DATABASE_URL!; + const dbName = `dorfteich_trail_${suffix}`; + let fresh: PrismaClient; + let maintenance: ReadTrailMaintenanceService; + const auditRecord = vi.fn().mockResolvedValue(undefined); + let retentionDays = 365; + + function freshUrl(): string { + const url = new URL(baseUrl); + url.pathname = `/${dbName}`; + return url.toString(); + } + + beforeAll(async () => { + const admin = new PrismaClient({ datasourceUrl: baseUrl }); + try { + await admin.$executeRawUnsafe(`CREATE DATABASE "${dbName}"`); + } finally { + await admin.$disconnect(); + } + execFileSync( + process.execPath, + [join(API_ROOT, 'node_modules', 'prisma', 'build', 'index.js'), 'migrate', 'deploy'], + { env: { ...process.env, DATABASE_URL: freshUrl() }, stdio: 'pipe', cwd: API_ROOT }, + ); + fresh = new PrismaClient({ datasourceUrl: freshUrl() }); + const settingsStub = { + get: async () => retentionDays, + } as unknown as InstanceSettingsService; + const auditStub = { record: auditRecord } as never; + const loggerStub = { + setContext: () => undefined, + debug: () => undefined, + info: () => undefined, + warn: () => undefined, + error: () => undefined, + } as unknown as PinoLogger; + maintenance = new ReadTrailMaintenanceService( + fresh as never, + settingsStub, + auditStub, + new ClockService(), + loggerStub, + ); + }, 120_000); + + afterAll(async () => { + await fresh.$disconnect(); + const admin = new PrismaClient({ datasourceUrl: baseUrl }); + try { + await admin.$executeRawUnsafe(`DROP DATABASE IF EXISTS "${dbName}" WITH (FORCE)`); + } finally { + await admin.$disconnect(); + } + }); + + it('migrates read_events to a partitioned parent with current-month coverage', async () => { + const kind = await fresh.$queryRaw<{ relkind: string }[]>` + SELECT relkind::text FROM pg_class WHERE relname = 'read_events'`; + expect(kind[0]!.relkind).toBe('p'); + + const partitions = await fresh.$queryRaw<{ relname: string }[]>` + SELECT c.relname::text FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = 'read_events'::regclass ORDER BY c.relname`; + const names = partitions.map((p) => p.relname); + expect(names).toContain('read_events_default'); + expect(names.some((n) => /^read_events_y\d{4}m\d{2}$/.test(n))).toBe(true); + }); + + it('keeps the dedup unique pair enforced per partition (P2002 on the duplicate)', async () => { + const row = { + actorId: null, + sessionKey: 'anon', + pageId: null, + pondId: 'pond-part', + channel: 'page_view', + classification: 'vs_nfd', + dedupKey: `dup-${suffix}`, + windowBucket: 42n, + windowSeconds: 300, + }; + await fresh.readEvent.create({ data: row }); + await expect(fresh.readEvent.create({ data: { ...row } })).rejects.toMatchObject({ + code: 'P2002', + }); + }); + + it('creates months ahead and prunes whole expired partitions by DROP, audited', async () => { + await maintenance.ensurePartitions(); + const next = new Date(); + const nextMonth = new Date(Date.UTC(next.getUTCFullYear(), next.getUTCMonth() + 2, 1)); + const nextName = `read_events_y${nextMonth.getUTCFullYear()}m${String( + nextMonth.getUTCMonth() + 1, + ).padStart(2, '0')}`; + const created = await fresh.$queryRawUnsafe<{ relname: string }[]>( + `SELECT relname::text FROM pg_class WHERE relname = '${nextName}'`, + ); + expect(created).toHaveLength(1); + // ...and each new partition carries its own dedup unique index. + const index = await fresh.$queryRawUnsafe<{ indexname: string }[]>( + `SELECT indexname::text FROM pg_indexes WHERE tablename = '${nextName}' + AND indexname = '${nextName}_dedup_key'`, + ); + expect(index).toHaveLength(1); + + // An old month: partition + one event well past retention. + await fresh.$executeRawUnsafe( + `CREATE TABLE "read_events_y2020m01" PARTITION OF "read_events" + FOR VALUES FROM ('2020-01-01') TO ('2020-02-01')`, + ); + await fresh.readEvent.create({ + data: { + occurredAt: new Date('2020-01-15T12:00:00Z'), + actorId: null, + sessionKey: 'anon', + pageId: null, + pondId: 'pond-part', + channel: 'export', + classification: 'vs_nfd', + dedupKey: `old-${suffix}`, + windowBucket: 1n, + windowSeconds: 300, + }, + }); + + const pruned = await maintenance.pruneExpired(); + expect(pruned).toBeGreaterThanOrEqual(1); + const gone = await fresh.$queryRawUnsafe<{ relname: string }[]>( + `SELECT relname::text FROM pg_class WHERE relname = 'read_events_y2020m01'`, + ); + expect(gone).toHaveLength(0); + expect(auditRecord).toHaveBeenCalledWith( + expect.objectContaining({ action: 'read_trail.pruned' }), + ); + // The current month's events survive. + const kept = await fresh.readEvent.count({ where: { dedupKey: `dup-${suffix}` } }); + expect(kept).toBe(1); + retentionDays = 365; // restore for any later use + }); + }); +}); diff --git a/apps/api/src/read-trail/read-trail.module.ts b/apps/api/src/read-trail/read-trail.module.ts index 61a12aa..3f812de 100644 --- a/apps/api/src/read-trail/read-trail.module.ts +++ b/apps/api/src/read-trail/read-trail.module.ts @@ -1,10 +1,16 @@ -import { Global, Module } from '@nestjs/common'; +import { Global, Module, OnModuleInit } from '@nestjs/common'; import { CommonModule } from '../common/common.module'; +import { SchedulerModule } from '../scheduler/scheduler.module'; +import { SchedulerService } from '../scheduler/scheduler.service'; import { SettingsModule } from '../settings/settings.module'; +import { ReadTrailMaintenanceService } from './read-trail-maintenance.service'; import { ReadTrailService } from './read-trail.service'; +/** Daily, per operations.md's maintenance-jobs table (issue #224). */ +const READ_TRAIL_MAINTENANCE_CADENCE_SECONDS = 24 * 60 * 60; + /** * Global like AuditModule and for the same reason: the read-access trail * (issue #222, ADR 0023) cuts across every module that serves page content — @@ -12,8 +18,23 @@ import { ReadTrailService } from './read-trail.service'; */ @Global() @Module({ - imports: [CommonModule, SettingsModule], - providers: [ReadTrailService], - exports: [ReadTrailService], + imports: [CommonModule, SchedulerModule, SettingsModule], + providers: [ReadTrailService, ReadTrailMaintenanceService], + exports: [ReadTrailService, ReadTrailMaintenanceService], }) -export class ReadTrailModule {} +export class ReadTrailModule implements OnModuleInit { + constructor( + private readonly scheduler: SchedulerService, + private readonly maintenance: ReadTrailMaintenanceService, + ) {} + + onModuleInit(): void { + this.scheduler.register({ + name: 'read-trail-maintenance', + cadenceSeconds: READ_TRAIL_MAINTENANCE_CADENCE_SECONDS, + run: async () => { + await this.maintenance.run(); + }, + }); + } +} diff --git a/apps/api/src/settings/instance-settings.service.ts b/apps/api/src/settings/instance-settings.service.ts index 30b7be0..11a84ec 100644 --- a/apps/api/src/settings/instance-settings.service.ts +++ b/apps/api/src/settings/instance-settings.service.ts @@ -56,6 +56,13 @@ export const INSTANCE_SETTINGS = { // 5 minutes keeps a live Yjs session (collab tokens every 60 s) at a // bounded ~12 events/hour/page while still evidencing distinct visits. 'readTrail.dedupWindowMinutes': z.number().int().min(1).default(5), + // Read-trail retention (issue #224): days a read event is kept before the + // daily maintenance job removes it — deliberately independent of + // `audit.retentionDays` (#196), because volume, purpose and legal basis + // differ. The deletion itself is audited (`read_trail.pruned`), so a gap + // is always explainable. One year mirrors the audit default; shortening + // it is an operator decision under the purpose limitation (#225). + 'readTrail.retentionDays': z.number().int().min(1).default(365), // Default VS-NfD classification for newly created pages (ADR 0022, // issue #204). An instance operated inside a classified environment sets // this to `vs_nfd` so nothing starts unmarked; inheritance from the diff --git a/apps/web/e2e/system.spec.ts b/apps/web/e2e/system.spec.ts index 203ca7e..2d3dd9a 100644 --- a/apps/web/e2e/system.spec.ts +++ b/apps/web/e2e/system.spec.ts @@ -20,8 +20,9 @@ test('lists maintenance jobs and triggers one manually', async ({ browser }) => // Keep in sync with the scheduler registrations: trash-purge, // version-thinning, page-compaction, data-export-purge, // notification-digest, orphan-file-sweep (#194), audit-retention (#196), - // conversion-payload-prune (#233), mail-outbox-retention (#234). - await expect(jobsTable.locator('tbody tr')).toHaveCount(9); + // conversion-payload-prune (#233), mail-outbox-retention (#234), + // read-trail-maintenance (#224). + await expect(jobsTable.locator('tbody tr')).toHaveCount(10); const firstRow = jobsTable.locator('tbody tr').first(); await firstRow.getByRole('button').click(); diff --git a/docs/architecture/adr/0023-read-access-audit-trail.md b/docs/architecture/adr/0023-read-access-audit-trail.md index a5681d7..38979c5 100644 --- a/docs/architecture/adr/0023-read-access-audit-trail.md +++ b/docs/architecture/adr/0023-read-access-audit-trail.md @@ -89,6 +89,23 @@ traffic per open document. anonymous browsers apart would require fingerprinting (IP/UA), which the purpose limitation (#225) rules out. +## Decisions taken in #224 + +- **Monthly RANGE partitions** on `occurred_at`, plus a DEFAULT partition + as a safety net: a lagging maintenance job must never make classified + reads fail (the trail write is hard-failing — an ops miss must not + become an outage). Retention DROPs whole expired months without + scanning; the pruning run is audited (`read_trail.pruned`) under the + trail's own `readTrail.retentionDays` (default 365). +- **The dedup unique pair lives per partition** (PostgreSQL cannot carry + it on the parent without the partition key). A dedup bucket spanning a + month boundary can therefore record one duplicate — over-recording is + acceptable, gaps are not. +- **Query path is API-only** (`GET /admin/system/read-events`, Site + Admin): the trail is an examiner's tool, not a daily screen. Evidence + nobody can read is not evidence — the path exists; a panel does not. +- **Growth measured**: ~1 MB per 1000 events including indexes. + ## Consequences - The scope limit is the feature's strongest argument in the works-council diff --git a/docs/architecture/audit-events.md b/docs/architecture/audit-events.md index bd2f5a8..8e2e695 100644 --- a/docs/architecture/audit-events.md +++ b/docs/architecture/audit-events.md @@ -1,7 +1,7 @@ # Audit event catalogue -**Catalogue version 1.1 (2026-07-31; 1.1 adds `page.classification_*`, -issue #205).** +**Catalogue version 1.2 (2026-07-31; 1.2 adds `read_trail.pruned`, +issue #224; 1.1 added `page.classification_*`, issue #205).** This is the operator-facing contract for the audit trail: every event id the application can emit, with its trigger, severity, actor/target @@ -103,11 +103,12 @@ failure), `warning` = feeds detection (suspicious or destructive), ### Content integrity & lifecycle (`file.*`, `pond.*`, `audit.*`) -| Id | Trigger | Severity | Actor | Target | Fields | -| ----------------------- | ------------------------------------------------------------------- | -------- | ---------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | -| `file.integrity_failed` | Attachment download hash mismatch — fail-closed (issue #199) | critical | `null` (any downloader; detection) | `attachment` | `pondId`, `expected` (stored sha256), `actual` (computed sha256) | -| `pond.purged` | Pond irreversibly destroyed (manual or trash retention, issue #193) | notice | admin, `null` when retention-run | `pond` | `trigger` (`manual` \| `retention`) plus per-object-type deletion counts (e.g. `pages`, `attachments`, … — informational, keys may grow) | -| `audit.pruned` | Audit retention deleted rows past the period (issue #196) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` | +| Id | Trigger | Severity | Actor | Target | Fields | +| ----------------------- | -------------------------------------------------------------------- | -------- | ---------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `file.integrity_failed` | Attachment download hash mismatch — fail-closed (issue #199) | critical | `null` (any downloader; detection) | `attachment` | `pondId`, `expected` (stored sha256), `actual` (computed sha256) | +| `pond.purged` | Pond irreversibly destroyed (manual or trash retention, issue #193) | notice | admin, `null` when retention-run | `pond` | `trigger` (`manual` \| `retention`) plus per-object-type deletion counts (e.g. `pages`, `attachments`, … — informational, keys may grow) | +| `audit.pruned` | Audit retention deleted rows past the period (issue #196) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` | +| `read_trail.pruned` | Read-trail retention removed events past its own period (issue #224) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` | ### Public API (`api.*`) diff --git a/docs/architecture/data-model.md b/docs/architecture/data-model.md index 7ad29d3..ed6a8b9 100644 --- a/docs/architecture/data-model.md +++ b/docs/architecture/data-model.md @@ -186,6 +186,40 @@ Cached counters per pond: `storage_bytes_used`, `editor_count`, - `jobs`: conversion jobs for import/export (ADR 0009) and maintenance jobs (compaction, trash purge, quota reconciliation) with status + timestamps. +## Read-access trail (`read_events`, issues #222–#224, ADR 0023) + +One row per read of a `classification = vs_nfd` page per channel and dedup +window — separate from `audit_log` because volume, purpose and legal basis +all differ. + +| Column | Notes | +| ---------------------------- | ------------------------------------------------------------------------------------ | +| `id`, `occurred_at` | composite PK (the partition key must be part of it) | +| `actor_id` | reader, null = anonymous; **no FK** — evidence outlives accounts | +| `session_key` | `session:` / `token:` / `job:` / `anon` | +| `page_id`, `pond_id` | plain ids, **no FK** — evidence outlives purges | +| `channel` | `page_view` / `no_js_shell` / `public_api` / `attachment` / `export` / `collab_join` | +| `classification` | at read time; reclassification never rewrites history | +| `dedup_key`, `window_bucket` | unique pair per partition — race-free dedup (#223) | +| `window_seconds` | the row represents up to this many seconds of access | +| `details` | small context (export format, attachment id) — never content | + +Storage shape (issue #224): RANGE-partitioned by month on `occurred_at`, +with a DEFAULT partition as safety net (a lagging maintenance job must +never make classified reads fail — the trail write is hard-failing by +design). The daily `read-trail-maintenance` job creates months ahead +(each with its per-partition dedup index) and applies the trail's own +retention `readTrail.retentionDays` (default 365): whole expired months +are DROPped without scanning, remainders deleted by range; every pruning +run is audited as `read_trail.pruned`. Growth, measured on PG 17: +**~1 MB per 1000 events** including indexes (≈970 bytes/row) — a +100-person instance at hundreds of classified reads/day stays in the tens +of MB per year. + +Query path: `GET /admin/system/read-events` (Site Admin, filters +`pageId`, `actor`, `channel`, `from`/`to`) — deliberately API-only, no +panel: the trail is an examiner's tool, not a daily screen. + ## Comments & notifications (later milestone) - `comments`: `page_id`, `author_id`, `body` (Markdown), `anchor` diff --git a/docs/architecture/operations.md b/docs/architecture/operations.md index 015b80c..b84a217 100644 --- a/docs/architecture/operations.md +++ b/docs/architecture/operations.md @@ -109,6 +109,7 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack. | conversion payload prune | daily | null finished conversion jobs' raw bytes (#233) | | mail outbox retention | daily | delete sent/failed outbox rows past period (#234) | | mail outbox retry | every minute | e-mail delivery with backoff | +| read-trail maintenance | daily | partition months ahead + prune `read_events` (#224) | Job outcomes are visible in the Site Admin UI (last run, status) — that panel is the operator's single glance for instance health. diff --git a/docs/vs-nfd/20-massnahmenplan.md b/docs/vs-nfd/20-massnahmenplan.md index ff4c835..f252195 100644 --- a/docs/vs-nfd/20-massnahmenplan.md +++ b/docs/vs-nfd/20-massnahmenplan.md @@ -136,7 +136,7 @@ Protokolliert werden Lesezugriffe **ausschließlich** auf Seiten mit Attachment-Download, Export, No-JS-Shell, Collab-WS-Join · 4 AT · #222 - [x] Dedup-Fenster (eine Sitzung + eine Seite innerhalb N Minuten = ein Ereignis), sonst erzeugt Yjs-Sync eine Ereignisflut · 2 AT · #223 -- [ ] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224 +- [x] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224 - [ ] Abschaltbar, Zweckbindung dokumentiert · 1–2 AT · #225 Vorteil über den Aufwand hinaus: Die Zweckbindung ist sauber begründbar diff --git a/docs/vs-nfd/50-haertungsleitfaden.md b/docs/vs-nfd/50-haertungsleitfaden.md index 3491c33..6ee05a7 100644 --- a/docs/vs-nfd/50-haertungsleitfaden.md +++ b/docs/vs-nfd/50-haertungsleitfaden.md @@ -40,6 +40,7 @@ Settings-Cache ist in-process (operations.md). | `backup.nextcloud.enabled` | `false` | `false` | „Backup nur lokal": kein Anwendungs-Upload von Restore-Sets zu Drittdiensten. Fernspiegel regelt ausschließlich die Deploy-Allowlist (1.2). | | `trash.retentionDays`, `audit.retentionDays`, `conversion.payloadRetentionDays`, `mail.outboxRetentionDays` | Defaults (30/365/30/30) | ebd. | Aufbewahrung bewusst begrenzt; Verkürzung nach Betreiber-Löschkonzept zulässig (Betriebshandbuch §5). | | `readTrail.dedupWindowMinutes` | Default (5) | `5` | Dedup-Fenster des Lesetrails (#223): je (Sitzung, Seite, Kanal) ein Ereignis pro Fenster — begrenzt die Ereignisflut einer Live-Sitzung auf ~12/h. Kleiner = feineres Protokoll und mehr Zeilen; Änderung mit dem Zweckbindungs-Dokument (#225) abstimmen. | +| `readTrail.retentionDays` | Default (365) | `365` | eigene Aufbewahrung des Lesetrails (#224), bewusst getrennt von `audit.retentionDays`; Löschläufe sind selbst auditiert (`read_trail.pruned`). Dauer mit der Zweckbindung (#225) und dem Betreiber-Löschkonzept abstimmen. | | `legal.imprint`, `legal.privacyPolicy` | befüllt | leer | Betreiberpflicht; leere Seiten zeigen einen Warnbanner. | ### 1.2 Deploy-Konfiguration (`.env` / Compose — nur Plattformzugriff, bewusst nicht per Admin-UI) diff --git a/packages/shared/src/system.ts b/packages/shared/src/system.ts index 182f085..ba10e07 100644 --- a/packages/shared/src/system.ts +++ b/packages/shared/src/system.ts @@ -164,6 +164,49 @@ export interface AuditListView { total: number; } +/** + * Site-Admin query path over the read-access trail (issue #224, ADR 0023) — + * evidence nobody can read is not evidence. Answers the two expected + * questions: "who read page X" (pageId) and "what did user Y read" (actor), + * both within a period. + */ +export const READ_EVENT_PAGE_SIZE = 50; + +export const readEventListQuerySchema = z.object({ + /** Page id the events belong to. */ + pageId: z.string().trim().min(1).optional(), + /** Exact username of the reading user. */ + actor: z.string().trim().min(1).optional(), + channel: z.string().trim().min(1).optional(), + from: z.coerce.date().optional(), + to: z.coerce.date().optional(), + page: z.coerce.number().int().min(1).default(1), +}); + +export type ReadEventListQuery = z.infer; + +export interface ReadEventView { + id: string; + occurredAt: string; + actor: AuditActorView | null; + /** Null for pond-level attachments (no page); the details name the file. */ + pageId: string | null; + pondId: string; + channel: string; + classification: string; + /** Seconds the event's dedup window spans (#223) — the row represents up + * to this much access time, not a single request. */ + windowSeconds: number; + details: Record | null; +} + +export interface ReadEventListView { + entries: ReadEventView[]; + page: number; + pageCount: number; + total: number; +} + export interface StoragePondView { pondId: string; name: string; -- 2.45.2 From 4af5e6e81f2fe08fc7b60c6228c2de0573e6979d Mon Sep 17 00:00:00 2001 From: Claude Fable 5 Date: Fri, 31 Jul 2026 12:24:57 +0200 Subject: [PATCH 3/3] #225: read-trail master switch and written purpose limitation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New instance switch readTrail.enabled, default OFF: read logging is employee monitoring in a works council's eyes — an ordinary instance must not surveil reads. Off means no event is written ANYWHERE (no row, no stdout line, verified by test); the api announces the switch position once per boot, so an eventless trail is never ambiguous — a gap reads as "was off", never "was lost". The written purpose limitation ships as section 7 of the VS-NfD security documentation (#228): what is recorded (no content, no titles, no IPs, no fingerprinting), why (evidence for reads of marked content only — variant A is the technical anchor of the promise), who may read it (Site Admin, API-only), for how long (readTrail.retentionDays, audited pruning), and what it may NOT be used for (no performance or behaviour monitoring). The hardening guide's reference configuration turns the trail on (reference value true) and points to that text; the existing trail suites now enable the switch explicitly. Refs #225. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AUtYMxwTCMHG9mVHnwbFg8 --- .../read-trail-dedup.e2e.db.test.ts | 4 + .../read-trail-switch.e2e.db.test.ts | 132 ++++++++++++++++++ .../src/read-trail/read-trail.e2e.db.test.ts | 7 +- apps/api/src/read-trail/read-trail.module.ts | 6 +- apps/api/src/read-trail/read-trail.service.ts | 25 ++++ .../src/settings/instance-settings.service.ts | 8 ++ .../adr/0023-read-access-audit-trail.md | 16 +++ docs/architecture/security.md | 5 + docs/vs-nfd/20-massnahmenplan.md | 2 +- docs/vs-nfd/50-haertungsleitfaden.md | 1 + docs/vs-nfd/60-sicherheitsdokumentation.md | 39 ++++++ 11 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 apps/api/src/read-trail/read-trail-switch.e2e.db.test.ts diff --git a/apps/api/src/read-trail/read-trail-dedup.e2e.db.test.ts b/apps/api/src/read-trail/read-trail-dedup.e2e.db.test.ts index 8b05f79..68e9b56 100644 --- a/apps/api/src/read-trail/read-trail-dedup.e2e.db.test.ts +++ b/apps/api/src/read-trail/read-trail-dedup.e2e.db.test.ts @@ -4,6 +4,7 @@ import request from 'supertest'; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import { ClockService } from '../common/clock.service'; +import { InstanceSettingsService } from '../settings/instance-settings.service'; import { createTestApp, sessionCookieOf } from '../testing/test-app'; import { createTestPrisma, grantOwnerAdmin, hasTestDb, uniqueSuffix } from '../testing/test-db'; import { UsersService } from '../users/users.service'; @@ -54,6 +55,8 @@ describe.skipIf(!hasTestDb)('read-trail dedup window (e2e, issue #223)', () => { }); await users.markEmailVerified(owner.id); ownerId = owner.id; + // The trail ships OFF by default (#225) — this suite needs it on. + await app.get(InstanceSettingsService).set('readTrail.enabled', true, ownerId); ownerCookie = await login(); const pond = await prisma.pond.create({ @@ -85,6 +88,7 @@ describe.skipIf(!hasTestDb)('read-trail dedup window (e2e, issue #223)', () => { }); afterAll(async () => { + await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.enabled' } }); await prisma.readEvent.deleteMany({ where: { pondId } }); await prisma.attachment.deleteMany({ where: { pondId } }); await prisma.roleGrant.deleteMany({ where: { pondId } }); diff --git a/apps/api/src/read-trail/read-trail-switch.e2e.db.test.ts b/apps/api/src/read-trail/read-trail-switch.e2e.db.test.ts new file mode 100644 index 0000000..3987608 --- /dev/null +++ b/apps/api/src/read-trail/read-trail-switch.e2e.db.test.ts @@ -0,0 +1,132 @@ +import { INestApplication } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; +import { PinoLogger } from 'nestjs-pino'; +import request from 'supertest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; + +import { InstanceSettingsService } from '../settings/instance-settings.service'; +import { createTestApp, sessionCookieOf } from '../testing/test-app'; +import { createTestPrisma, grantOwnerAdmin, hasTestDb, uniqueSuffix } from '../testing/test-db'; +import { UsersService } from '../users/users.service'; + +import { ReadTrailService } from './read-trail.service'; + +/** + * The read-trail master switch (issue #225, ADR 0023): OFF is the default + * and means no event is written ANYWHERE — no row, no stdout line; ON + * restores the full #222 semantics. The switch position is announced so a + * silent trail is never ambiguous. + */ +describe.skipIf(!hasTestDb)('read-trail switch (e2e, issue #225)', () => { + let app: INestApplication; + let prisma: PrismaClient; + const suffix = uniqueSuffix(); + const password = 'schalter zeugen 123'; + + let ownerId: string; + let ownerCookie: string; + let pondId: string; + let classifiedId: string; + + const api = () => request(app.getHttpServer()); + + beforeAll(async () => { + prisma = createTestPrisma(); + await prisma.rateLimit.deleteMany({}); + // No leftover switch row: this suite tests the DEFAULT (off). + await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.enabled' } }); + app = await createTestApp(); + const users = app.get(UsersService); + + const owner = await users.createUser({ + username: `switch-owner-${suffix}`, + email: `switch-owner-${suffix}@example.test`, + displayName: 'Switch Owner', + password, + locale: 'en', + }); + await users.markEmailVerified(owner.id); + ownerId = owner.id; + ownerCookie = sessionCookieOf( + await api() + .post('/api/v1/auth/login') + .send({ usernameOrEmail: `switch-owner-${suffix}`, password }) + .expect(200), + ); + + const pond = await prisma.pond.create({ + data: { slug: `switch-pond-${suffix}`, name: 'Switch Pond', type: 'SHARED', ownerId }, + }); + pondId = pond.id; + await grantOwnerAdmin(prisma, pondId, ownerId); + const page = await prisma.page.create({ + data: { + pondId, + slug: `classified-${suffix}`, + title: 'Classified', + classification: 'VS_NFD', + createdBy: ownerId, + sortKey: 'a0', + ydocState: new Uint8Array(), + contentCache: { + create: { plainText: 'x', markdown: 'x', html: '

x

', outline: [] }, + }, + }, + }); + classifiedId = page.id; + }); + + afterAll(async () => { + await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.enabled' } }); + await prisma.readEvent.deleteMany({ where: { pondId } }); + await prisma.roleGrant.deleteMany({ where: { pondId } }); + await prisma.pageContentCache.deleteMany({ where: { page: { pondId } } }); + await prisma.page.deleteMany({ where: { pondId } }); + await prisma.pond.deleteMany({ where: { id: pondId } }); + await prisma.user.deleteMany({ where: { id: ownerId } }); + await prisma.$disconnect(); + await app.close(); + }); + + it('is OFF by default: a classified read writes nothing — no row, no stdout line', async () => { + const trail = app.get(ReadTrailService); + const logger = (trail as unknown as { logger: PinoLogger }).logger; + const infoSpy = vi.spyOn(logger, 'info'); + try { + await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200); + } finally { + infoSpy.mockRestore(); + } + expect(await prisma.readEvent.count({ where: { pondId } })).toBe(0); + expect(infoSpy).not.toHaveBeenCalled(); + }); + + it('records again the moment the switch turns on', async () => { + await app.get(InstanceSettingsService).set('readTrail.enabled', true, ownerId); + try { + await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200); + expect(await prisma.readEvent.count({ where: { pondId } })).toBe(1); + } finally { + await app.get(InstanceSettingsService).set('readTrail.enabled', false, ownerId); + } + }); + + it('announces the switch position so silence is never ambiguous', async () => { + const trail = app.get(ReadTrailService); + const logger = (trail as unknown as { logger: PinoLogger }).logger; + const warnSpy = vi.spyOn(logger, 'warn'); + const infoSpy = vi.spyOn(logger, 'info'); + try { + await trail.announceState(); // switch is off after the previous test + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('NOT evidenced')); + + await app.get(InstanceSettingsService).set('readTrail.enabled', true, ownerId); + await trail.announceState(); + expect(infoSpy).toHaveBeenCalledWith(expect.stringContaining('enabled')); + } finally { + warnSpy.mockRestore(); + infoSpy.mockRestore(); + await app.get(InstanceSettingsService).set('readTrail.enabled', false, ownerId); + } + }); +}); diff --git a/apps/api/src/read-trail/read-trail.e2e.db.test.ts b/apps/api/src/read-trail/read-trail.e2e.db.test.ts index b9def60..f33dc0b 100644 --- a/apps/api/src/read-trail/read-trail.e2e.db.test.ts +++ b/apps/api/src/read-trail/read-trail.e2e.db.test.ts @@ -74,6 +74,9 @@ describe.skipIf(!hasTestDb)('read-access trail (e2e, issue #222)', () => { }); await users.markEmailVerified(owner.id); ownerId = owner.id; + // The trail ships OFF by default (#225) — these suites test the + // instrumented channels, so they run with the switch on. + await app.get(InstanceSettingsService).set('readTrail.enabled', true, ownerId); ownerCookie = sessionCookieOf( await api() .post('/api/v1/auth/login') @@ -131,7 +134,9 @@ describe.skipIf(!hasTestDb)('read-access trail (e2e, issue #222)', () => { }); afterAll(async () => { - await prisma.instanceSetting.deleteMany({ where: { key: 'api.enabled' } }); + await prisma.instanceSetting.deleteMany({ + where: { key: { in: ['api.enabled', 'readTrail.enabled'] } }, + }); await prisma.readEvent.deleteMany({ where: { pondId } }); await prisma.conversionJob.deleteMany({ where: { ownerId } }); await prisma.apiToken.deleteMany({ where: { userId: ownerId } }); diff --git a/apps/api/src/read-trail/read-trail.module.ts b/apps/api/src/read-trail/read-trail.module.ts index 3f812de..577d3a2 100644 --- a/apps/api/src/read-trail/read-trail.module.ts +++ b/apps/api/src/read-trail/read-trail.module.ts @@ -26,9 +26,10 @@ export class ReadTrailModule implements OnModuleInit { constructor( private readonly scheduler: SchedulerService, private readonly maintenance: ReadTrailMaintenanceService, + private readonly trail: ReadTrailService, ) {} - onModuleInit(): void { + async onModuleInit(): Promise { this.scheduler.register({ name: 'read-trail-maintenance', cadenceSeconds: READ_TRAIL_MAINTENANCE_CADENCE_SECONDS, @@ -36,5 +37,8 @@ export class ReadTrailModule implements OnModuleInit { await this.maintenance.run(); }, }); + // One line per boot stating the switch position (issue #225) — a silent + // trail must never be ambiguous. + await this.trail.announceState(); } } diff --git a/apps/api/src/read-trail/read-trail.service.ts b/apps/api/src/read-trail/read-trail.service.ts index de0845f..e67bc32 100644 --- a/apps/api/src/read-trail/read-trail.service.ts +++ b/apps/api/src/read-trail/read-trail.service.ts @@ -84,7 +84,32 @@ export class ReadTrailService { this.logger.setContext(ReadTrailService.name); } + /** States the switch position once at startup (issue #225): a trail + * without events must be distinguishable from a disabled trail — the log + * line makes the gap explainable either way. Must never fail the boot: + * healthz-only environments come up without a reachable database. */ + async announceState(): Promise { + let enabled: boolean; + try { + enabled = await this.settings.get('readTrail.enabled'); + } catch { + this.logger.warn('read_trail: switch position unknown at startup (settings unavailable)'); + return; + } + if (enabled) { + this.logger.info('read_trail: enabled — reads of classified pages are recorded'); + } else { + this.logger.warn( + 'read_trail: disabled — reads of classified pages are NOT evidenced (readTrail.enabled)', + ); + } + } + async record(event: ReadEventInput): Promise { + // The master switch (issue #225): off means nothing is written anywhere + // — no row, no stdout line. The startup announcement above is what keeps + // the resulting silence unambiguous. + if (!(await this.settings.get('readTrail.enabled'))) return; const { actorId, sessionKey, pageId, pondId, channel, details } = event; const windowSeconds = (await this.settings.get('readTrail.dedupWindowMinutes')) * 60; const dedupKey = `${sessionKey}:${pageId ?? '-'}:${channel}`; diff --git a/apps/api/src/settings/instance-settings.service.ts b/apps/api/src/settings/instance-settings.service.ts index 11a84ec..aebcaee 100644 --- a/apps/api/src/settings/instance-settings.service.ts +++ b/apps/api/src/settings/instance-settings.service.ts @@ -51,6 +51,14 @@ export const INSTANCE_SETTINGS = { // bounded. PENDING rows — including failed-but-retryable ones — are // never touched; the retry loop owns them. 'mail.outboxRetentionDays': z.number().int().min(1).default(30), + // Read-trail master switch (issue #225, ADR 0023). Default OFF: read + // logging is employee monitoring in a works council's eyes — an ordinary + // instance must not surveil reads. The VS-NfD reference configuration + // (#227) turns it on together with the written purpose limitation + // (60-sicherheitsdokumentation.md §7). Off means NO event is written + // anywhere, including stdout; the api states the switch position once at + // startup, so a gap in the evidence is never ambiguous. + 'readTrail.enabled': z.boolean().default(false), // Read-trail dedup window (issue #223, ADR 0023): one event per // (session, page, channel) within an aligned window of this many minutes. // 5 minutes keeps a live Yjs session (collab tokens every 60 s) at a diff --git a/docs/architecture/adr/0023-read-access-audit-trail.md b/docs/architecture/adr/0023-read-access-audit-trail.md index 38979c5..da3947b 100644 --- a/docs/architecture/adr/0023-read-access-audit-trail.md +++ b/docs/architecture/adr/0023-read-access-audit-trail.md @@ -106,6 +106,22 @@ traffic per open document. nobody can read is not evidence — the path exists; a panel does not. - **Growth measured**: ~1 MB per 1000 events including indexes. +## Decisions taken in #225 + +- **Default OFF** (`readTrail.enabled`): read logging is employee + monitoring in a works council's eyes — an ordinary instance must not + surveil reads. The VS-NfD reference configuration turns it on together + with the written purpose limitation + (`docs/vs-nfd/60-sicherheitsdokumentation.md` §7, referenced from the + hardening guide). +- **Off means silence everywhere** — no row, no stdout line; the api + announces the switch position once per boot, so an eventless trail is + never ambiguous (a gap reads as "was off", not "was lost"). +- The purpose limitation names what is recorded, why, who may read it + (Site Admin, API-only), for how long, and what it may NOT be used for + (no performance or behaviour monitoring). Its technical anchor is the + variant-A scope: unmarked content produces no trace. + ## Consequences - The scope limit is the feature's strongest argument in the works-council diff --git a/docs/architecture/security.md b/docs/architecture/security.md index 882a16b..5d0a780 100644 --- a/docs/architecture/security.md +++ b/docs/architecture/security.md @@ -214,6 +214,11 @@ scan docker-archive:/image.tar -o cyclonedx-json` respectively - **Read-access trail** (issue #222, ADR 0023): reads of pages with `classification = vs_nfd` land as `read_events` rows — only classified pages, which is what keeps the purpose limitation defensible (variant A). + Master switch `readTrail.enabled`, **default off** (#225): off means no + event is written anywhere, including stdout, and the api announces the + switch position once per boot so an eventless trail is never ambiguous. + The written purpose limitation lives in + `docs/vs-nfd/60-sicherheitsdokumentation.md` §7. The instrumented channels, and the emission point of each: - `page_view` — authenticated SPA state fetch (`GET /pages/:id`, the by-slug variant), the rendered read view (`/read/...`), the public JSON diff --git a/docs/vs-nfd/20-massnahmenplan.md b/docs/vs-nfd/20-massnahmenplan.md index f252195..bfe503f 100644 --- a/docs/vs-nfd/20-massnahmenplan.md +++ b/docs/vs-nfd/20-massnahmenplan.md @@ -137,7 +137,7 @@ Protokolliert werden Lesezugriffe **ausschließlich** auf Seiten mit - [x] Dedup-Fenster (eine Sitzung + eine Seite innerhalb N Minuten = ein Ereignis), sonst erzeugt Yjs-Sync eine Ereignisflut · 2 AT · #223 - [x] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224 -- [ ] Abschaltbar, Zweckbindung dokumentiert · 1–2 AT · #225 +- [x] Abschaltbar, Zweckbindung dokumentiert · 1–2 AT · #225 Vorteil über den Aufwand hinaus: Die Zweckbindung ist sauber begründbar („nur eingestufte Inhalte"), was die Personalrats-Diskussion beim Kunden diff --git a/docs/vs-nfd/50-haertungsleitfaden.md b/docs/vs-nfd/50-haertungsleitfaden.md index 6ee05a7..445e9e3 100644 --- a/docs/vs-nfd/50-haertungsleitfaden.md +++ b/docs/vs-nfd/50-haertungsleitfaden.md @@ -39,6 +39,7 @@ Settings-Cache ist in-process (operations.md). | `upload.allowedExtensions` | nur das dienstlich Nötige (z. B. `pdf`) | Standardliste | jede zusätzliche Endung vergrößert die Menge nicht prüfbarer Binärformate im Bestand. Bilder sind davon unabhängig immer erlaubt (Magic-Byte-geprüft). | | `backup.nextcloud.enabled` | `false` | `false` | „Backup nur lokal": kein Anwendungs-Upload von Restore-Sets zu Drittdiensten. Fernspiegel regelt ausschließlich die Deploy-Allowlist (1.2). | | `trash.retentionDays`, `audit.retentionDays`, `conversion.payloadRetentionDays`, `mail.outboxRetentionDays` | Defaults (30/365/30/30) | ebd. | Aufbewahrung bewusst begrenzt; Verkürzung nach Betreiber-Löschkonzept zulässig (Betriebshandbuch §5). | +| `readTrail.enabled` | `true` | `false` | **explizit setzen** — der Lesetrail (#222–#225) evidenziert Lesezugriffe auf eingestufte Seiten; Default aus, weil Lesebeobachtung mitbestimmungsrelevant ist. Einschalten NUR zusammen mit der Zweckbindung (Sicherheitsdokumentation §7); die api meldet die Schalterstellung beim Start. | | `readTrail.dedupWindowMinutes` | Default (5) | `5` | Dedup-Fenster des Lesetrails (#223): je (Sitzung, Seite, Kanal) ein Ereignis pro Fenster — begrenzt die Ereignisflut einer Live-Sitzung auf ~12/h. Kleiner = feineres Protokoll und mehr Zeilen; Änderung mit dem Zweckbindungs-Dokument (#225) abstimmen. | | `readTrail.retentionDays` | Default (365) | `365` | eigene Aufbewahrung des Lesetrails (#224), bewusst getrennt von `audit.retentionDays`; Löschläufe sind selbst auditiert (`read_trail.pruned`). Dauer mit der Zweckbindung (#225) und dem Betreiber-Löschkonzept abstimmen. | | `legal.imprint`, `legal.privacyPolicy` | befüllt | leer | Betreiberpflicht; leere Seiten zeigen einen Warnbanner. | diff --git a/docs/vs-nfd/60-sicherheitsdokumentation.md b/docs/vs-nfd/60-sicherheitsdokumentation.md index bb686f5..e17766b 100644 --- a/docs/vs-nfd/60-sicherheitsdokumentation.md +++ b/docs/vs-nfd/60-sicherheitsdokumentation.md @@ -282,3 +282,42 @@ HKDF-Zweckableitung (#188), SHA-256-Integritätshashes für Attachments (#199, fail-closed beim Download). Keine Inhalts- oder Backup-Verschlüsselung, kein eigenes Schlüsselmanagement — bewusst (§52 VSA). + +## 7 Lesetrail: Zweckbindung (Issue #225, ADR 0023) + +Der Lesetrail ist abschaltbar (`readTrail.enabled`, **Default: aus**); +die Referenzkonfiguration (Härtungsleitfaden §1.1) schaltet ihn ein. +Aus heißt: es wird **nirgends** ein Ereignis geschrieben — keine +Tabellenzeile, keine Logzeile; die api meldet die Schalterstellung +einmal beim Start, damit ein ereignisloser Trail nie zweideutig ist. +Diese Zweckbindung ist Teil der Betriebsdokumentation und gegenüber +der Personalvertretung offenzulegen. + +**Was aufgezeichnet wird:** je (Sitzung, Seite, Kanal) und Dedup-Fenster +(§3.5) ein Ereignis mit Zeitstempel, Konto-Id (oder dem Marker `anon`), +Sitzungsschlüssel, Seiten- und Teich-Id, Kanal und der Einstufung zum +Lesezeitpunkt. **Kein** Seiteninhalt, **keine** Titel, **keine** +IP-Adressen, **kein** Fingerprinting anonymer Leser. + +**Warum:** ausschließlich Beweissicherung für Lesezugriffe auf **als +VS-NfD gekennzeichnete** Inhalte (§52-konforme Nachvollziehbarkeit: +„welche eingestufte Seite wurde wann über welchen Kanal gelesen") — +Plattform-Logs kennen URLs, nicht Einstufungen. Zugriffe auf nicht +eingestufte Inhalte werden **nie** erfasst (Variante A, ADR 0023). + +**Wer lesen darf:** ausschließlich Site-Admins über +`GET /admin/system/read-events` (kein UI-Panel — Prüfwerkzeug, kein +Alltagsbildschirm). Organisatorische Beschränkung auf den +Sicherheitsbeauftragten: Betreibersache (Betriebshandbuch §6 +Rollentrennung). + +**Wie lange:** `readTrail.retentionDays` (Default 365 Tage); Löschläufe +sind selbst auditiert (`read_trail.pruned`), sodass jede Lücke erklärbar +ist. + +**Wofür NICHT:** keine Leistungs- oder Verhaltenskontrolle der +Beschäftigten, keine Auswertung von Arbeitsmustern, keine +Anwesenheitskontrolle, keine Weitergabe außerhalb des +Sicherheitsvorfalls- bzw. Prüfkontexts. Die Beschränkung auf +eingestufte Inhalte ist die technische Absicherung dieser Zusage: was +nicht gekennzeichnet ist, erzeugt keine Spur. -- 2.45.2