#225: read-trail master switch and purpose limitation #281

Merged
fable-5 merged 3 commits from issue-225-read-trail-switch into main 2026-07-31 12:53:37 +02:00
25 changed files with 1288 additions and 33 deletions

View File

@ -0,0 +1,32 @@
-- #223 (ADR 0023): dedup window for the read trail. Aligned buckets
-- (floor(epoch / window)) with a unique (dedup_key, window_bucket) pair make
-- concurrent duplicates collapse race-free at insert time.
ALTER TABLE "read_events"
ADD COLUMN "dedup_key" TEXT,
ADD COLUMN "window_bucket" BIGINT,
ADD COLUMN "window_seconds" INTEGER;
-- Backfill rows written between the #222 and #223 deploys under the default
-- 5-minute window, then apply the window's own semantics retroactively:
-- within one (key, bucket) pair only the FIRST event is the evidence row —
-- exactly what the window would have recorded had it existed.
UPDATE "read_events"
SET "dedup_key" = "session_key" || ':' || COALESCE("page_id", '-') || ':' || "channel",
"window_bucket" = FLOOR(EXTRACT(EPOCH FROM "occurred_at") / 300)::BIGINT,
"window_seconds" = 300
WHERE "dedup_key" IS NULL;
DELETE FROM "read_events" keep
USING "read_events" first
WHERE keep."dedup_key" = first."dedup_key"
AND keep."window_bucket" = first."window_bucket"
AND (first."occurred_at" < keep."occurred_at"
OR (first."occurred_at" = keep."occurred_at" AND first."id" < keep."id"));
ALTER TABLE "read_events"
ALTER COLUMN "dedup_key" SET NOT NULL,
ALTER COLUMN "window_bucket" SET NOT NULL,
ALTER COLUMN "window_seconds" SET NOT NULL;
CREATE UNIQUE INDEX "read_events_dedup_key_window_bucket_key"
ON "read_events"("dedup_key", "window_bucket");

View File

@ -0,0 +1,76 @@
-- #224 (ADR 0023): convert read_events to monthly RANGE partitions on
-- occurred_at. Volume grows unbounded with use; retention then DROPs whole
-- expired partitions instead of scanning deletes. The primary key gains the
-- partition column (PostgreSQL requirement); the dedup unique pair
-- (dedup_key, window_bucket) moves to PER-PARTITION unique indexes — a
-- partitioned parent cannot carry it without the partition key. A bucket
-- spanning a month boundary can therefore record one duplicate; documented
-- in ADR 0023, over-recording is acceptable, gaps are not.
--
-- A DEFAULT partition catches rows outside every maintained range, so a
-- lagging maintenance job can never make classified reads fail (the trail's
-- hard-failure semantics would otherwise turn an ops miss into an outage).
ALTER TABLE "read_events" RENAME TO "read_events_old";
ALTER INDEX "read_events_pkey" RENAME TO "read_events_old_pkey";
ALTER INDEX "read_events_dedup_key_window_bucket_key" RENAME TO "read_events_old_dedup_key";
ALTER INDEX "read_events_page_id_occurred_at_idx" RENAME TO "read_events_old_page_idx";
ALTER INDEX "read_events_actor_id_occurred_at_idx" RENAME TO "read_events_old_actor_idx";
ALTER INDEX "read_events_occurred_at_idx" RENAME TO "read_events_old_at_idx";
CREATE TABLE "read_events" (
"id" TEXT NOT NULL,
"occurred_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"actor_id" TEXT,
"session_key" TEXT NOT NULL,
"page_id" TEXT,
"pond_id" TEXT NOT NULL,
"channel" TEXT NOT NULL,
"classification" TEXT NOT NULL,
"details" JSONB,
"dedup_key" TEXT NOT NULL,
"window_bucket" BIGINT NOT NULL,
"window_seconds" INTEGER NOT NULL,
CONSTRAINT "read_events_pkey" PRIMARY KEY ("id", "occurred_at")
) PARTITION BY RANGE ("occurred_at");
-- Non-unique parent indexes propagate to every partition automatically.
CREATE INDEX "read_events_page_id_occurred_at_idx" ON "read_events"("page_id", "occurred_at");
CREATE INDEX "read_events_actor_id_occurred_at_idx" ON "read_events"("actor_id", "occurred_at");
CREATE INDEX "read_events_occurred_at_idx" ON "read_events"("occurred_at");
-- The safety-net partition, plus the current and the next month — the daily
-- maintenance job (read-trail-maintenance) keeps creating months ahead and
-- adds the same per-partition dedup index to each new one.
CREATE TABLE "read_events_default" PARTITION OF "read_events" DEFAULT;
CREATE UNIQUE INDEX "read_events_default_dedup_key"
ON "read_events_default"("dedup_key", "window_bucket");
DO $$
DECLARE
m DATE;
part TEXT;
BEGIN
FOR i IN 0..1 LOOP
m := date_trunc('month', now())::date + (i || ' month')::interval;
part := 'read_events_y' || to_char(m, 'YYYY') || 'm' || to_char(m, 'MM');
EXECUTE format(
'CREATE TABLE %I PARTITION OF "read_events" FOR VALUES FROM (%L) TO (%L)',
part, m, m + interval '1 month');
EXECUTE format(
'CREATE UNIQUE INDEX %I ON %I ("dedup_key", "window_bucket")',
part || '_dedup_key', part);
END LOOP;
END $$;
INSERT INTO "read_events"
("id", "occurred_at", "actor_id", "session_key", "page_id", "pond_id",
"channel", "classification", "details", "dedup_key", "window_bucket",
"window_seconds")
SELECT "id", "occurred_at", "actor_id", "session_key", "page_id", "pond_id",
"channel", "classification", "details", "dedup_key", "window_bucket",
"window_seconds"
FROM "read_events_old";
DROP TABLE "read_events_old";

View File

@ -96,8 +96,13 @@ model AuditEntry {
/// volume, purpose and legal basis all differ. Deliberately WITHOUT foreign
/// keys: evidence must survive a page purge and a hard user deletion — the
/// ids stay as recorded (pseudonymous uuids), history is never rewritten.
///
/// In migrated databases the table is RANGE-partitioned by `occurred_at`
/// (monthly, issue #224) — hence the composite id. The dedup unique pair
/// lives per partition there (a partitioned parent cannot carry it without
/// the partition key); `db push` test databases get it on the plain table.
model ReadEvent {
id String @id @default(uuid())
id String @default(uuid())
occurredAt DateTime @default(now()) @map("occurred_at")
/// Null = anonymous reader (public grant); `sessionKey` still names the
/// browsing session, so the anonymous marker is explicit, not an accident.
@ -115,7 +120,17 @@ model ReadEvent {
/// rewrite history (ADR 0023).
classification String
details Json?
/// Dedup window (issue #223): `<sessionKey>:<pageId|->:<channel>` plus the
/// aligned bucket `floor(epoch / windowSeconds)`. The unique pair makes
/// concurrent duplicate reads collapse race-free (insert or P2002-skip).
dedupKey String @map("dedup_key")
windowBucket BigInt @map("window_bucket")
/// Window length the event was recorded under — the row itself states it
/// represents up to this many seconds, so the evidence is not overread.
windowSeconds Int @map("window_seconds")
@@id([id, occurredAt])
@@unique([dedupKey, windowBucket])
@@index([pageId, occurredAt])
@@index([actorId, occurredAt])
@@index([occurredAt])

View File

@ -1,9 +1,12 @@
import { Controller, Get, Param, Post, Query, Req, UseGuards } from '@nestjs/common';
import {
auditListQuerySchema,
readEventListQuerySchema,
type AuditListQuery,
type AuditListView,
type JobTriggerResult,
type ReadEventListQuery,
type ReadEventListView,
type StorageOverviewView,
type SystemBackupView,
type SystemJobView,
@ -45,6 +48,15 @@ export class SystemAdminController {
return this.system.auditLog(query);
}
/** Read-access trail queries (issue #224): "who read page X", "what did
* user Y read" Site-Admin only, like the audit viewer above. */
@Get('read-events')
async readEvents(
@Query(new ZodValidationPipe(readEventListQuerySchema)) query: ReadEventListQuery,
): Promise<ReadEventListView> {
return this.system.readEvents(query);
}
@Get('storage')
async storage(): Promise<StorageOverviewView> {
return this.system.storage();

View File

@ -7,10 +7,13 @@ import {
AUDIT_PAGE_SIZE,
BACKUP_FRESH_MAX_AGE_HOURS,
BACKUP_STATUS_FILE,
READ_EVENT_PAGE_SIZE,
type AuditListQuery,
type AuditListView,
type BackupStatus,
type JobTriggerResult,
type ReadEventListQuery,
type ReadEventListView,
type StorageOverviewView,
type SystemBackupView,
type SystemJobView,
@ -160,6 +163,66 @@ export class SystemAdminService {
};
}
/**
* The Site-Admin query path over the read-access trail (issue #224,
* ADR 0023) evidence nobody can read is not evidence. Answers "who read
* page X" and "what did user Y read" within a period. API-only by design
* (no panel yet): the trail is an examiner's tool, not a daily screen
* documented in data-model.md §read_events.
*/
async readEvents(query: ReadEventListQuery): Promise<ReadEventListView> {
const where: Prisma.ReadEventWhereInput = {};
if (query.pageId) where.pageId = query.pageId;
if (query.actor) {
const actor = await this.prisma.user.findUnique({ where: { username: query.actor } });
// An unknown username matches nothing rather than everything.
where.actorId = actor?.id ?? '00000000-0000-0000-0000-000000000000';
}
if (query.channel) where.channel = query.channel;
if (query.from || query.to) {
where.occurredAt = {
...(query.from ? { gte: query.from } : {}),
...(query.to ? { lte: query.to } : {}),
};
}
const total = await this.prisma.readEvent.count({ where });
const pageCount = Math.max(1, Math.ceil(total / READ_EVENT_PAGE_SIZE));
const page = Math.min(query.page, pageCount);
const events = await this.prisma.readEvent.findMany({
where,
orderBy: { occurredAt: 'desc' },
skip: (page - 1) * READ_EVENT_PAGE_SIZE,
take: READ_EVENT_PAGE_SIZE,
});
// No FK on actor_id (evidence outlives accounts) — resolve what still
// exists in one query, show the bare id otherwise.
const actorIds = [...new Set(events.map((e) => e.actorId).filter((id): id is string => !!id))];
const actors = actorIds.length
? await this.prisma.user.findMany({
where: { id: { in: actorIds } },
select: { id: true, username: true, displayName: true },
})
: [];
const actorById = new Map(actors.map((a) => [a.id, a]));
return {
entries: events.map((event) => ({
id: event.id,
occurredAt: event.occurredAt.toISOString(),
actor: event.actorId ? (actorById.get(event.actorId) ?? null) : null,
pageId: event.pageId,
pondId: event.pondId,
channel: event.channel,
classification: event.classification,
windowSeconds: event.windowSeconds,
details: (event.details as Record<string, unknown> | null) ?? null,
})),
page,
pageCount,
total,
};
}
async storage(): Promise<StorageOverviewView> {
const usages = await this.prisma.pondUsage.findMany({
where: { pond: { deletedAt: null } },

View File

@ -40,6 +40,7 @@ export const AUDIT_EVENTS = {
'pond.purged': { severity: 'notice' },
'quota.override_cleared': { severity: 'notice' },
'quota.override_set': { severity: 'notice' },
'read_trail.pruned': { severity: 'info' },
'settings.changed': { severity: 'notice' },
'setup.admin_created': { severity: 'notice' },
'setup.completed': { severity: 'info' },

View File

@ -18,11 +18,12 @@ import { AUDIT_EVENTS } from './audit-actions';
const doc = readFileSync(join(__dirname, '../../../../docs/architecture/audit-events.md'), 'utf8');
/** Event rows are `| \`ns.event\` | trigger | severity | ` the dot in the
* id keeps field-set rows (`msg`, `severity`, ) out of the match. */
* id keeps field-set rows (`msg`, `severity`, ) out of the match. The
* namespace may carry an underscore since `read_trail.*` (issue #224). */
function documentedEvents(): Map<string, string> {
const events = new Map<string, string>();
for (const line of doc.split('\n')) {
const id = /^\| `([a-z]+\.[a-z_]+)` +\|/.exec(line)?.[1];
const id = /^\| `([a-z_]+\.[a-z_]+)` +\|/.exec(line)?.[1];
if (!id) continue;
const cells = line.split('|').map((cell) => cell.trim());
// cells[0] is the empty string before the leading pipe.

View File

@ -0,0 +1,181 @@
import { INestApplication } from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import request from 'supertest';
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
import { ClockService } from '../common/clock.service';
import { InstanceSettingsService } from '../settings/instance-settings.service';
import { createTestApp, sessionCookieOf } from '../testing/test-app';
import { createTestPrisma, grantOwnerAdmin, hasTestDb, uniqueSuffix } from '../testing/test-db';
import { UsersService } from '../users/users.service';
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
/**
* The read-trail dedup window (issue #223, ADR 0023): one event per
* (session, page, channel) within an aligned `readTrail.dedupWindowMinutes`
* window repeats and reconnect-style re-requests collapse, a new session
* or a new window does not, and the recorded row states the window length
* it represents.
*/
describe.skipIf(!hasTestDb)('read-trail dedup window (e2e, issue #223)', () => {
let app: INestApplication;
let prisma: PrismaClient;
const suffix = uniqueSuffix();
const password = 'dedup fenster zeugen 123';
let ownerId: string;
let ownerCookie: string;
let pondId: string;
let classifiedId: string;
const api = () => request(app.getHttpServer());
const login = async () =>
sessionCookieOf(
await api()
.post('/api/v1/auth/login')
.send({ usernameOrEmail: `dedup-owner-${suffix}`, password })
.expect(200),
);
const events = () =>
prisma.readEvent.findMany({ where: { pondId }, orderBy: { occurredAt: 'asc' } });
beforeAll(async () => {
prisma = createTestPrisma();
await prisma.rateLimit.deleteMany({});
app = await createTestApp();
const users = app.get(UsersService);
const owner = await users.createUser({
username: `dedup-owner-${suffix}`,
email: `dedup-owner-${suffix}@example.test`,
displayName: 'Dedup Owner',
password,
locale: 'en',
});
await users.markEmailVerified(owner.id);
ownerId = owner.id;
// The trail ships OFF by default (#225) — this suite needs it on.
await app.get(InstanceSettingsService).set('readTrail.enabled', true, ownerId);
ownerCookie = await login();
const pond = await prisma.pond.create({
data: { slug: `dedup-pond-${suffix}`, name: 'Dedup Pond', type: 'SHARED', ownerId },
});
pondId = pond.id;
await grantOwnerAdmin(prisma, pondId, ownerId);
const page = await prisma.page.create({
data: {
pondId,
slug: `classified-${suffix}`,
title: 'Classified',
classification: 'VS_NFD',
createdBy: ownerId,
sortKey: 'a0',
ydocState: new Uint8Array(),
contentCache: {
create: { plainText: 'x', markdown: 'x', html: '<p>x</p>', outline: [] },
},
},
});
classifiedId = page.id;
});
afterEach(async () => {
await prisma.readEvent.deleteMany({ where: { pondId } });
await prisma.rateLimit.deleteMany({});
});
afterAll(async () => {
await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.enabled' } });
await prisma.readEvent.deleteMany({ where: { pondId } });
await prisma.attachment.deleteMany({ where: { pondId } });
await prisma.roleGrant.deleteMany({ where: { pondId } });
await prisma.pageContentCache.deleteMany({ where: { page: { pondId } } });
await prisma.page.deleteMany({ where: { pondId } });
await prisma.pond.deleteMany({ where: { id: pondId } });
await prisma.user.deleteMany({ where: { id: ownerId } });
await prisma.$disconnect();
await app.close();
});
it('collapses repeated reads of one page in one session+channel to a single event that names its window', async () => {
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
const rows = await events();
expect(rows).toHaveLength(1);
// The row itself states that it represents a window, not a request.
expect(rows[0]!.windowSeconds).toBe(5 * 60);
expect(rows[0]!.dedupKey).toBe(`${rows[0]!.sessionKey}:${classifiedId}:page_view`);
});
it('keeps channels apart: the same session reading and joining collab yields one event each', async () => {
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
await api()
.get(`/api/v1/pages/${classifiedId}/collab-token`)
.set('Cookie', ownerCookie)
.expect(200);
const rows = await events();
expect(rows.map((r) => r.channel).sort()).toEqual(['collab_join', 'page_view']);
});
it('a new session records again even for the same user — a reconnect within the session does not', async () => {
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
// Same session, later request ("reconnect"): deduped.
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
expect(await events()).toHaveLength(1);
const secondCookie = await login();
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', secondCookie).expect(200);
const rows = await events();
expect(rows).toHaveLength(2);
expect(new Set(rows.map((r) => r.sessionKey)).size).toBe(2);
expect(rows.every((r) => r.actorId === ownerId)).toBe(true);
});
it('bounds a live editing session: 30 collab-token renewals in one window are one event', async () => {
for (let i = 0; i < 30; i += 1) {
await api()
.get(`/api/v1/pages/${classifiedId}/collab-token`)
.set('Cookie', ownerCookie)
.expect(200);
}
const rows = await events();
expect(rows).toHaveLength(1);
expect(rows[0]!.channel).toBe('collab_join');
});
it('opens a new window when the clock moves past the bucket boundary', async () => {
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
const clock = app.get(ClockService);
const later = new Date(Date.now() + 10 * 60 * 1000);
const spy = vi.spyOn(clock, 'now').mockReturnValue(later);
try {
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
} finally {
spy.mockRestore();
}
expect(await events()).toHaveLength(2);
});
it('dedups page-less attachment downloads on the placeholder page key', async () => {
// A pond-level upload has no page; its effective classification falls
// back to the pond maximum (#212) and the dedup key carries `-`.
const uploaded = await api()
.post(`/api/v1/ponds/${pondId}/files`)
.set('Cookie', ownerCookie)
.attach('file', Buffer.concat([PNG_SIGNATURE, Buffer.from('img')]), 'a.png')
.expect(201);
await prisma.readEvent.deleteMany({ where: { pondId } });
await api().get(`/api/v1/media/${uploaded.body.id}`).set('Cookie', ownerCookie).expect(200);
await api().get(`/api/v1/media/${uploaded.body.id}`).set('Cookie', ownerCookie).expect(200);
const rows = await events();
expect(rows).toHaveLength(1);
expect(rows[0]!.channel).toBe('attachment');
expect(rows[0]!.pageId).toBeNull();
expect(rows[0]!.dedupKey).toBe(`${rows[0]!.sessionKey}:-:attachment`);
});
});

View File

@ -0,0 +1,139 @@
import { Injectable } from '@nestjs/common';
import { PinoLogger } from 'nestjs-pino';
import { AuditService } from '../audit/audit.service';
import { ClockService } from '../common/clock.service';
import { PrismaService } from '../prisma/prisma.service';
import { InstanceSettingsService } from '../settings/instance-settings.service';
const MS_PER_DAY = 24 * 60 * 60 * 1000;
/** How many months ahead of "now" a partition must exist. Two keeps a
* multi-week job outage from ever reaching an uncovered month (the DEFAULT
* partition would still catch it reads never fail on a missing month). */
const MONTHS_AHEAD = 2;
/**
* Read-trail storage maintenance (issue #224, ADR 0023), one daily job with
* two duties:
*
* 1. **Partition upkeep** `read_events` is RANGE-partitioned by month
* (migration `20260731170000`); this creates the next {@link MONTHS_AHEAD}
* monthly partitions, each with its per-partition dedup unique index
* (#223 the partitioned parent cannot carry it). A `db push` database
* (tests) has a plain table; partition work skips itself there.
* 2. **Retention** events older than `readTrail.retentionDays` are
* removed: whole months by dropping their partition (no scan), the
* remainder (default partition, plain tables) by a ranged delete. The
* deletion is audited (`read_trail.pruned`) so a gap in the evidence is
* always explainable same principle as `audit.pruned` (#196), but a
* deliberately separate period.
*/
@Injectable()
export class ReadTrailMaintenanceService {
constructor(
private readonly prisma: PrismaService,
private readonly settings: InstanceSettingsService,
private readonly audit: AuditService,
private readonly clock: ClockService,
private readonly logger: PinoLogger,
) {
this.logger.setContext(ReadTrailMaintenanceService.name);
}
async run(): Promise<void> {
await this.ensurePartitions();
await this.pruneExpired();
}
/** True when read_events is a partitioned parent (relkind `p`). */
private async isPartitioned(): Promise<boolean> {
const rows = await this.prisma.$queryRaw<{ relkind: string }[]>`
SELECT relkind::text FROM pg_class
WHERE relname = 'read_events' AND relnamespace = 'public'::regnamespace`;
return rows[0]?.relkind === 'p';
}
/** `read_events_y2026m08` for 2026-08. */
private partitionName(month: Date): string {
const y = month.getUTCFullYear();
const m = String(month.getUTCMonth() + 1).padStart(2, '0');
return `read_events_y${y}m${m}`;
}
private monthStart(base: Date, offsetMonths: number): Date {
return new Date(Date.UTC(base.getUTCFullYear(), base.getUTCMonth() + offsetMonths, 1));
}
async ensurePartitions(): Promise<void> {
if (!(await this.isPartitioned())) {
this.logger.debug('read_events is not partitioned here; skipping partition upkeep');
return;
}
const now = this.clock.now();
for (let offset = 0; offset <= MONTHS_AHEAD; offset += 1) {
const from = this.monthStart(now, offset);
const to = this.monthStart(now, offset + 1);
const name = this.partitionName(from);
try {
await this.prisma.$executeRawUnsafe(
`CREATE TABLE IF NOT EXISTS "${name}" PARTITION OF "read_events"
FOR VALUES FROM ('${from.toISOString()}') TO ('${to.toISOString()}')`,
);
await this.prisma.$executeRawUnsafe(
`CREATE UNIQUE INDEX IF NOT EXISTS "${name}_dedup_key"
ON "${name}" ("dedup_key", "window_bucket")`,
);
} catch (error) {
// Most likely: the DEFAULT partition already holds rows of this month
// (the job lagged past a month boundary). Nothing is lost — those
// rows live in the default partition and age out through the ranged
// delete below; the month just cannot get its own partition anymore.
this.logger.warn({ partition: name, err: error }, 'read-trail partition not created');
}
}
}
async pruneExpired(): Promise<number> {
const retentionDays = await this.settings.get('readTrail.retentionDays');
const cutoff = new Date(this.clock.now().getTime() - retentionDays * MS_PER_DAY);
let dropped = 0;
if (await this.isPartitioned()) {
// Whole months strictly before the cutoff month go by DROP — no scan,
// and the dropped range is exact (every row in them is < cutoff).
const partitions = await this.prisma.$queryRaw<{ relname: string }[]>`
SELECT c.relname::text
FROM pg_inherits i
JOIN pg_class c ON c.oid = i.inhrelid
WHERE i.inhparent = 'read_events'::regclass
AND c.relname ~ '^read_events_y[0-9]{4}m[0-9]{2}$'`;
const cutoffMonth = this.monthStart(cutoff, 0);
for (const { relname } of partitions) {
const match = /^read_events_y(\d{4})m(\d{2})$/.exec(relname);
if (!match) continue;
const monthEnd = new Date(Date.UTC(Number(match[1]), Number(match[2]), 1));
if (monthEnd.getTime() > cutoffMonth.getTime()) continue;
const counted = await this.prisma.$queryRawUnsafe<{ count: bigint }[]>(
`SELECT count(*)::bigint AS count FROM "${relname}"`,
);
dropped += Number(counted[0]?.count ?? 0n);
await this.prisma.$executeRawUnsafe(`DROP TABLE "${relname}"`);
}
}
// The remainder: rows before the cutoff inside surviving partitions,
// the default partition, or a plain (test) table.
const deleted = await this.prisma.readEvent.deleteMany({
where: { occurredAt: { lt: cutoff } },
});
const count = dropped + deleted.count;
if (count > 0) {
await this.audit.record({
action: 'read_trail.pruned',
details: { count, cutoff: cutoff.toISOString(), retentionDays },
});
}
return count;
}
}

View File

@ -0,0 +1,308 @@
import { execFileSync } from 'node:child_process';
import { join } from 'node:path';
import { INestApplication } from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import { PinoLogger } from 'nestjs-pino';
import request from 'supertest';
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import { ClockService } from '../common/clock.service';
import { InstanceSettingsService } from '../settings/instance-settings.service';
import { createTestApp, sessionCookieOf } from '../testing/test-app';
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
import { UsersService } from '../users/users.service';
import { ReadTrailMaintenanceService } from './read-trail-maintenance.service';
const API_ROOT = join(__dirname, '..', '..');
const MS_PER_DAY = 24 * 60 * 60 * 1000;
/**
* Read-trail storage (issue #224, ADR 0023): its own retention period with
* an audited deletion, the Site-Admin query path, and against a freshly
* migrated database, where the real DDL applies the monthly RANGE
* partitioning with per-partition dedup indexes and DROP-based pruning.
*/
describe.skipIf(!hasTestDb)('read-trail storage (e2e, issue #224)', () => {
const suffix = uniqueSuffix();
describe('retention and admin query path (shared database)', () => {
let app: INestApplication;
let prisma: PrismaClient;
const password = 'lesetrail speicher 123';
let adminId: string;
let adminCookie: string;
let readerCookie: string;
// Plain text ids — read_events has no FKs by design (#222).
const pondId = `pond-${suffix}`;
const api = () => request(app.getHttpServer());
async function makeUser(handle: string, siteAdmin: boolean) {
const users = app.get(UsersService);
const user = await users.createUser({
username: `${handle}-${suffix}`,
email: `${handle}-${suffix}@example.test`,
displayName: handle,
password,
locale: 'en',
});
await users.markEmailVerified(user.id);
if (siteAdmin) {
await prisma.user.update({ where: { id: user.id }, data: { isSiteAdmin: true } });
}
const cookie = sessionCookieOf(
await api()
.post('/api/v1/auth/login')
.send({ usernameOrEmail: `${handle}-${suffix}`, password })
.expect(200),
);
return { id: user.id, cookie };
}
function eventRow(overrides: Record<string, unknown>) {
return {
actorId: null,
sessionKey: 'anon',
pageId: null,
pondId,
channel: 'page_view',
classification: 'vs_nfd',
dedupKey: `k-${suffix}-${Math.random().toString(36).slice(2)}`,
windowBucket: BigInt(Math.floor(Math.random() * 1_000_000_000)),
windowSeconds: 300,
...overrides,
};
}
beforeAll(async () => {
prisma = createTestPrisma();
await prisma.rateLimit.deleteMany({});
app = await createTestApp();
const admin = await makeUser('storage-admin', true);
adminId = admin.id;
adminCookie = admin.cookie;
const reader = await makeUser('storage-reader', false);
readerCookie = reader.cookie;
});
afterAll(async () => {
await prisma.readEvent.deleteMany({ where: { pondId } });
await prisma.auditEntry.deleteMany({ where: { action: 'read_trail.pruned' } });
await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.retentionDays' } });
await prisma.user.deleteMany({ where: { username: { contains: suffix } } });
await prisma.$disconnect();
await app.close();
});
it('prunes only events past its own period and audits the deletion', async () => {
await app.get(InstanceSettingsService).set('readTrail.retentionDays', 30, adminId);
const now = Date.now();
await prisma.readEvent.createMany({
data: [
eventRow({ occurredAt: new Date(now - 40 * MS_PER_DAY) }),
eventRow({ occurredAt: new Date(now - 31 * MS_PER_DAY) }),
eventRow({ occurredAt: new Date(now - 5 * MS_PER_DAY) }),
],
});
const maintenance = app.get(ReadTrailMaintenanceService);
const pruned = await maintenance.pruneExpired();
expect(pruned).toBeGreaterThanOrEqual(2);
const remaining = await prisma.readEvent.findMany({ where: { pondId } });
expect(remaining).toHaveLength(1);
const audit = await prisma.auditEntry.findFirst({
where: { action: 'read_trail.pruned' },
orderBy: { at: 'desc' },
});
expect(audit).not.toBeNull();
expect(audit!.details).toMatchObject({ retentionDays: 30 });
});
it('answers "who read page X" and "what did user Y read" for Site Admins only', async () => {
const pageId = `page-${suffix}`;
await prisma.readEvent.createMany({
data: [
eventRow({ pageId, actorId: adminId, channel: 'export' }),
eventRow({ pageId: `other-${suffix}`, actorId: null }),
],
});
const byPage = await api()
.get(`/api/v1/admin/system/read-events?pageId=${pageId}`)
.set('Cookie', adminCookie)
.expect(200);
expect(byPage.body.total).toBe(1);
expect(byPage.body.entries[0]).toMatchObject({
pageId,
channel: 'export',
windowSeconds: 300,
});
expect(byPage.body.entries[0].actor).toMatchObject({ id: adminId });
const byActor = await api()
.get(`/api/v1/admin/system/read-events?actor=storage-admin-${suffix}`)
.set('Cookie', adminCookie)
.expect(200);
expect(byActor.body.total).toBe(1);
// An unknown username matches nothing rather than everything.
const unknown = await api()
.get(`/api/v1/admin/system/read-events?actor=nobody-${suffix}`)
.set('Cookie', adminCookie)
.expect(200);
expect(unknown.body.total).toBe(0);
await api()
.get(`/api/v1/admin/system/read-events?pageId=${pageId}`)
.set('Cookie', readerCookie)
.expect(403);
});
});
describe('partitioned shape (fresh database, real migrations)', () => {
const baseUrl = process.env.TEST_DATABASE_URL!;
const dbName = `dorfteich_trail_${suffix}`;
let fresh: PrismaClient;
let maintenance: ReadTrailMaintenanceService;
const auditRecord = vi.fn().mockResolvedValue(undefined);
let retentionDays = 365;
function freshUrl(): string {
const url = new URL(baseUrl);
url.pathname = `/${dbName}`;
return url.toString();
}
beforeAll(async () => {
const admin = new PrismaClient({ datasourceUrl: baseUrl });
try {
await admin.$executeRawUnsafe(`CREATE DATABASE "${dbName}"`);
} finally {
await admin.$disconnect();
}
execFileSync(
process.execPath,
[join(API_ROOT, 'node_modules', 'prisma', 'build', 'index.js'), 'migrate', 'deploy'],
{ env: { ...process.env, DATABASE_URL: freshUrl() }, stdio: 'pipe', cwd: API_ROOT },
);
fresh = new PrismaClient({ datasourceUrl: freshUrl() });
const settingsStub = {
get: async () => retentionDays,
} as unknown as InstanceSettingsService;
const auditStub = { record: auditRecord } as never;
const loggerStub = {
setContext: () => undefined,
debug: () => undefined,
info: () => undefined,
warn: () => undefined,
error: () => undefined,
} as unknown as PinoLogger;
maintenance = new ReadTrailMaintenanceService(
fresh as never,
settingsStub,
auditStub,
new ClockService(),
loggerStub,
);
}, 120_000);
afterAll(async () => {
await fresh.$disconnect();
const admin = new PrismaClient({ datasourceUrl: baseUrl });
try {
await admin.$executeRawUnsafe(`DROP DATABASE IF EXISTS "${dbName}" WITH (FORCE)`);
} finally {
await admin.$disconnect();
}
});
it('migrates read_events to a partitioned parent with current-month coverage', async () => {
const kind = await fresh.$queryRaw<{ relkind: string }[]>`
SELECT relkind::text FROM pg_class WHERE relname = 'read_events'`;
expect(kind[0]!.relkind).toBe('p');
const partitions = await fresh.$queryRaw<{ relname: string }[]>`
SELECT c.relname::text FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
WHERE i.inhparent = 'read_events'::regclass ORDER BY c.relname`;
const names = partitions.map((p) => p.relname);
expect(names).toContain('read_events_default');
expect(names.some((n) => /^read_events_y\d{4}m\d{2}$/.test(n))).toBe(true);
});
it('keeps the dedup unique pair enforced per partition (P2002 on the duplicate)', async () => {
const row = {
actorId: null,
sessionKey: 'anon',
pageId: null,
pondId: 'pond-part',
channel: 'page_view',
classification: 'vs_nfd',
dedupKey: `dup-${suffix}`,
windowBucket: 42n,
windowSeconds: 300,
};
await fresh.readEvent.create({ data: row });
await expect(fresh.readEvent.create({ data: { ...row } })).rejects.toMatchObject({
code: 'P2002',
});
});
it('creates months ahead and prunes whole expired partitions by DROP, audited', async () => {
await maintenance.ensurePartitions();
const next = new Date();
const nextMonth = new Date(Date.UTC(next.getUTCFullYear(), next.getUTCMonth() + 2, 1));
const nextName = `read_events_y${nextMonth.getUTCFullYear()}m${String(
nextMonth.getUTCMonth() + 1,
).padStart(2, '0')}`;
const created = await fresh.$queryRawUnsafe<{ relname: string }[]>(
`SELECT relname::text FROM pg_class WHERE relname = '${nextName}'`,
);
expect(created).toHaveLength(1);
// ...and each new partition carries its own dedup unique index.
const index = await fresh.$queryRawUnsafe<{ indexname: string }[]>(
`SELECT indexname::text FROM pg_indexes WHERE tablename = '${nextName}'
AND indexname = '${nextName}_dedup_key'`,
);
expect(index).toHaveLength(1);
// An old month: partition + one event well past retention.
await fresh.$executeRawUnsafe(
`CREATE TABLE "read_events_y2020m01" PARTITION OF "read_events"
FOR VALUES FROM ('2020-01-01') TO ('2020-02-01')`,
);
await fresh.readEvent.create({
data: {
occurredAt: new Date('2020-01-15T12:00:00Z'),
actorId: null,
sessionKey: 'anon',
pageId: null,
pondId: 'pond-part',
channel: 'export',
classification: 'vs_nfd',
dedupKey: `old-${suffix}`,
windowBucket: 1n,
windowSeconds: 300,
},
});
const pruned = await maintenance.pruneExpired();
expect(pruned).toBeGreaterThanOrEqual(1);
const gone = await fresh.$queryRawUnsafe<{ relname: string }[]>(
`SELECT relname::text FROM pg_class WHERE relname = 'read_events_y2020m01'`,
);
expect(gone).toHaveLength(0);
expect(auditRecord).toHaveBeenCalledWith(
expect.objectContaining({ action: 'read_trail.pruned' }),
);
// The current month's events survive.
const kept = await fresh.readEvent.count({ where: { dedupKey: `dup-${suffix}` } });
expect(kept).toBe(1);
retentionDays = 365; // restore for any later use
});
});
});

View File

@ -0,0 +1,132 @@
import { INestApplication } from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import { PinoLogger } from 'nestjs-pino';
import request from 'supertest';
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import { InstanceSettingsService } from '../settings/instance-settings.service';
import { createTestApp, sessionCookieOf } from '../testing/test-app';
import { createTestPrisma, grantOwnerAdmin, hasTestDb, uniqueSuffix } from '../testing/test-db';
import { UsersService } from '../users/users.service';
import { ReadTrailService } from './read-trail.service';
/**
* The read-trail master switch (issue #225, ADR 0023): OFF is the default
* and means no event is written ANYWHERE no row, no stdout line; ON
* restores the full #222 semantics. The switch position is announced so a
* silent trail is never ambiguous.
*/
describe.skipIf(!hasTestDb)('read-trail switch (e2e, issue #225)', () => {
let app: INestApplication;
let prisma: PrismaClient;
const suffix = uniqueSuffix();
const password = 'schalter zeugen 123';
let ownerId: string;
let ownerCookie: string;
let pondId: string;
let classifiedId: string;
const api = () => request(app.getHttpServer());
beforeAll(async () => {
prisma = createTestPrisma();
await prisma.rateLimit.deleteMany({});
// No leftover switch row: this suite tests the DEFAULT (off).
await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.enabled' } });
app = await createTestApp();
const users = app.get(UsersService);
const owner = await users.createUser({
username: `switch-owner-${suffix}`,
email: `switch-owner-${suffix}@example.test`,
displayName: 'Switch Owner',
password,
locale: 'en',
});
await users.markEmailVerified(owner.id);
ownerId = owner.id;
ownerCookie = sessionCookieOf(
await api()
.post('/api/v1/auth/login')
.send({ usernameOrEmail: `switch-owner-${suffix}`, password })
.expect(200),
);
const pond = await prisma.pond.create({
data: { slug: `switch-pond-${suffix}`, name: 'Switch Pond', type: 'SHARED', ownerId },
});
pondId = pond.id;
await grantOwnerAdmin(prisma, pondId, ownerId);
const page = await prisma.page.create({
data: {
pondId,
slug: `classified-${suffix}`,
title: 'Classified',
classification: 'VS_NFD',
createdBy: ownerId,
sortKey: 'a0',
ydocState: new Uint8Array(),
contentCache: {
create: { plainText: 'x', markdown: 'x', html: '<p>x</p>', outline: [] },
},
},
});
classifiedId = page.id;
});
afterAll(async () => {
await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.enabled' } });
await prisma.readEvent.deleteMany({ where: { pondId } });
await prisma.roleGrant.deleteMany({ where: { pondId } });
await prisma.pageContentCache.deleteMany({ where: { page: { pondId } } });
await prisma.page.deleteMany({ where: { pondId } });
await prisma.pond.deleteMany({ where: { id: pondId } });
await prisma.user.deleteMany({ where: { id: ownerId } });
await prisma.$disconnect();
await app.close();
});
it('is OFF by default: a classified read writes nothing — no row, no stdout line', async () => {
const trail = app.get(ReadTrailService);
const logger = (trail as unknown as { logger: PinoLogger }).logger;
const infoSpy = vi.spyOn(logger, 'info');
try {
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
} finally {
infoSpy.mockRestore();
}
expect(await prisma.readEvent.count({ where: { pondId } })).toBe(0);
expect(infoSpy).not.toHaveBeenCalled();
});
it('records again the moment the switch turns on', async () => {
await app.get(InstanceSettingsService).set('readTrail.enabled', true, ownerId);
try {
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
expect(await prisma.readEvent.count({ where: { pondId } })).toBe(1);
} finally {
await app.get(InstanceSettingsService).set('readTrail.enabled', false, ownerId);
}
});
it('announces the switch position so silence is never ambiguous', async () => {
const trail = app.get(ReadTrailService);
const logger = (trail as unknown as { logger: PinoLogger }).logger;
const warnSpy = vi.spyOn(logger, 'warn');
const infoSpy = vi.spyOn(logger, 'info');
try {
await trail.announceState(); // switch is off after the previous test
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('NOT evidenced'));
await app.get(InstanceSettingsService).set('readTrail.enabled', true, ownerId);
await trail.announceState();
expect(infoSpy).toHaveBeenCalledWith(expect.stringContaining('enabled'));
} finally {
warnSpy.mockRestore();
infoSpy.mockRestore();
await app.get(InstanceSettingsService).set('readTrail.enabled', false, ownerId);
}
});
});

View File

@ -74,6 +74,9 @@ describe.skipIf(!hasTestDb)('read-access trail (e2e, issue #222)', () => {
});
await users.markEmailVerified(owner.id);
ownerId = owner.id;
// The trail ships OFF by default (#225) — these suites test the
// instrumented channels, so they run with the switch on.
await app.get(InstanceSettingsService).set('readTrail.enabled', true, ownerId);
ownerCookie = sessionCookieOf(
await api()
.post('/api/v1/auth/login')
@ -131,7 +134,9 @@ describe.skipIf(!hasTestDb)('read-access trail (e2e, issue #222)', () => {
});
afterAll(async () => {
await prisma.instanceSetting.deleteMany({ where: { key: 'api.enabled' } });
await prisma.instanceSetting.deleteMany({
where: { key: { in: ['api.enabled', 'readTrail.enabled'] } },
});
await prisma.readEvent.deleteMany({ where: { pondId } });
await prisma.conversionJob.deleteMany({ where: { ownerId } });
await prisma.apiToken.deleteMany({ where: { userId: ownerId } });

View File

@ -1,7 +1,16 @@
import { Global, Module } from '@nestjs/common';
import { Global, Module, OnModuleInit } from '@nestjs/common';
import { CommonModule } from '../common/common.module';
import { SchedulerModule } from '../scheduler/scheduler.module';
import { SchedulerService } from '../scheduler/scheduler.service';
import { SettingsModule } from '../settings/settings.module';
import { ReadTrailMaintenanceService } from './read-trail-maintenance.service';
import { ReadTrailService } from './read-trail.service';
/** Daily, per operations.md's maintenance-jobs table (issue #224). */
const READ_TRAIL_MAINTENANCE_CADENCE_SECONDS = 24 * 60 * 60;
/**
* Global like AuditModule and for the same reason: the read-access trail
* (issue #222, ADR 0023) cuts across every module that serves page content
@ -9,7 +18,27 @@ import { ReadTrailService } from './read-trail.service';
*/
@Global()
@Module({
providers: [ReadTrailService],
exports: [ReadTrailService],
imports: [CommonModule, SchedulerModule, SettingsModule],
providers: [ReadTrailService, ReadTrailMaintenanceService],
exports: [ReadTrailService, ReadTrailMaintenanceService],
})
export class ReadTrailModule {}
export class ReadTrailModule implements OnModuleInit {
constructor(
private readonly scheduler: SchedulerService,
private readonly maintenance: ReadTrailMaintenanceService,
private readonly trail: ReadTrailService,
) {}
async onModuleInit(): Promise<void> {
this.scheduler.register({
name: 'read-trail-maintenance',
cadenceSeconds: READ_TRAIL_MAINTENANCE_CADENCE_SECONDS,
run: async () => {
await this.maintenance.run();
},
});
// One line per boot stating the switch position (issue #225) — a silent
// trail must never be ambiguous.
await this.trail.announceState();
}
}

View File

@ -2,7 +2,9 @@ import { Injectable } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { PinoLogger } from 'nestjs-pino';
import { ClockService } from '../common/clock.service';
import { PrismaService } from '../prisma/prisma.service';
import { InstanceSettingsService } from '../settings/instance-settings.service';
/**
* Every read surface classified content can leave through (issue #222,
@ -61,33 +63,83 @@ export interface ReadEventInput extends ReadActor {
* swallowed. A lost event is a gap in evidence, so a failed write aborts
* the read with the ordinary 500 the reader retries, the evidence stays
* complete (decision recorded in ADR 0023 and security.md).
*
* Dedup window (issue #223): one event per (session, page, channel) within
* an aligned window of `readTrail.dedupWindowMinutes` (default 5). Buckets
* are `floor(epoch / windowSeconds)`, and the unique
* (dedupKey, windowBucket) pair collapses concurrent duplicates race-free:
* the first insert wins, every later one lands on P2002 and is skipped
* a skipped DUPLICATE is not a gap, so it must not abort the read. The row
* carries `windowSeconds`, so each event states that it represents up to
* that many seconds of access, not a single request.
*/
@Injectable()
export class ReadTrailService {
constructor(
private readonly prisma: PrismaService,
private readonly settings: InstanceSettingsService,
private readonly clock: ClockService,
private readonly logger: PinoLogger,
) {
this.logger.setContext(ReadTrailService.name);
}
/** States the switch position once at startup (issue #225): a trail
* without events must be distinguishable from a disabled trail the log
* line makes the gap explainable either way. Must never fail the boot:
* healthz-only environments come up without a reachable database. */
async announceState(): Promise<void> {
let enabled: boolean;
try {
enabled = await this.settings.get('readTrail.enabled');
} catch {
this.logger.warn('read_trail: switch position unknown at startup (settings unavailable)');
return;
}
if (enabled) {
this.logger.info('read_trail: enabled — reads of classified pages are recorded');
} else {
this.logger.warn(
'read_trail: disabled — reads of classified pages are NOT evidenced (readTrail.enabled)',
);
}
}
async record(event: ReadEventInput): Promise<void> {
// The master switch (issue #225): off means nothing is written anywhere
// — no row, no stdout line. The startup announcement above is what keeps
// the resulting silence unambiguous.
if (!(await this.settings.get('readTrail.enabled'))) return;
const { actorId, sessionKey, pageId, pondId, channel, details } = event;
await this.prisma.readEvent.create({
data: {
actorId,
sessionKey,
pageId,
pondId,
channel,
classification: 'vs_nfd',
details: details ? (details as Prisma.InputJsonObject) : undefined,
},
});
const windowSeconds = (await this.settings.get('readTrail.dedupWindowMinutes')) * 60;
const dedupKey = `${sessionKey}:${pageId ?? '-'}:${channel}`;
const windowBucket = BigInt(Math.floor(this.clock.now().getTime() / 1000 / windowSeconds));
try {
await this.prisma.readEvent.create({
data: {
actorId,
sessionKey,
pageId,
pondId,
channel,
classification: 'vs_nfd',
details: details ? (details as Prisma.InputJsonObject) : undefined,
dedupKey,
windowBucket,
windowSeconds,
},
});
} catch (error) {
const isDuplicate =
error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002';
if (!isDuplicate) throw error;
this.logger.debug({ dedupKey, channel }, 'read_trail: deduped within window');
return;
}
// The stdout line mirrors the row (SIEM forwarding beyond this is out of
// scope, #224); it fires only after the row is safely persisted.
this.logger.info(
{ actor: actorId, sessionKey, pageId, pondId, channel },
{ actor: actorId, sessionKey, pageId, pondId, channel, windowSeconds },
'read_trail: classified page read',
);
}

View File

@ -51,6 +51,26 @@ export const INSTANCE_SETTINGS = {
// bounded. PENDING rows — including failed-but-retryable ones — are
// never touched; the retry loop owns them.
'mail.outboxRetentionDays': z.number().int().min(1).default(30),
// Read-trail master switch (issue #225, ADR 0023). Default OFF: read
// logging is employee monitoring in a works council's eyes — an ordinary
// instance must not surveil reads. The VS-NfD reference configuration
// (#227) turns it on together with the written purpose limitation
// (60-sicherheitsdokumentation.md §7). Off means NO event is written
// anywhere, including stdout; the api states the switch position once at
// startup, so a gap in the evidence is never ambiguous.
'readTrail.enabled': z.boolean().default(false),
// Read-trail dedup window (issue #223, ADR 0023): one event per
// (session, page, channel) within an aligned window of this many minutes.
// 5 minutes keeps a live Yjs session (collab tokens every 60 s) at a
// bounded ~12 events/hour/page while still evidencing distinct visits.
'readTrail.dedupWindowMinutes': z.number().int().min(1).default(5),
// Read-trail retention (issue #224): days a read event is kept before the
// daily maintenance job removes it — deliberately independent of
// `audit.retentionDays` (#196), because volume, purpose and legal basis
// differ. The deletion itself is audited (`read_trail.pruned`), so a gap
// is always explainable. One year mirrors the audit default; shortening
// it is an operator decision under the purpose limitation (#225).
'readTrail.retentionDays': z.number().int().min(1).default(365),
// Default VS-NfD classification for newly created pages (ADR 0022,
// issue #204). An instance operated inside a classified environment sets
// this to `vs_nfd` so nothing starts unmarked; inheritance from the

View File

@ -20,8 +20,9 @@ test('lists maintenance jobs and triggers one manually', async ({ browser }) =>
// Keep in sync with the scheduler registrations: trash-purge,
// version-thinning, page-compaction, data-export-purge,
// notification-digest, orphan-file-sweep (#194), audit-retention (#196),
// conversion-payload-prune (#233), mail-outbox-retention (#234).
await expect(jobsTable.locator('tbody tr')).toHaveCount(9);
// conversion-payload-prune (#233), mail-outbox-retention (#234),
// read-trail-maintenance (#224).
await expect(jobsTable.locator('tbody tr')).toHaveCount(10);
const firstRow = jobsTable.locator('tbody tr').first();
await firstRow.getByRole('button').click();

View File

@ -67,6 +67,61 @@ traffic per open document.
also the MCP `read_page` path), `job:<id>` (background builds such as the
account data export), `anon` (anonymous reader on a public grant).
## Decisions taken in #223
- **Aligned dedup windows.** One event per (session, page, channel) within
an aligned window of `readTrail.dedupWindowMinutes` (default 5):
buckets are `floor(epoch / windowSeconds)`, and a unique
(dedupKey, windowBucket) pair collapses concurrent duplicates race-free
at insert time. Sliding windows (measured from the first event) were
rejected — they need a read-before-write and lose the race-safety of the
plain unique insert. Consequence: two reads just across a bucket
boundary yield two events; over-recording is acceptable, gaps are not.
- **A skipped duplicate is not a gap** — the unique-violation path returns
quietly (debug log), only real write failures abort the read.
- **Each row states its window** (`windowSeconds`), so the evidence reads
as "accessed at least once in these N minutes", never as a request
count. What the trail can prove about a live session: the per-minute
collab-token renewals collapse to one `collab_join` event per window —
presence during the window, not activity within it.
- **Anonymous readers share one `anon` key**: all anonymous reads of a page
through one channel inside a window are one event. Deliberate — telling
anonymous browsers apart would require fingerprinting (IP/UA), which the
purpose limitation (#225) rules out.
## Decisions taken in #224
- **Monthly RANGE partitions** on `occurred_at`, plus a DEFAULT partition
as a safety net: a lagging maintenance job must never make classified
reads fail (the trail write is hard-failing — an ops miss must not
become an outage). Retention DROPs whole expired months without
scanning; the pruning run is audited (`read_trail.pruned`) under the
trail's own `readTrail.retentionDays` (default 365).
- **The dedup unique pair lives per partition** (PostgreSQL cannot carry
it on the parent without the partition key). A dedup bucket spanning a
month boundary can therefore record one duplicate — over-recording is
acceptable, gaps are not.
- **Query path is API-only** (`GET /admin/system/read-events`, Site
Admin): the trail is an examiner's tool, not a daily screen. Evidence
nobody can read is not evidence — the path exists; a panel does not.
- **Growth measured**: ~1 MB per 1000 events including indexes.
## Decisions taken in #225
- **Default OFF** (`readTrail.enabled`): read logging is employee
monitoring in a works council's eyes — an ordinary instance must not
surveil reads. The VS-NfD reference configuration turns it on together
with the written purpose limitation
(`docs/vs-nfd/60-sicherheitsdokumentation.md` §7, referenced from the
hardening guide).
- **Off means silence everywhere** — no row, no stdout line; the api
announces the switch position once per boot, so an eventless trail is
never ambiguous (a gap reads as "was off", not "was lost").
- The purpose limitation names what is recorded, why, who may read it
(Site Admin, API-only), for how long, and what it may NOT be used for
(no performance or behaviour monitoring). Its technical anchor is the
variant-A scope: unmarked content produces no trace.
## Consequences
- The scope limit is the feature's strongest argument in the works-council

View File

@ -1,7 +1,7 @@
# Audit event catalogue
**Catalogue version 1.1 (2026-07-31; 1.1 adds `page.classification_*`,
issue #205).**
**Catalogue version 1.2 (2026-07-31; 1.2 adds `read_trail.pruned`,
issue #224; 1.1 added `page.classification_*`, issue #205).**
This is the operator-facing contract for the audit trail: every event id
the application can emit, with its trigger, severity, actor/target
@ -103,11 +103,12 @@ failure), `warning` = feeds detection (suspicious or destructive),
### Content integrity & lifecycle (`file.*`, `pond.*`, `audit.*`)
| Id | Trigger | Severity | Actor | Target | Fields |
| ----------------------- | ------------------------------------------------------------------- | -------- | ---------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- |
| `file.integrity_failed` | Attachment download hash mismatch — fail-closed (issue #199) | critical | `null` (any downloader; detection) | `attachment` | `pondId`, `expected` (stored sha256), `actual` (computed sha256) |
| `pond.purged` | Pond irreversibly destroyed (manual or trash retention, issue #193) | notice | admin, `null` when retention-run | `pond` | `trigger` (`manual` \| `retention`) plus per-object-type deletion counts (e.g. `pages`, `attachments`, … — informational, keys may grow) |
| `audit.pruned` | Audit retention deleted rows past the period (issue #196) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` |
| Id | Trigger | Severity | Actor | Target | Fields |
| ----------------------- | -------------------------------------------------------------------- | -------- | ---------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- |
| `file.integrity_failed` | Attachment download hash mismatch — fail-closed (issue #199) | critical | `null` (any downloader; detection) | `attachment` | `pondId`, `expected` (stored sha256), `actual` (computed sha256) |
| `pond.purged` | Pond irreversibly destroyed (manual or trash retention, issue #193) | notice | admin, `null` when retention-run | `pond` | `trigger` (`manual` \| `retention`) plus per-object-type deletion counts (e.g. `pages`, `attachments`, … — informational, keys may grow) |
| `audit.pruned` | Audit retention deleted rows past the period (issue #196) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` |
| `read_trail.pruned` | Read-trail retention removed events past its own period (issue #224) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` |
### Public API (`api.*`)

View File

@ -186,6 +186,40 @@ Cached counters per pond: `storage_bytes_used`, `editor_count`,
- `jobs`: conversion jobs for import/export (ADR 0009) and maintenance jobs
(compaction, trash purge, quota reconciliation) with status + timestamps.
## Read-access trail (`read_events`, issues #222#224, ADR 0023)
One row per read of a `classification = vs_nfd` page per channel and dedup
window — separate from `audit_log` because volume, purpose and legal basis
all differ.
| Column | Notes |
| ---------------------------- | ------------------------------------------------------------------------------------ |
| `id`, `occurred_at` | composite PK (the partition key must be part of it) |
| `actor_id` | reader, null = anonymous; **no FK** — evidence outlives accounts |
| `session_key` | `session:<id>` / `token:<id>` / `job:<id>` / `anon` |
| `page_id`, `pond_id` | plain ids, **no FK** — evidence outlives purges |
| `channel` | `page_view` / `no_js_shell` / `public_api` / `attachment` / `export` / `collab_join` |
| `classification` | at read time; reclassification never rewrites history |
| `dedup_key`, `window_bucket` | unique pair per partition — race-free dedup (#223) |
| `window_seconds` | the row represents up to this many seconds of access |
| `details` | small context (export format, attachment id) — never content |
Storage shape (issue #224): RANGE-partitioned by month on `occurred_at`,
with a DEFAULT partition as safety net (a lagging maintenance job must
never make classified reads fail — the trail write is hard-failing by
design). The daily `read-trail-maintenance` job creates months ahead
(each with its per-partition dedup index) and applies the trail's own
retention `readTrail.retentionDays` (default 365): whole expired months
are DROPped without scanning, remainders deleted by range; every pruning
run is audited as `read_trail.pruned`. Growth, measured on PG 17:
**~1 MB per 1000 events** including indexes (≈970 bytes/row) — a
100-person instance at hundreds of classified reads/day stays in the tens
of MB per year.
Query path: `GET /admin/system/read-events` (Site Admin, filters
`pageId`, `actor`, `channel`, `from`/`to`) — deliberately API-only, no
panel: the trail is an examiner's tool, not a daily screen.
## Comments & notifications (later milestone)
- `comments`: `page_id`, `author_id`, `body` (Markdown), `anchor`

View File

@ -109,6 +109,7 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack.
| conversion payload prune | daily | null finished conversion jobs' raw bytes (#233) |
| mail outbox retention | daily | delete sent/failed outbox rows past period (#234) |
| mail outbox retry | every minute | e-mail delivery with backoff |
| read-trail maintenance | daily | partition months ahead + prune `read_events` (#224) |
Job outcomes are visible in the Site Admin UI (last run, status) — that
panel is the operator's single glance for instance health.

View File

@ -214,6 +214,11 @@ scan docker-archive:/image.tar -o cyclonedx-json` respectively
- **Read-access trail** (issue #222, ADR 0023): reads of pages with
`classification = vs_nfd` land as `read_events` rows — only classified
pages, which is what keeps the purpose limitation defensible (variant A).
Master switch `readTrail.enabled`, **default off** (#225): off means no
event is written anywhere, including stdout, and the api announces the
switch position once per boot so an eventless trail is never ambiguous.
The written purpose limitation lives in
`docs/vs-nfd/60-sicherheitsdokumentation.md` §7.
The instrumented channels, and the emission point of each:
- `page_view` — authenticated SPA state fetch (`GET /pages/:id`, the
by-slug variant), the rendered read view (`/read/...`), the public JSON

View File

@ -134,10 +134,10 @@ Protokolliert werden Lesezugriffe **ausschließlich** auf Seiten mit
- [x] Instrumentierung der Lesepfade: Seitenansicht, Public-API-GET,
Attachment-Download, Export, No-JS-Shell, Collab-WS-Join · 4 AT · #222
- [ ] Dedup-Fenster (eine Sitzung + eine Seite innerhalb N Minuten = ein
- [x] Dedup-Fenster (eine Sitzung + eine Seite innerhalb N Minuten = ein
Ereignis), sonst erzeugt Yjs-Sync eine Ereignisflut · 2 AT · #223
- [ ] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224
- [ ] Abschaltbar, Zweckbindung dokumentiert · 12 AT · #225
- [x] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224
- [x] Abschaltbar, Zweckbindung dokumentiert · 12 AT · #225
Vorteil über den Aufwand hinaus: Die Zweckbindung ist sauber begründbar
(„nur eingestufte Inhalte"), was die Personalrats-Diskussion beim Kunden

View File

@ -39,6 +39,9 @@ Settings-Cache ist in-process (operations.md).
| `upload.allowedExtensions` | nur das dienstlich Nötige (z. B. `pdf`) | Standardliste | jede zusätzliche Endung vergrößert die Menge nicht prüfbarer Binärformate im Bestand. Bilder sind davon unabhängig immer erlaubt (Magic-Byte-geprüft). |
| `backup.nextcloud.enabled` | `false` | `false` | „Backup nur lokal": kein Anwendungs-Upload von Restore-Sets zu Drittdiensten. Fernspiegel regelt ausschließlich die Deploy-Allowlist (1.2). |
| `trash.retentionDays`, `audit.retentionDays`, `conversion.payloadRetentionDays`, `mail.outboxRetentionDays` | Defaults (30/365/30/30) | ebd. | Aufbewahrung bewusst begrenzt; Verkürzung nach Betreiber-Löschkonzept zulässig (Betriebshandbuch §5). |
| `readTrail.enabled` | `true` | `false` | **explizit setzen** — der Lesetrail (#222#225) evidenziert Lesezugriffe auf eingestufte Seiten; Default aus, weil Lesebeobachtung mitbestimmungsrelevant ist. Einschalten NUR zusammen mit der Zweckbindung (Sicherheitsdokumentation §7); die api meldet die Schalterstellung beim Start. |
| `readTrail.dedupWindowMinutes` | Default (5) | `5` | Dedup-Fenster des Lesetrails (#223): je (Sitzung, Seite, Kanal) ein Ereignis pro Fenster — begrenzt die Ereignisflut einer Live-Sitzung auf ~12/h. Kleiner = feineres Protokoll und mehr Zeilen; Änderung mit dem Zweckbindungs-Dokument (#225) abstimmen. |
| `readTrail.retentionDays` | Default (365) | `365` | eigene Aufbewahrung des Lesetrails (#224), bewusst getrennt von `audit.retentionDays`; Löschläufe sind selbst auditiert (`read_trail.pruned`). Dauer mit der Zweckbindung (#225) und dem Betreiber-Löschkonzept abstimmen. |
| `legal.imprint`, `legal.privacyPolicy` | befüllt | leer | Betreiberpflicht; leere Seiten zeigen einen Warnbanner. |
### 1.2 Deploy-Konfiguration (`.env` / Compose — nur Plattformzugriff, bewusst nicht per Admin-UI)

View File

@ -181,7 +181,14 @@ Datenträger- und Transportschutz ist Plattformsache
### 3.5 Lesekanäle
Jeder Kanal, über den Seiteninhalt die Anwendung verlässt (zugleich die
Instrumentierungsliste für den Lesetrail, #222):
Instrumentierungsliste für den Lesetrail, #222). Der Lesetrail
protokolliert je (Sitzung, Seite, Kanal) **ein Ereignis pro
Dedup-Fenster** (`readTrail.dedupWindowMinutes`, Default 5 Minuten,
#223); jede Zeile trägt die Fensterlänge. Beweiswert damit: „mindestens
ein Zugriff in diesem Fenster" — **nicht** eine Zugriffszählung, und
beim Kanal Collab „Live-Verbindung bestand während des Fensters", nicht
einzelne Sync-Frames. Anonyme Leser teilen sich den Marker `anon`
(keine Fingerprinting-Unterscheidung, Zweckbindung #225):
| Kanal | Pfad | Rechteprüfung | Kennzeichnung (M26) |
| -------------------------- | -------------------------------- | -------------------------------------- | ----------------------------------- |
@ -275,3 +282,42 @@ HKDF-Zweckableitung (#188), SHA-256-Integritätshashes für Attachments
(#199, fail-closed beim Download). Keine Inhalts- oder
Backup-Verschlüsselung, kein eigenes Schlüsselmanagement — bewusst
(§52 VSA).
## 7 Lesetrail: Zweckbindung (Issue #225, ADR 0023)
Der Lesetrail ist abschaltbar (`readTrail.enabled`, **Default: aus**);
die Referenzkonfiguration (Härtungsleitfaden §1.1) schaltet ihn ein.
Aus heißt: es wird **nirgends** ein Ereignis geschrieben — keine
Tabellenzeile, keine Logzeile; die api meldet die Schalterstellung
einmal beim Start, damit ein ereignisloser Trail nie zweideutig ist.
Diese Zweckbindung ist Teil der Betriebsdokumentation und gegenüber
der Personalvertretung offenzulegen.
**Was aufgezeichnet wird:** je (Sitzung, Seite, Kanal) und Dedup-Fenster
(§3.5) ein Ereignis mit Zeitstempel, Konto-Id (oder dem Marker `anon`),
Sitzungsschlüssel, Seiten- und Teich-Id, Kanal und der Einstufung zum
Lesezeitpunkt. **Kein** Seiteninhalt, **keine** Titel, **keine**
IP-Adressen, **kein** Fingerprinting anonymer Leser.
**Warum:** ausschließlich Beweissicherung für Lesezugriffe auf **als
VS-NfD gekennzeichnete** Inhalte (§52-konforme Nachvollziehbarkeit:
„welche eingestufte Seite wurde wann über welchen Kanal gelesen") —
Plattform-Logs kennen URLs, nicht Einstufungen. Zugriffe auf nicht
eingestufte Inhalte werden **nie** erfasst (Variante A, ADR 0023).
**Wer lesen darf:** ausschließlich Site-Admins über
`GET /admin/system/read-events` (kein UI-Panel — Prüfwerkzeug, kein
Alltagsbildschirm). Organisatorische Beschränkung auf den
Sicherheitsbeauftragten: Betreibersache (Betriebshandbuch §6
Rollentrennung).
**Wie lange:** `readTrail.retentionDays` (Default 365 Tage); Löschläufe
sind selbst auditiert (`read_trail.pruned`), sodass jede Lücke erklärbar
ist.
**Wofür NICHT:** keine Leistungs- oder Verhaltenskontrolle der
Beschäftigten, keine Auswertung von Arbeitsmustern, keine
Anwesenheitskontrolle, keine Weitergabe außerhalb des
Sicherheitsvorfalls- bzw. Prüfkontexts. Die Beschränkung auf
eingestufte Inhalte ist die technische Absicherung dieser Zusage: was
nicht gekennzeichnet ist, erzeugt keine Spur.

View File

@ -164,6 +164,49 @@ export interface AuditListView {
total: number;
}
/**
* Site-Admin query path over the read-access trail (issue #224, ADR 0023)
* evidence nobody can read is not evidence. Answers the two expected
* questions: "who read page X" (pageId) and "what did user Y read" (actor),
* both within a period.
*/
export const READ_EVENT_PAGE_SIZE = 50;
export const readEventListQuerySchema = z.object({
/** Page id the events belong to. */
pageId: z.string().trim().min(1).optional(),
/** Exact username of the reading user. */
actor: z.string().trim().min(1).optional(),
channel: z.string().trim().min(1).optional(),
from: z.coerce.date().optional(),
to: z.coerce.date().optional(),
page: z.coerce.number().int().min(1).default(1),
});
export type ReadEventListQuery = z.infer<typeof readEventListQuerySchema>;
export interface ReadEventView {
id: string;
occurredAt: string;
actor: AuditActorView | null;
/** Null for pond-level attachments (no page); the details name the file. */
pageId: string | null;
pondId: string;
channel: string;
classification: string;
/** Seconds the event's dedup window spans (#223) the row represents up
* to this much access time, not a single request. */
windowSeconds: number;
details: Record<string, unknown> | null;
}
export interface ReadEventListView {
entries: ReadEventView[];
page: number;
pageCount: number;
total: number;
}
export interface StoragePondView {
pondId: string;
name: string;