#225: read-trail master switch and purpose limitation #281

Merged
fable-5 merged 3 commits from issue-225-read-trail-switch into main 2026-07-31 12:53:37 +02:00
18 changed files with 750 additions and 18 deletions
Showing only changes of commit 2bdb0ec2cf - Show all commits

View File

@ -0,0 +1,76 @@
-- #224 (ADR 0023): convert read_events to monthly RANGE partitions on
-- occurred_at. Volume grows unbounded with use; retention then DROPs whole
-- expired partitions instead of scanning deletes. The primary key gains the
-- partition column (PostgreSQL requirement); the dedup unique pair
-- (dedup_key, window_bucket) moves to PER-PARTITION unique indexes — a
-- partitioned parent cannot carry it without the partition key. A bucket
-- spanning a month boundary can therefore record one duplicate; documented
-- in ADR 0023, over-recording is acceptable, gaps are not.
--
-- A DEFAULT partition catches rows outside every maintained range, so a
-- lagging maintenance job can never make classified reads fail (the trail's
-- hard-failure semantics would otherwise turn an ops miss into an outage).
ALTER TABLE "read_events" RENAME TO "read_events_old";
ALTER INDEX "read_events_pkey" RENAME TO "read_events_old_pkey";
ALTER INDEX "read_events_dedup_key_window_bucket_key" RENAME TO "read_events_old_dedup_key";
ALTER INDEX "read_events_page_id_occurred_at_idx" RENAME TO "read_events_old_page_idx";
ALTER INDEX "read_events_actor_id_occurred_at_idx" RENAME TO "read_events_old_actor_idx";
ALTER INDEX "read_events_occurred_at_idx" RENAME TO "read_events_old_at_idx";
CREATE TABLE "read_events" (
"id" TEXT NOT NULL,
"occurred_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"actor_id" TEXT,
"session_key" TEXT NOT NULL,
"page_id" TEXT,
"pond_id" TEXT NOT NULL,
"channel" TEXT NOT NULL,
"classification" TEXT NOT NULL,
"details" JSONB,
"dedup_key" TEXT NOT NULL,
"window_bucket" BIGINT NOT NULL,
"window_seconds" INTEGER NOT NULL,
CONSTRAINT "read_events_pkey" PRIMARY KEY ("id", "occurred_at")
) PARTITION BY RANGE ("occurred_at");
-- Non-unique parent indexes propagate to every partition automatically.
CREATE INDEX "read_events_page_id_occurred_at_idx" ON "read_events"("page_id", "occurred_at");
CREATE INDEX "read_events_actor_id_occurred_at_idx" ON "read_events"("actor_id", "occurred_at");
CREATE INDEX "read_events_occurred_at_idx" ON "read_events"("occurred_at");
-- The safety-net partition, plus the current and the next month — the daily
-- maintenance job (read-trail-maintenance) keeps creating months ahead and
-- adds the same per-partition dedup index to each new one.
CREATE TABLE "read_events_default" PARTITION OF "read_events" DEFAULT;
CREATE UNIQUE INDEX "read_events_default_dedup_key"
ON "read_events_default"("dedup_key", "window_bucket");
DO $$
DECLARE
m DATE;
part TEXT;
BEGIN
FOR i IN 0..1 LOOP
m := date_trunc('month', now())::date + (i || ' month')::interval;
part := 'read_events_y' || to_char(m, 'YYYY') || 'm' || to_char(m, 'MM');
EXECUTE format(
'CREATE TABLE %I PARTITION OF "read_events" FOR VALUES FROM (%L) TO (%L)',
part, m, m + interval '1 month');
EXECUTE format(
'CREATE UNIQUE INDEX %I ON %I ("dedup_key", "window_bucket")',
part || '_dedup_key', part);
END LOOP;
END $$;
INSERT INTO "read_events"
("id", "occurred_at", "actor_id", "session_key", "page_id", "pond_id",
"channel", "classification", "details", "dedup_key", "window_bucket",
"window_seconds")
SELECT "id", "occurred_at", "actor_id", "session_key", "page_id", "pond_id",
"channel", "classification", "details", "dedup_key", "window_bucket",
"window_seconds"
FROM "read_events_old";
DROP TABLE "read_events_old";

View File

@ -96,8 +96,13 @@ model AuditEntry {
/// volume, purpose and legal basis all differ. Deliberately WITHOUT foreign
/// keys: evidence must survive a page purge and a hard user deletion — the
/// ids stay as recorded (pseudonymous uuids), history is never rewritten.
///
/// In migrated databases the table is RANGE-partitioned by `occurred_at`
/// (monthly, issue #224) — hence the composite id. The dedup unique pair
/// lives per partition there (a partitioned parent cannot carry it without
/// the partition key); `db push` test databases get it on the plain table.
model ReadEvent {
id String @id @default(uuid())
id String @default(uuid())
occurredAt DateTime @default(now()) @map("occurred_at")
/// Null = anonymous reader (public grant); `sessionKey` still names the
/// browsing session, so the anonymous marker is explicit, not an accident.
@ -124,6 +129,7 @@ model ReadEvent {
/// represents up to this many seconds, so the evidence is not overread.
windowSeconds Int @map("window_seconds")
@@id([id, occurredAt])
@@unique([dedupKey, windowBucket])
@@index([pageId, occurredAt])
@@index([actorId, occurredAt])

View File

@ -1,9 +1,12 @@
import { Controller, Get, Param, Post, Query, Req, UseGuards } from '@nestjs/common';
import {
auditListQuerySchema,
readEventListQuerySchema,
type AuditListQuery,
type AuditListView,
type JobTriggerResult,
type ReadEventListQuery,
type ReadEventListView,
type StorageOverviewView,
type SystemBackupView,
type SystemJobView,
@ -45,6 +48,15 @@ export class SystemAdminController {
return this.system.auditLog(query);
}
/** Read-access trail queries (issue #224): "who read page X", "what did
* user Y read" Site-Admin only, like the audit viewer above. */
@Get('read-events')
async readEvents(
@Query(new ZodValidationPipe(readEventListQuerySchema)) query: ReadEventListQuery,
): Promise<ReadEventListView> {
return this.system.readEvents(query);
}
@Get('storage')
async storage(): Promise<StorageOverviewView> {
return this.system.storage();

View File

@ -7,10 +7,13 @@ import {
AUDIT_PAGE_SIZE,
BACKUP_FRESH_MAX_AGE_HOURS,
BACKUP_STATUS_FILE,
READ_EVENT_PAGE_SIZE,
type AuditListQuery,
type AuditListView,
type BackupStatus,
type JobTriggerResult,
type ReadEventListQuery,
type ReadEventListView,
type StorageOverviewView,
type SystemBackupView,
type SystemJobView,
@ -160,6 +163,66 @@ export class SystemAdminService {
};
}
/**
* The Site-Admin query path over the read-access trail (issue #224,
* ADR 0023) evidence nobody can read is not evidence. Answers "who read
* page X" and "what did user Y read" within a period. API-only by design
* (no panel yet): the trail is an examiner's tool, not a daily screen
* documented in data-model.md §read_events.
*/
async readEvents(query: ReadEventListQuery): Promise<ReadEventListView> {
const where: Prisma.ReadEventWhereInput = {};
if (query.pageId) where.pageId = query.pageId;
if (query.actor) {
const actor = await this.prisma.user.findUnique({ where: { username: query.actor } });
// An unknown username matches nothing rather than everything.
where.actorId = actor?.id ?? '00000000-0000-0000-0000-000000000000';
}
if (query.channel) where.channel = query.channel;
if (query.from || query.to) {
where.occurredAt = {
...(query.from ? { gte: query.from } : {}),
...(query.to ? { lte: query.to } : {}),
};
}
const total = await this.prisma.readEvent.count({ where });
const pageCount = Math.max(1, Math.ceil(total / READ_EVENT_PAGE_SIZE));
const page = Math.min(query.page, pageCount);
const events = await this.prisma.readEvent.findMany({
where,
orderBy: { occurredAt: 'desc' },
skip: (page - 1) * READ_EVENT_PAGE_SIZE,
take: READ_EVENT_PAGE_SIZE,
});
// No FK on actor_id (evidence outlives accounts) — resolve what still
// exists in one query, show the bare id otherwise.
const actorIds = [...new Set(events.map((e) => e.actorId).filter((id): id is string => !!id))];
const actors = actorIds.length
? await this.prisma.user.findMany({
where: { id: { in: actorIds } },
select: { id: true, username: true, displayName: true },
})
: [];
const actorById = new Map(actors.map((a) => [a.id, a]));
return {
entries: events.map((event) => ({
id: event.id,
occurredAt: event.occurredAt.toISOString(),
actor: event.actorId ? (actorById.get(event.actorId) ?? null) : null,
pageId: event.pageId,
pondId: event.pondId,
channel: event.channel,
classification: event.classification,
windowSeconds: event.windowSeconds,
details: (event.details as Record<string, unknown> | null) ?? null,
})),
page,
pageCount,
total,
};
}
async storage(): Promise<StorageOverviewView> {
const usages = await this.prisma.pondUsage.findMany({
where: { pond: { deletedAt: null } },

View File

@ -40,6 +40,7 @@ export const AUDIT_EVENTS = {
'pond.purged': { severity: 'notice' },
'quota.override_cleared': { severity: 'notice' },
'quota.override_set': { severity: 'notice' },
'read_trail.pruned': { severity: 'info' },
'settings.changed': { severity: 'notice' },
'setup.admin_created': { severity: 'notice' },
'setup.completed': { severity: 'info' },

View File

@ -18,11 +18,12 @@ import { AUDIT_EVENTS } from './audit-actions';
const doc = readFileSync(join(__dirname, '../../../../docs/architecture/audit-events.md'), 'utf8');
/** Event rows are `| \`ns.event\` | trigger | severity | ` the dot in the
* id keeps field-set rows (`msg`, `severity`, ) out of the match. */
* id keeps field-set rows (`msg`, `severity`, ) out of the match. The
* namespace may carry an underscore since `read_trail.*` (issue #224). */
function documentedEvents(): Map<string, string> {
const events = new Map<string, string>();
for (const line of doc.split('\n')) {
const id = /^\| `([a-z]+\.[a-z_]+)` +\|/.exec(line)?.[1];
const id = /^\| `([a-z_]+\.[a-z_]+)` +\|/.exec(line)?.[1];
if (!id) continue;
const cells = line.split('|').map((cell) => cell.trim());
// cells[0] is the empty string before the leading pipe.

View File

@ -0,0 +1,139 @@
import { Injectable } from '@nestjs/common';
import { PinoLogger } from 'nestjs-pino';
import { AuditService } from '../audit/audit.service';
import { ClockService } from '../common/clock.service';
import { PrismaService } from '../prisma/prisma.service';
import { InstanceSettingsService } from '../settings/instance-settings.service';
const MS_PER_DAY = 24 * 60 * 60 * 1000;
/** How many months ahead of "now" a partition must exist. Two keeps a
* multi-week job outage from ever reaching an uncovered month (the DEFAULT
* partition would still catch it reads never fail on a missing month). */
const MONTHS_AHEAD = 2;
/**
* Read-trail storage maintenance (issue #224, ADR 0023), one daily job with
* two duties:
*
* 1. **Partition upkeep** `read_events` is RANGE-partitioned by month
* (migration `20260731170000`); this creates the next {@link MONTHS_AHEAD}
* monthly partitions, each with its per-partition dedup unique index
* (#223 the partitioned parent cannot carry it). A `db push` database
* (tests) has a plain table; partition work skips itself there.
* 2. **Retention** events older than `readTrail.retentionDays` are
* removed: whole months by dropping their partition (no scan), the
* remainder (default partition, plain tables) by a ranged delete. The
* deletion is audited (`read_trail.pruned`) so a gap in the evidence is
* always explainable same principle as `audit.pruned` (#196), but a
* deliberately separate period.
*/
@Injectable()
export class ReadTrailMaintenanceService {
constructor(
private readonly prisma: PrismaService,
private readonly settings: InstanceSettingsService,
private readonly audit: AuditService,
private readonly clock: ClockService,
private readonly logger: PinoLogger,
) {
this.logger.setContext(ReadTrailMaintenanceService.name);
}
async run(): Promise<void> {
await this.ensurePartitions();
await this.pruneExpired();
}
/** True when read_events is a partitioned parent (relkind `p`). */
private async isPartitioned(): Promise<boolean> {
const rows = await this.prisma.$queryRaw<{ relkind: string }[]>`
SELECT relkind::text FROM pg_class
WHERE relname = 'read_events' AND relnamespace = 'public'::regnamespace`;
return rows[0]?.relkind === 'p';
}
/** `read_events_y2026m08` for 2026-08. */
private partitionName(month: Date): string {
const y = month.getUTCFullYear();
const m = String(month.getUTCMonth() + 1).padStart(2, '0');
return `read_events_y${y}m${m}`;
}
private monthStart(base: Date, offsetMonths: number): Date {
return new Date(Date.UTC(base.getUTCFullYear(), base.getUTCMonth() + offsetMonths, 1));
}
async ensurePartitions(): Promise<void> {
if (!(await this.isPartitioned())) {
this.logger.debug('read_events is not partitioned here; skipping partition upkeep');
return;
}
const now = this.clock.now();
for (let offset = 0; offset <= MONTHS_AHEAD; offset += 1) {
const from = this.monthStart(now, offset);
const to = this.monthStart(now, offset + 1);
const name = this.partitionName(from);
try {
await this.prisma.$executeRawUnsafe(
`CREATE TABLE IF NOT EXISTS "${name}" PARTITION OF "read_events"
FOR VALUES FROM ('${from.toISOString()}') TO ('${to.toISOString()}')`,
);
await this.prisma.$executeRawUnsafe(
`CREATE UNIQUE INDEX IF NOT EXISTS "${name}_dedup_key"
ON "${name}" ("dedup_key", "window_bucket")`,
);
} catch (error) {
// Most likely: the DEFAULT partition already holds rows of this month
// (the job lagged past a month boundary). Nothing is lost — those
// rows live in the default partition and age out through the ranged
// delete below; the month just cannot get its own partition anymore.
this.logger.warn({ partition: name, err: error }, 'read-trail partition not created');
}
}
}
async pruneExpired(): Promise<number> {
const retentionDays = await this.settings.get('readTrail.retentionDays');
const cutoff = new Date(this.clock.now().getTime() - retentionDays * MS_PER_DAY);
let dropped = 0;
if (await this.isPartitioned()) {
// Whole months strictly before the cutoff month go by DROP — no scan,
// and the dropped range is exact (every row in them is < cutoff).
const partitions = await this.prisma.$queryRaw<{ relname: string }[]>`
SELECT c.relname::text
FROM pg_inherits i
JOIN pg_class c ON c.oid = i.inhrelid
WHERE i.inhparent = 'read_events'::regclass
AND c.relname ~ '^read_events_y[0-9]{4}m[0-9]{2}$'`;
const cutoffMonth = this.monthStart(cutoff, 0);
for (const { relname } of partitions) {
const match = /^read_events_y(\d{4})m(\d{2})$/.exec(relname);
if (!match) continue;
const monthEnd = new Date(Date.UTC(Number(match[1]), Number(match[2]), 1));
if (monthEnd.getTime() > cutoffMonth.getTime()) continue;
const counted = await this.prisma.$queryRawUnsafe<{ count: bigint }[]>(
`SELECT count(*)::bigint AS count FROM "${relname}"`,
);
dropped += Number(counted[0]?.count ?? 0n);
await this.prisma.$executeRawUnsafe(`DROP TABLE "${relname}"`);
}
}
// The remainder: rows before the cutoff inside surviving partitions,
// the default partition, or a plain (test) table.
const deleted = await this.prisma.readEvent.deleteMany({
where: { occurredAt: { lt: cutoff } },
});
const count = dropped + deleted.count;
if (count > 0) {
await this.audit.record({
action: 'read_trail.pruned',
details: { count, cutoff: cutoff.toISOString(), retentionDays },
});
}
return count;
}
}

View File

@ -0,0 +1,308 @@
import { execFileSync } from 'node:child_process';
import { join } from 'node:path';
import { INestApplication } from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import { PinoLogger } from 'nestjs-pino';
import request from 'supertest';
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import { ClockService } from '../common/clock.service';
import { InstanceSettingsService } from '../settings/instance-settings.service';
import { createTestApp, sessionCookieOf } from '../testing/test-app';
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
import { UsersService } from '../users/users.service';
import { ReadTrailMaintenanceService } from './read-trail-maintenance.service';
const API_ROOT = join(__dirname, '..', '..');
const MS_PER_DAY = 24 * 60 * 60 * 1000;
/**
* Read-trail storage (issue #224, ADR 0023): its own retention period with
* an audited deletion, the Site-Admin query path, and against a freshly
* migrated database, where the real DDL applies the monthly RANGE
* partitioning with per-partition dedup indexes and DROP-based pruning.
*/
describe.skipIf(!hasTestDb)('read-trail storage (e2e, issue #224)', () => {
const suffix = uniqueSuffix();
describe('retention and admin query path (shared database)', () => {
let app: INestApplication;
let prisma: PrismaClient;
const password = 'lesetrail speicher 123';
let adminId: string;
let adminCookie: string;
let readerCookie: string;
// Plain text ids — read_events has no FKs by design (#222).
const pondId = `pond-${suffix}`;
const api = () => request(app.getHttpServer());
async function makeUser(handle: string, siteAdmin: boolean) {
const users = app.get(UsersService);
const user = await users.createUser({
username: `${handle}-${suffix}`,
email: `${handle}-${suffix}@example.test`,
displayName: handle,
password,
locale: 'en',
});
await users.markEmailVerified(user.id);
if (siteAdmin) {
await prisma.user.update({ where: { id: user.id }, data: { isSiteAdmin: true } });
}
const cookie = sessionCookieOf(
await api()
.post('/api/v1/auth/login')
.send({ usernameOrEmail: `${handle}-${suffix}`, password })
.expect(200),
);
return { id: user.id, cookie };
}
function eventRow(overrides: Record<string, unknown>) {
return {
actorId: null,
sessionKey: 'anon',
pageId: null,
pondId,
channel: 'page_view',
classification: 'vs_nfd',
dedupKey: `k-${suffix}-${Math.random().toString(36).slice(2)}`,
windowBucket: BigInt(Math.floor(Math.random() * 1_000_000_000)),
windowSeconds: 300,
...overrides,
};
}
beforeAll(async () => {
prisma = createTestPrisma();
await prisma.rateLimit.deleteMany({});
app = await createTestApp();
const admin = await makeUser('storage-admin', true);
adminId = admin.id;
adminCookie = admin.cookie;
const reader = await makeUser('storage-reader', false);
readerCookie = reader.cookie;
});
afterAll(async () => {
await prisma.readEvent.deleteMany({ where: { pondId } });
await prisma.auditEntry.deleteMany({ where: { action: 'read_trail.pruned' } });
await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.retentionDays' } });
await prisma.user.deleteMany({ where: { username: { contains: suffix } } });
await prisma.$disconnect();
await app.close();
});
it('prunes only events past its own period and audits the deletion', async () => {
await app.get(InstanceSettingsService).set('readTrail.retentionDays', 30, adminId);
const now = Date.now();
await prisma.readEvent.createMany({
data: [
eventRow({ occurredAt: new Date(now - 40 * MS_PER_DAY) }),
eventRow({ occurredAt: new Date(now - 31 * MS_PER_DAY) }),
eventRow({ occurredAt: new Date(now - 5 * MS_PER_DAY) }),
],
});
const maintenance = app.get(ReadTrailMaintenanceService);
const pruned = await maintenance.pruneExpired();
expect(pruned).toBeGreaterThanOrEqual(2);
const remaining = await prisma.readEvent.findMany({ where: { pondId } });
expect(remaining).toHaveLength(1);
const audit = await prisma.auditEntry.findFirst({
where: { action: 'read_trail.pruned' },
orderBy: { at: 'desc' },
});
expect(audit).not.toBeNull();
expect(audit!.details).toMatchObject({ retentionDays: 30 });
});
it('answers "who read page X" and "what did user Y read" for Site Admins only', async () => {
const pageId = `page-${suffix}`;
await prisma.readEvent.createMany({
data: [
eventRow({ pageId, actorId: adminId, channel: 'export' }),
eventRow({ pageId: `other-${suffix}`, actorId: null }),
],
});
const byPage = await api()
.get(`/api/v1/admin/system/read-events?pageId=${pageId}`)
.set('Cookie', adminCookie)
.expect(200);
expect(byPage.body.total).toBe(1);
expect(byPage.body.entries[0]).toMatchObject({
pageId,
channel: 'export',
windowSeconds: 300,
});
expect(byPage.body.entries[0].actor).toMatchObject({ id: adminId });
const byActor = await api()
.get(`/api/v1/admin/system/read-events?actor=storage-admin-${suffix}`)
.set('Cookie', adminCookie)
.expect(200);
expect(byActor.body.total).toBe(1);
// An unknown username matches nothing rather than everything.
const unknown = await api()
.get(`/api/v1/admin/system/read-events?actor=nobody-${suffix}`)
.set('Cookie', adminCookie)
.expect(200);
expect(unknown.body.total).toBe(0);
await api()
.get(`/api/v1/admin/system/read-events?pageId=${pageId}`)
.set('Cookie', readerCookie)
.expect(403);
});
});
describe('partitioned shape (fresh database, real migrations)', () => {
const baseUrl = process.env.TEST_DATABASE_URL!;
const dbName = `dorfteich_trail_${suffix}`;
let fresh: PrismaClient;
let maintenance: ReadTrailMaintenanceService;
const auditRecord = vi.fn().mockResolvedValue(undefined);
let retentionDays = 365;
function freshUrl(): string {
const url = new URL(baseUrl);
url.pathname = `/${dbName}`;
return url.toString();
}
beforeAll(async () => {
const admin = new PrismaClient({ datasourceUrl: baseUrl });
try {
await admin.$executeRawUnsafe(`CREATE DATABASE "${dbName}"`);
} finally {
await admin.$disconnect();
}
execFileSync(
process.execPath,
[join(API_ROOT, 'node_modules', 'prisma', 'build', 'index.js'), 'migrate', 'deploy'],
{ env: { ...process.env, DATABASE_URL: freshUrl() }, stdio: 'pipe', cwd: API_ROOT },
);
fresh = new PrismaClient({ datasourceUrl: freshUrl() });
const settingsStub = {
get: async () => retentionDays,
} as unknown as InstanceSettingsService;
const auditStub = { record: auditRecord } as never;
const loggerStub = {
setContext: () => undefined,
debug: () => undefined,
info: () => undefined,
warn: () => undefined,
error: () => undefined,
} as unknown as PinoLogger;
maintenance = new ReadTrailMaintenanceService(
fresh as never,
settingsStub,
auditStub,
new ClockService(),
loggerStub,
);
}, 120_000);
afterAll(async () => {
await fresh.$disconnect();
const admin = new PrismaClient({ datasourceUrl: baseUrl });
try {
await admin.$executeRawUnsafe(`DROP DATABASE IF EXISTS "${dbName}" WITH (FORCE)`);
} finally {
await admin.$disconnect();
}
});
it('migrates read_events to a partitioned parent with current-month coverage', async () => {
const kind = await fresh.$queryRaw<{ relkind: string }[]>`
SELECT relkind::text FROM pg_class WHERE relname = 'read_events'`;
expect(kind[0]!.relkind).toBe('p');
const partitions = await fresh.$queryRaw<{ relname: string }[]>`
SELECT c.relname::text FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
WHERE i.inhparent = 'read_events'::regclass ORDER BY c.relname`;
const names = partitions.map((p) => p.relname);
expect(names).toContain('read_events_default');
expect(names.some((n) => /^read_events_y\d{4}m\d{2}$/.test(n))).toBe(true);
});
it('keeps the dedup unique pair enforced per partition (P2002 on the duplicate)', async () => {
const row = {
actorId: null,
sessionKey: 'anon',
pageId: null,
pondId: 'pond-part',
channel: 'page_view',
classification: 'vs_nfd',
dedupKey: `dup-${suffix}`,
windowBucket: 42n,
windowSeconds: 300,
};
await fresh.readEvent.create({ data: row });
await expect(fresh.readEvent.create({ data: { ...row } })).rejects.toMatchObject({
code: 'P2002',
});
});
it('creates months ahead and prunes whole expired partitions by DROP, audited', async () => {
await maintenance.ensurePartitions();
const next = new Date();
const nextMonth = new Date(Date.UTC(next.getUTCFullYear(), next.getUTCMonth() + 2, 1));
const nextName = `read_events_y${nextMonth.getUTCFullYear()}m${String(
nextMonth.getUTCMonth() + 1,
).padStart(2, '0')}`;
const created = await fresh.$queryRawUnsafe<{ relname: string }[]>(
`SELECT relname::text FROM pg_class WHERE relname = '${nextName}'`,
);
expect(created).toHaveLength(1);
// ...and each new partition carries its own dedup unique index.
const index = await fresh.$queryRawUnsafe<{ indexname: string }[]>(
`SELECT indexname::text FROM pg_indexes WHERE tablename = '${nextName}'
AND indexname = '${nextName}_dedup_key'`,
);
expect(index).toHaveLength(1);
// An old month: partition + one event well past retention.
await fresh.$executeRawUnsafe(
`CREATE TABLE "read_events_y2020m01" PARTITION OF "read_events"
FOR VALUES FROM ('2020-01-01') TO ('2020-02-01')`,
);
await fresh.readEvent.create({
data: {
occurredAt: new Date('2020-01-15T12:00:00Z'),
actorId: null,
sessionKey: 'anon',
pageId: null,
pondId: 'pond-part',
channel: 'export',
classification: 'vs_nfd',
dedupKey: `old-${suffix}`,
windowBucket: 1n,
windowSeconds: 300,
},
});
const pruned = await maintenance.pruneExpired();
expect(pruned).toBeGreaterThanOrEqual(1);
const gone = await fresh.$queryRawUnsafe<{ relname: string }[]>(
`SELECT relname::text FROM pg_class WHERE relname = 'read_events_y2020m01'`,
);
expect(gone).toHaveLength(0);
expect(auditRecord).toHaveBeenCalledWith(
expect.objectContaining({ action: 'read_trail.pruned' }),
);
// The current month's events survive.
const kept = await fresh.readEvent.count({ where: { dedupKey: `dup-${suffix}` } });
expect(kept).toBe(1);
retentionDays = 365; // restore for any later use
});
});
});

View File

@ -1,10 +1,16 @@
import { Global, Module } from '@nestjs/common';
import { Global, Module, OnModuleInit } from '@nestjs/common';
import { CommonModule } from '../common/common.module';
import { SchedulerModule } from '../scheduler/scheduler.module';
import { SchedulerService } from '../scheduler/scheduler.service';
import { SettingsModule } from '../settings/settings.module';
import { ReadTrailMaintenanceService } from './read-trail-maintenance.service';
import { ReadTrailService } from './read-trail.service';
/** Daily, per operations.md's maintenance-jobs table (issue #224). */
const READ_TRAIL_MAINTENANCE_CADENCE_SECONDS = 24 * 60 * 60;
/**
* Global like AuditModule and for the same reason: the read-access trail
* (issue #222, ADR 0023) cuts across every module that serves page content
@ -12,8 +18,23 @@ import { ReadTrailService } from './read-trail.service';
*/
@Global()
@Module({
imports: [CommonModule, SettingsModule],
providers: [ReadTrailService],
exports: [ReadTrailService],
imports: [CommonModule, SchedulerModule, SettingsModule],
providers: [ReadTrailService, ReadTrailMaintenanceService],
exports: [ReadTrailService, ReadTrailMaintenanceService],
})
export class ReadTrailModule {}
export class ReadTrailModule implements OnModuleInit {
constructor(
private readonly scheduler: SchedulerService,
private readonly maintenance: ReadTrailMaintenanceService,
) {}
onModuleInit(): void {
this.scheduler.register({
name: 'read-trail-maintenance',
cadenceSeconds: READ_TRAIL_MAINTENANCE_CADENCE_SECONDS,
run: async () => {
await this.maintenance.run();
},
});
}
}

View File

@ -56,6 +56,13 @@ export const INSTANCE_SETTINGS = {
// 5 minutes keeps a live Yjs session (collab tokens every 60 s) at a
// bounded ~12 events/hour/page while still evidencing distinct visits.
'readTrail.dedupWindowMinutes': z.number().int().min(1).default(5),
// Read-trail retention (issue #224): days a read event is kept before the
// daily maintenance job removes it — deliberately independent of
// `audit.retentionDays` (#196), because volume, purpose and legal basis
// differ. The deletion itself is audited (`read_trail.pruned`), so a gap
// is always explainable. One year mirrors the audit default; shortening
// it is an operator decision under the purpose limitation (#225).
'readTrail.retentionDays': z.number().int().min(1).default(365),
// Default VS-NfD classification for newly created pages (ADR 0022,
// issue #204). An instance operated inside a classified environment sets
// this to `vs_nfd` so nothing starts unmarked; inheritance from the

View File

@ -20,8 +20,9 @@ test('lists maintenance jobs and triggers one manually', async ({ browser }) =>
// Keep in sync with the scheduler registrations: trash-purge,
// version-thinning, page-compaction, data-export-purge,
// notification-digest, orphan-file-sweep (#194), audit-retention (#196),
// conversion-payload-prune (#233), mail-outbox-retention (#234).
await expect(jobsTable.locator('tbody tr')).toHaveCount(9);
// conversion-payload-prune (#233), mail-outbox-retention (#234),
// read-trail-maintenance (#224).
await expect(jobsTable.locator('tbody tr')).toHaveCount(10);
const firstRow = jobsTable.locator('tbody tr').first();
await firstRow.getByRole('button').click();

View File

@ -89,6 +89,23 @@ traffic per open document.
anonymous browsers apart would require fingerprinting (IP/UA), which the
purpose limitation (#225) rules out.
## Decisions taken in #224
- **Monthly RANGE partitions** on `occurred_at`, plus a DEFAULT partition
as a safety net: a lagging maintenance job must never make classified
reads fail (the trail write is hard-failing — an ops miss must not
become an outage). Retention DROPs whole expired months without
scanning; the pruning run is audited (`read_trail.pruned`) under the
trail's own `readTrail.retentionDays` (default 365).
- **The dedup unique pair lives per partition** (PostgreSQL cannot carry
it on the parent without the partition key). A dedup bucket spanning a
month boundary can therefore record one duplicate — over-recording is
acceptable, gaps are not.
- **Query path is API-only** (`GET /admin/system/read-events`, Site
Admin): the trail is an examiner's tool, not a daily screen. Evidence
nobody can read is not evidence — the path exists; a panel does not.
- **Growth measured**: ~1 MB per 1000 events including indexes.
## Consequences
- The scope limit is the feature's strongest argument in the works-council

View File

@ -1,7 +1,7 @@
# Audit event catalogue
**Catalogue version 1.1 (2026-07-31; 1.1 adds `page.classification_*`,
issue #205).**
**Catalogue version 1.2 (2026-07-31; 1.2 adds `read_trail.pruned`,
issue #224; 1.1 added `page.classification_*`, issue #205).**
This is the operator-facing contract for the audit trail: every event id
the application can emit, with its trigger, severity, actor/target
@ -103,11 +103,12 @@ failure), `warning` = feeds detection (suspicious or destructive),
### Content integrity & lifecycle (`file.*`, `pond.*`, `audit.*`)
| Id | Trigger | Severity | Actor | Target | Fields |
| ----------------------- | ------------------------------------------------------------------- | -------- | ---------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- |
| `file.integrity_failed` | Attachment download hash mismatch — fail-closed (issue #199) | critical | `null` (any downloader; detection) | `attachment` | `pondId`, `expected` (stored sha256), `actual` (computed sha256) |
| `pond.purged` | Pond irreversibly destroyed (manual or trash retention, issue #193) | notice | admin, `null` when retention-run | `pond` | `trigger` (`manual` \| `retention`) plus per-object-type deletion counts (e.g. `pages`, `attachments`, … — informational, keys may grow) |
| `audit.pruned` | Audit retention deleted rows past the period (issue #196) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` |
| Id | Trigger | Severity | Actor | Target | Fields |
| ----------------------- | -------------------------------------------------------------------- | -------- | ---------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- |
| `file.integrity_failed` | Attachment download hash mismatch — fail-closed (issue #199) | critical | `null` (any downloader; detection) | `attachment` | `pondId`, `expected` (stored sha256), `actual` (computed sha256) |
| `pond.purged` | Pond irreversibly destroyed (manual or trash retention, issue #193) | notice | admin, `null` when retention-run | `pond` | `trigger` (`manual` \| `retention`) plus per-object-type deletion counts (e.g. `pages`, `attachments`, … — informational, keys may grow) |
| `audit.pruned` | Audit retention deleted rows past the period (issue #196) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` |
| `read_trail.pruned` | Read-trail retention removed events past its own period (issue #224) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` |
### Public API (`api.*`)

View File

@ -186,6 +186,40 @@ Cached counters per pond: `storage_bytes_used`, `editor_count`,
- `jobs`: conversion jobs for import/export (ADR 0009) and maintenance jobs
(compaction, trash purge, quota reconciliation) with status + timestamps.
## Read-access trail (`read_events`, issues #222#224, ADR 0023)
One row per read of a `classification = vs_nfd` page per channel and dedup
window — separate from `audit_log` because volume, purpose and legal basis
all differ.
| Column | Notes |
| ---------------------------- | ------------------------------------------------------------------------------------ |
| `id`, `occurred_at` | composite PK (the partition key must be part of it) |
| `actor_id` | reader, null = anonymous; **no FK** — evidence outlives accounts |
| `session_key` | `session:<id>` / `token:<id>` / `job:<id>` / `anon` |
| `page_id`, `pond_id` | plain ids, **no FK** — evidence outlives purges |
| `channel` | `page_view` / `no_js_shell` / `public_api` / `attachment` / `export` / `collab_join` |
| `classification` | at read time; reclassification never rewrites history |
| `dedup_key`, `window_bucket` | unique pair per partition — race-free dedup (#223) |
| `window_seconds` | the row represents up to this many seconds of access |
| `details` | small context (export format, attachment id) — never content |
Storage shape (issue #224): RANGE-partitioned by month on `occurred_at`,
with a DEFAULT partition as safety net (a lagging maintenance job must
never make classified reads fail — the trail write is hard-failing by
design). The daily `read-trail-maintenance` job creates months ahead
(each with its per-partition dedup index) and applies the trail's own
retention `readTrail.retentionDays` (default 365): whole expired months
are DROPped without scanning, remainders deleted by range; every pruning
run is audited as `read_trail.pruned`. Growth, measured on PG 17:
**~1 MB per 1000 events** including indexes (≈970 bytes/row) — a
100-person instance at hundreds of classified reads/day stays in the tens
of MB per year.
Query path: `GET /admin/system/read-events` (Site Admin, filters
`pageId`, `actor`, `channel`, `from`/`to`) — deliberately API-only, no
panel: the trail is an examiner's tool, not a daily screen.
## Comments & notifications (later milestone)
- `comments`: `page_id`, `author_id`, `body` (Markdown), `anchor`

View File

@ -109,6 +109,7 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack.
| conversion payload prune | daily | null finished conversion jobs' raw bytes (#233) |
| mail outbox retention | daily | delete sent/failed outbox rows past period (#234) |
| mail outbox retry | every minute | e-mail delivery with backoff |
| read-trail maintenance | daily | partition months ahead + prune `read_events` (#224) |
Job outcomes are visible in the Site Admin UI (last run, status) — that
panel is the operator's single glance for instance health.

View File

@ -136,7 +136,7 @@ Protokolliert werden Lesezugriffe **ausschließlich** auf Seiten mit
Attachment-Download, Export, No-JS-Shell, Collab-WS-Join · 4 AT · #222
- [x] Dedup-Fenster (eine Sitzung + eine Seite innerhalb N Minuten = ein
Ereignis), sonst erzeugt Yjs-Sync eine Ereignisflut · 2 AT · #223
- [ ] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224
- [x] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224
- [ ] Abschaltbar, Zweckbindung dokumentiert · 12 AT · #225
Vorteil über den Aufwand hinaus: Die Zweckbindung ist sauber begründbar

View File

@ -40,6 +40,7 @@ Settings-Cache ist in-process (operations.md).
| `backup.nextcloud.enabled` | `false` | `false` | „Backup nur lokal": kein Anwendungs-Upload von Restore-Sets zu Drittdiensten. Fernspiegel regelt ausschließlich die Deploy-Allowlist (1.2). |
| `trash.retentionDays`, `audit.retentionDays`, `conversion.payloadRetentionDays`, `mail.outboxRetentionDays` | Defaults (30/365/30/30) | ebd. | Aufbewahrung bewusst begrenzt; Verkürzung nach Betreiber-Löschkonzept zulässig (Betriebshandbuch §5). |
| `readTrail.dedupWindowMinutes` | Default (5) | `5` | Dedup-Fenster des Lesetrails (#223): je (Sitzung, Seite, Kanal) ein Ereignis pro Fenster — begrenzt die Ereignisflut einer Live-Sitzung auf ~12/h. Kleiner = feineres Protokoll und mehr Zeilen; Änderung mit dem Zweckbindungs-Dokument (#225) abstimmen. |
| `readTrail.retentionDays` | Default (365) | `365` | eigene Aufbewahrung des Lesetrails (#224), bewusst getrennt von `audit.retentionDays`; Löschläufe sind selbst auditiert (`read_trail.pruned`). Dauer mit der Zweckbindung (#225) und dem Betreiber-Löschkonzept abstimmen. |
| `legal.imprint`, `legal.privacyPolicy` | befüllt | leer | Betreiberpflicht; leere Seiten zeigen einen Warnbanner. |
### 1.2 Deploy-Konfiguration (`.env` / Compose — nur Plattformzugriff, bewusst nicht per Admin-UI)

View File

@ -164,6 +164,49 @@ export interface AuditListView {
total: number;
}
/**
* Site-Admin query path over the read-access trail (issue #224, ADR 0023)
* evidence nobody can read is not evidence. Answers the two expected
* questions: "who read page X" (pageId) and "what did user Y read" (actor),
* both within a period.
*/
export const READ_EVENT_PAGE_SIZE = 50;
export const readEventListQuerySchema = z.object({
/** Page id the events belong to. */
pageId: z.string().trim().min(1).optional(),
/** Exact username of the reading user. */
actor: z.string().trim().min(1).optional(),
channel: z.string().trim().min(1).optional(),
from: z.coerce.date().optional(),
to: z.coerce.date().optional(),
page: z.coerce.number().int().min(1).default(1),
});
export type ReadEventListQuery = z.infer<typeof readEventListQuerySchema>;
export interface ReadEventView {
id: string;
occurredAt: string;
actor: AuditActorView | null;
/** Null for pond-level attachments (no page); the details name the file. */
pageId: string | null;
pondId: string;
channel: string;
classification: string;
/** Seconds the event's dedup window spans (#223) the row represents up
* to this much access time, not a single request. */
windowSeconds: number;
details: Record<string, unknown> | null;
}
export interface ReadEventListView {
entries: ReadEventView[];
page: number;
pageCount: number;
total: number;
}
export interface StoragePondView {
pondId: string;
name: string;