#225: read-trail master switch and purpose limitation #281
@ -0,0 +1,32 @@
|
||||
-- #223 (ADR 0023): dedup window for the read trail. Aligned buckets
|
||||
-- (floor(epoch / window)) with a unique (dedup_key, window_bucket) pair make
|
||||
-- concurrent duplicates collapse race-free at insert time.
|
||||
ALTER TABLE "read_events"
|
||||
ADD COLUMN "dedup_key" TEXT,
|
||||
ADD COLUMN "window_bucket" BIGINT,
|
||||
ADD COLUMN "window_seconds" INTEGER;
|
||||
|
||||
-- Backfill rows written between the #222 and #223 deploys under the default
|
||||
-- 5-minute window, then apply the window's own semantics retroactively:
|
||||
-- within one (key, bucket) pair only the FIRST event is the evidence row —
|
||||
-- exactly what the window would have recorded had it existed.
|
||||
UPDATE "read_events"
|
||||
SET "dedup_key" = "session_key" || ':' || COALESCE("page_id", '-') || ':' || "channel",
|
||||
"window_bucket" = FLOOR(EXTRACT(EPOCH FROM "occurred_at") / 300)::BIGINT,
|
||||
"window_seconds" = 300
|
||||
WHERE "dedup_key" IS NULL;
|
||||
|
||||
DELETE FROM "read_events" keep
|
||||
USING "read_events" first
|
||||
WHERE keep."dedup_key" = first."dedup_key"
|
||||
AND keep."window_bucket" = first."window_bucket"
|
||||
AND (first."occurred_at" < keep."occurred_at"
|
||||
OR (first."occurred_at" = keep."occurred_at" AND first."id" < keep."id"));
|
||||
|
||||
ALTER TABLE "read_events"
|
||||
ALTER COLUMN "dedup_key" SET NOT NULL,
|
||||
ALTER COLUMN "window_bucket" SET NOT NULL,
|
||||
ALTER COLUMN "window_seconds" SET NOT NULL;
|
||||
|
||||
CREATE UNIQUE INDEX "read_events_dedup_key_window_bucket_key"
|
||||
ON "read_events"("dedup_key", "window_bucket");
|
||||
@ -0,0 +1,76 @@
|
||||
-- #224 (ADR 0023): convert read_events to monthly RANGE partitions on
|
||||
-- occurred_at. Volume grows unbounded with use; retention then DROPs whole
|
||||
-- expired partitions instead of scanning deletes. The primary key gains the
|
||||
-- partition column (PostgreSQL requirement); the dedup unique pair
|
||||
-- (dedup_key, window_bucket) moves to PER-PARTITION unique indexes — a
|
||||
-- partitioned parent cannot carry it without the partition key. A bucket
|
||||
-- spanning a month boundary can therefore record one duplicate; documented
|
||||
-- in ADR 0023, over-recording is acceptable, gaps are not.
|
||||
--
|
||||
-- A DEFAULT partition catches rows outside every maintained range, so a
|
||||
-- lagging maintenance job can never make classified reads fail (the trail's
|
||||
-- hard-failure semantics would otherwise turn an ops miss into an outage).
|
||||
|
||||
ALTER TABLE "read_events" RENAME TO "read_events_old";
|
||||
ALTER INDEX "read_events_pkey" RENAME TO "read_events_old_pkey";
|
||||
ALTER INDEX "read_events_dedup_key_window_bucket_key" RENAME TO "read_events_old_dedup_key";
|
||||
ALTER INDEX "read_events_page_id_occurred_at_idx" RENAME TO "read_events_old_page_idx";
|
||||
ALTER INDEX "read_events_actor_id_occurred_at_idx" RENAME TO "read_events_old_actor_idx";
|
||||
ALTER INDEX "read_events_occurred_at_idx" RENAME TO "read_events_old_at_idx";
|
||||
|
||||
CREATE TABLE "read_events" (
|
||||
"id" TEXT NOT NULL,
|
||||
"occurred_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"actor_id" TEXT,
|
||||
"session_key" TEXT NOT NULL,
|
||||
"page_id" TEXT,
|
||||
"pond_id" TEXT NOT NULL,
|
||||
"channel" TEXT NOT NULL,
|
||||
"classification" TEXT NOT NULL,
|
||||
"details" JSONB,
|
||||
"dedup_key" TEXT NOT NULL,
|
||||
"window_bucket" BIGINT NOT NULL,
|
||||
"window_seconds" INTEGER NOT NULL,
|
||||
|
||||
CONSTRAINT "read_events_pkey" PRIMARY KEY ("id", "occurred_at")
|
||||
) PARTITION BY RANGE ("occurred_at");
|
||||
|
||||
-- Non-unique parent indexes propagate to every partition automatically.
|
||||
CREATE INDEX "read_events_page_id_occurred_at_idx" ON "read_events"("page_id", "occurred_at");
|
||||
CREATE INDEX "read_events_actor_id_occurred_at_idx" ON "read_events"("actor_id", "occurred_at");
|
||||
CREATE INDEX "read_events_occurred_at_idx" ON "read_events"("occurred_at");
|
||||
|
||||
-- The safety-net partition, plus the current and the next month — the daily
|
||||
-- maintenance job (read-trail-maintenance) keeps creating months ahead and
|
||||
-- adds the same per-partition dedup index to each new one.
|
||||
CREATE TABLE "read_events_default" PARTITION OF "read_events" DEFAULT;
|
||||
CREATE UNIQUE INDEX "read_events_default_dedup_key"
|
||||
ON "read_events_default"("dedup_key", "window_bucket");
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
m DATE;
|
||||
part TEXT;
|
||||
BEGIN
|
||||
FOR i IN 0..1 LOOP
|
||||
m := date_trunc('month', now())::date + (i || ' month')::interval;
|
||||
part := 'read_events_y' || to_char(m, 'YYYY') || 'm' || to_char(m, 'MM');
|
||||
EXECUTE format(
|
||||
'CREATE TABLE %I PARTITION OF "read_events" FOR VALUES FROM (%L) TO (%L)',
|
||||
part, m, m + interval '1 month');
|
||||
EXECUTE format(
|
||||
'CREATE UNIQUE INDEX %I ON %I ("dedup_key", "window_bucket")',
|
||||
part || '_dedup_key', part);
|
||||
END LOOP;
|
||||
END $$;
|
||||
|
||||
INSERT INTO "read_events"
|
||||
("id", "occurred_at", "actor_id", "session_key", "page_id", "pond_id",
|
||||
"channel", "classification", "details", "dedup_key", "window_bucket",
|
||||
"window_seconds")
|
||||
SELECT "id", "occurred_at", "actor_id", "session_key", "page_id", "pond_id",
|
||||
"channel", "classification", "details", "dedup_key", "window_bucket",
|
||||
"window_seconds"
|
||||
FROM "read_events_old";
|
||||
|
||||
DROP TABLE "read_events_old";
|
||||
@ -96,8 +96,13 @@ model AuditEntry {
|
||||
/// volume, purpose and legal basis all differ. Deliberately WITHOUT foreign
|
||||
/// keys: evidence must survive a page purge and a hard user deletion — the
|
||||
/// ids stay as recorded (pseudonymous uuids), history is never rewritten.
|
||||
///
|
||||
/// In migrated databases the table is RANGE-partitioned by `occurred_at`
|
||||
/// (monthly, issue #224) — hence the composite id. The dedup unique pair
|
||||
/// lives per partition there (a partitioned parent cannot carry it without
|
||||
/// the partition key); `db push` test databases get it on the plain table.
|
||||
model ReadEvent {
|
||||
id String @id @default(uuid())
|
||||
id String @default(uuid())
|
||||
occurredAt DateTime @default(now()) @map("occurred_at")
|
||||
/// Null = anonymous reader (public grant); `sessionKey` still names the
|
||||
/// browsing session, so the anonymous marker is explicit, not an accident.
|
||||
@ -115,7 +120,17 @@ model ReadEvent {
|
||||
/// rewrite history (ADR 0023).
|
||||
classification String
|
||||
details Json?
|
||||
/// Dedup window (issue #223): `<sessionKey>:<pageId|->:<channel>` plus the
|
||||
/// aligned bucket `floor(epoch / windowSeconds)`. The unique pair makes
|
||||
/// concurrent duplicate reads collapse race-free (insert or P2002-skip).
|
||||
dedupKey String @map("dedup_key")
|
||||
windowBucket BigInt @map("window_bucket")
|
||||
/// Window length the event was recorded under — the row itself states it
|
||||
/// represents up to this many seconds, so the evidence is not overread.
|
||||
windowSeconds Int @map("window_seconds")
|
||||
|
||||
@@id([id, occurredAt])
|
||||
@@unique([dedupKey, windowBucket])
|
||||
@@index([pageId, occurredAt])
|
||||
@@index([actorId, occurredAt])
|
||||
@@index([occurredAt])
|
||||
|
||||
@ -1,9 +1,12 @@
|
||||
import { Controller, Get, Param, Post, Query, Req, UseGuards } from '@nestjs/common';
|
||||
import {
|
||||
auditListQuerySchema,
|
||||
readEventListQuerySchema,
|
||||
type AuditListQuery,
|
||||
type AuditListView,
|
||||
type JobTriggerResult,
|
||||
type ReadEventListQuery,
|
||||
type ReadEventListView,
|
||||
type StorageOverviewView,
|
||||
type SystemBackupView,
|
||||
type SystemJobView,
|
||||
@ -45,6 +48,15 @@ export class SystemAdminController {
|
||||
return this.system.auditLog(query);
|
||||
}
|
||||
|
||||
/** Read-access trail queries (issue #224): "who read page X", "what did
|
||||
* user Y read" — Site-Admin only, like the audit viewer above. */
|
||||
@Get('read-events')
|
||||
async readEvents(
|
||||
@Query(new ZodValidationPipe(readEventListQuerySchema)) query: ReadEventListQuery,
|
||||
): Promise<ReadEventListView> {
|
||||
return this.system.readEvents(query);
|
||||
}
|
||||
|
||||
@Get('storage')
|
||||
async storage(): Promise<StorageOverviewView> {
|
||||
return this.system.storage();
|
||||
|
||||
@ -7,10 +7,13 @@ import {
|
||||
AUDIT_PAGE_SIZE,
|
||||
BACKUP_FRESH_MAX_AGE_HOURS,
|
||||
BACKUP_STATUS_FILE,
|
||||
READ_EVENT_PAGE_SIZE,
|
||||
type AuditListQuery,
|
||||
type AuditListView,
|
||||
type BackupStatus,
|
||||
type JobTriggerResult,
|
||||
type ReadEventListQuery,
|
||||
type ReadEventListView,
|
||||
type StorageOverviewView,
|
||||
type SystemBackupView,
|
||||
type SystemJobView,
|
||||
@ -160,6 +163,66 @@ export class SystemAdminService {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The Site-Admin query path over the read-access trail (issue #224,
|
||||
* ADR 0023) — evidence nobody can read is not evidence. Answers "who read
|
||||
* page X" and "what did user Y read" within a period. API-only by design
|
||||
* (no panel yet): the trail is an examiner's tool, not a daily screen —
|
||||
* documented in data-model.md §read_events.
|
||||
*/
|
||||
async readEvents(query: ReadEventListQuery): Promise<ReadEventListView> {
|
||||
const where: Prisma.ReadEventWhereInput = {};
|
||||
if (query.pageId) where.pageId = query.pageId;
|
||||
if (query.actor) {
|
||||
const actor = await this.prisma.user.findUnique({ where: { username: query.actor } });
|
||||
// An unknown username matches nothing rather than everything.
|
||||
where.actorId = actor?.id ?? '00000000-0000-0000-0000-000000000000';
|
||||
}
|
||||
if (query.channel) where.channel = query.channel;
|
||||
if (query.from || query.to) {
|
||||
where.occurredAt = {
|
||||
...(query.from ? { gte: query.from } : {}),
|
||||
...(query.to ? { lte: query.to } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
const total = await this.prisma.readEvent.count({ where });
|
||||
const pageCount = Math.max(1, Math.ceil(total / READ_EVENT_PAGE_SIZE));
|
||||
const page = Math.min(query.page, pageCount);
|
||||
const events = await this.prisma.readEvent.findMany({
|
||||
where,
|
||||
orderBy: { occurredAt: 'desc' },
|
||||
skip: (page - 1) * READ_EVENT_PAGE_SIZE,
|
||||
take: READ_EVENT_PAGE_SIZE,
|
||||
});
|
||||
// No FK on actor_id (evidence outlives accounts) — resolve what still
|
||||
// exists in one query, show the bare id otherwise.
|
||||
const actorIds = [...new Set(events.map((e) => e.actorId).filter((id): id is string => !!id))];
|
||||
const actors = actorIds.length
|
||||
? await this.prisma.user.findMany({
|
||||
where: { id: { in: actorIds } },
|
||||
select: { id: true, username: true, displayName: true },
|
||||
})
|
||||
: [];
|
||||
const actorById = new Map(actors.map((a) => [a.id, a]));
|
||||
return {
|
||||
entries: events.map((event) => ({
|
||||
id: event.id,
|
||||
occurredAt: event.occurredAt.toISOString(),
|
||||
actor: event.actorId ? (actorById.get(event.actorId) ?? null) : null,
|
||||
pageId: event.pageId,
|
||||
pondId: event.pondId,
|
||||
channel: event.channel,
|
||||
classification: event.classification,
|
||||
windowSeconds: event.windowSeconds,
|
||||
details: (event.details as Record<string, unknown> | null) ?? null,
|
||||
})),
|
||||
page,
|
||||
pageCount,
|
||||
total,
|
||||
};
|
||||
}
|
||||
|
||||
async storage(): Promise<StorageOverviewView> {
|
||||
const usages = await this.prisma.pondUsage.findMany({
|
||||
where: { pond: { deletedAt: null } },
|
||||
|
||||
@ -40,6 +40,7 @@ export const AUDIT_EVENTS = {
|
||||
'pond.purged': { severity: 'notice' },
|
||||
'quota.override_cleared': { severity: 'notice' },
|
||||
'quota.override_set': { severity: 'notice' },
|
||||
'read_trail.pruned': { severity: 'info' },
|
||||
'settings.changed': { severity: 'notice' },
|
||||
'setup.admin_created': { severity: 'notice' },
|
||||
'setup.completed': { severity: 'info' },
|
||||
|
||||
@ -18,11 +18,12 @@ import { AUDIT_EVENTS } from './audit-actions';
|
||||
const doc = readFileSync(join(__dirname, '../../../../docs/architecture/audit-events.md'), 'utf8');
|
||||
|
||||
/** Event rows are `| \`ns.event\` | trigger | severity | …` — the dot in the
|
||||
* id keeps field-set rows (`msg`, `severity`, …) out of the match. */
|
||||
* id keeps field-set rows (`msg`, `severity`, …) out of the match. The
|
||||
* namespace may carry an underscore since `read_trail.*` (issue #224). */
|
||||
function documentedEvents(): Map<string, string> {
|
||||
const events = new Map<string, string>();
|
||||
for (const line of doc.split('\n')) {
|
||||
const id = /^\| `([a-z]+\.[a-z_]+)` +\|/.exec(line)?.[1];
|
||||
const id = /^\| `([a-z_]+\.[a-z_]+)` +\|/.exec(line)?.[1];
|
||||
if (!id) continue;
|
||||
const cells = line.split('|').map((cell) => cell.trim());
|
||||
// cells[0] is the empty string before the leading pipe.
|
||||
|
||||
181
apps/api/src/read-trail/read-trail-dedup.e2e.db.test.ts
Normal file
181
apps/api/src/read-trail/read-trail-dedup.e2e.db.test.ts
Normal file
@ -0,0 +1,181 @@
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import request from 'supertest';
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { ClockService } from '../common/clock.service';
|
||||
import { InstanceSettingsService } from '../settings/instance-settings.service';
|
||||
import { createTestApp, sessionCookieOf } from '../testing/test-app';
|
||||
import { createTestPrisma, grantOwnerAdmin, hasTestDb, uniqueSuffix } from '../testing/test-db';
|
||||
import { UsersService } from '../users/users.service';
|
||||
|
||||
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||
|
||||
/**
|
||||
* The read-trail dedup window (issue #223, ADR 0023): one event per
|
||||
* (session, page, channel) within an aligned `readTrail.dedupWindowMinutes`
|
||||
* window — repeats and reconnect-style re-requests collapse, a new session
|
||||
* or a new window does not, and the recorded row states the window length
|
||||
* it represents.
|
||||
*/
|
||||
describe.skipIf(!hasTestDb)('read-trail dedup window (e2e, issue #223)', () => {
|
||||
let app: INestApplication;
|
||||
let prisma: PrismaClient;
|
||||
const suffix = uniqueSuffix();
|
||||
const password = 'dedup fenster zeugen 123';
|
||||
|
||||
let ownerId: string;
|
||||
let ownerCookie: string;
|
||||
let pondId: string;
|
||||
let classifiedId: string;
|
||||
|
||||
const api = () => request(app.getHttpServer());
|
||||
const login = async () =>
|
||||
sessionCookieOf(
|
||||
await api()
|
||||
.post('/api/v1/auth/login')
|
||||
.send({ usernameOrEmail: `dedup-owner-${suffix}`, password })
|
||||
.expect(200),
|
||||
);
|
||||
const events = () =>
|
||||
prisma.readEvent.findMany({ where: { pondId }, orderBy: { occurredAt: 'asc' } });
|
||||
|
||||
beforeAll(async () => {
|
||||
prisma = createTestPrisma();
|
||||
await prisma.rateLimit.deleteMany({});
|
||||
app = await createTestApp();
|
||||
const users = app.get(UsersService);
|
||||
|
||||
const owner = await users.createUser({
|
||||
username: `dedup-owner-${suffix}`,
|
||||
email: `dedup-owner-${suffix}@example.test`,
|
||||
displayName: 'Dedup Owner',
|
||||
password,
|
||||
locale: 'en',
|
||||
});
|
||||
await users.markEmailVerified(owner.id);
|
||||
ownerId = owner.id;
|
||||
// The trail ships OFF by default (#225) — this suite needs it on.
|
||||
await app.get(InstanceSettingsService).set('readTrail.enabled', true, ownerId);
|
||||
ownerCookie = await login();
|
||||
|
||||
const pond = await prisma.pond.create({
|
||||
data: { slug: `dedup-pond-${suffix}`, name: 'Dedup Pond', type: 'SHARED', ownerId },
|
||||
});
|
||||
pondId = pond.id;
|
||||
await grantOwnerAdmin(prisma, pondId, ownerId);
|
||||
|
||||
const page = await prisma.page.create({
|
||||
data: {
|
||||
pondId,
|
||||
slug: `classified-${suffix}`,
|
||||
title: 'Classified',
|
||||
classification: 'VS_NFD',
|
||||
createdBy: ownerId,
|
||||
sortKey: 'a0',
|
||||
ydocState: new Uint8Array(),
|
||||
contentCache: {
|
||||
create: { plainText: 'x', markdown: 'x', html: '<p>x</p>', outline: [] },
|
||||
},
|
||||
},
|
||||
});
|
||||
classifiedId = page.id;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await prisma.readEvent.deleteMany({ where: { pondId } });
|
||||
await prisma.rateLimit.deleteMany({});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.enabled' } });
|
||||
await prisma.readEvent.deleteMany({ where: { pondId } });
|
||||
await prisma.attachment.deleteMany({ where: { pondId } });
|
||||
await prisma.roleGrant.deleteMany({ where: { pondId } });
|
||||
await prisma.pageContentCache.deleteMany({ where: { page: { pondId } } });
|
||||
await prisma.page.deleteMany({ where: { pondId } });
|
||||
await prisma.pond.deleteMany({ where: { id: pondId } });
|
||||
await prisma.user.deleteMany({ where: { id: ownerId } });
|
||||
await prisma.$disconnect();
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it('collapses repeated reads of one page in one session+channel to a single event that names its window', async () => {
|
||||
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
|
||||
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
|
||||
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
|
||||
const rows = await events();
|
||||
expect(rows).toHaveLength(1);
|
||||
// The row itself states that it represents a window, not a request.
|
||||
expect(rows[0]!.windowSeconds).toBe(5 * 60);
|
||||
expect(rows[0]!.dedupKey).toBe(`${rows[0]!.sessionKey}:${classifiedId}:page_view`);
|
||||
});
|
||||
|
||||
it('keeps channels apart: the same session reading and joining collab yields one event each', async () => {
|
||||
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
|
||||
await api()
|
||||
.get(`/api/v1/pages/${classifiedId}/collab-token`)
|
||||
.set('Cookie', ownerCookie)
|
||||
.expect(200);
|
||||
const rows = await events();
|
||||
expect(rows.map((r) => r.channel).sort()).toEqual(['collab_join', 'page_view']);
|
||||
});
|
||||
|
||||
it('a new session records again even for the same user — a reconnect within the session does not', async () => {
|
||||
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
|
||||
// Same session, later request ("reconnect"): deduped.
|
||||
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
|
||||
expect(await events()).toHaveLength(1);
|
||||
|
||||
const secondCookie = await login();
|
||||
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', secondCookie).expect(200);
|
||||
const rows = await events();
|
||||
expect(rows).toHaveLength(2);
|
||||
expect(new Set(rows.map((r) => r.sessionKey)).size).toBe(2);
|
||||
expect(rows.every((r) => r.actorId === ownerId)).toBe(true);
|
||||
});
|
||||
|
||||
it('bounds a live editing session: 30 collab-token renewals in one window are one event', async () => {
|
||||
for (let i = 0; i < 30; i += 1) {
|
||||
await api()
|
||||
.get(`/api/v1/pages/${classifiedId}/collab-token`)
|
||||
.set('Cookie', ownerCookie)
|
||||
.expect(200);
|
||||
}
|
||||
const rows = await events();
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]!.channel).toBe('collab_join');
|
||||
});
|
||||
|
||||
it('opens a new window when the clock moves past the bucket boundary', async () => {
|
||||
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
|
||||
const clock = app.get(ClockService);
|
||||
const later = new Date(Date.now() + 10 * 60 * 1000);
|
||||
const spy = vi.spyOn(clock, 'now').mockReturnValue(later);
|
||||
try {
|
||||
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
expect(await events()).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('dedups page-less attachment downloads on the placeholder page key', async () => {
|
||||
// A pond-level upload has no page; its effective classification falls
|
||||
// back to the pond maximum (#212) and the dedup key carries `-`.
|
||||
const uploaded = await api()
|
||||
.post(`/api/v1/ponds/${pondId}/files`)
|
||||
.set('Cookie', ownerCookie)
|
||||
.attach('file', Buffer.concat([PNG_SIGNATURE, Buffer.from('img')]), 'a.png')
|
||||
.expect(201);
|
||||
await prisma.readEvent.deleteMany({ where: { pondId } });
|
||||
|
||||
await api().get(`/api/v1/media/${uploaded.body.id}`).set('Cookie', ownerCookie).expect(200);
|
||||
await api().get(`/api/v1/media/${uploaded.body.id}`).set('Cookie', ownerCookie).expect(200);
|
||||
const rows = await events();
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]!.channel).toBe('attachment');
|
||||
expect(rows[0]!.pageId).toBeNull();
|
||||
expect(rows[0]!.dedupKey).toBe(`${rows[0]!.sessionKey}:-:attachment`);
|
||||
});
|
||||
});
|
||||
139
apps/api/src/read-trail/read-trail-maintenance.service.ts
Normal file
139
apps/api/src/read-trail/read-trail-maintenance.service.ts
Normal file
@ -0,0 +1,139 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { PinoLogger } from 'nestjs-pino';
|
||||
|
||||
import { AuditService } from '../audit/audit.service';
|
||||
import { ClockService } from '../common/clock.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { InstanceSettingsService } from '../settings/instance-settings.service';
|
||||
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
||||
|
||||
/** How many months ahead of "now" a partition must exist. Two keeps a
|
||||
* multi-week job outage from ever reaching an uncovered month (the DEFAULT
|
||||
* partition would still catch it — reads never fail on a missing month). */
|
||||
const MONTHS_AHEAD = 2;
|
||||
|
||||
/**
|
||||
* Read-trail storage maintenance (issue #224, ADR 0023), one daily job with
|
||||
* two duties:
|
||||
*
|
||||
* 1. **Partition upkeep** — `read_events` is RANGE-partitioned by month
|
||||
* (migration `20260731170000`); this creates the next {@link MONTHS_AHEAD}
|
||||
* monthly partitions, each with its per-partition dedup unique index
|
||||
* (#223 — the partitioned parent cannot carry it). A `db push` database
|
||||
* (tests) has a plain table; partition work skips itself there.
|
||||
* 2. **Retention** — events older than `readTrail.retentionDays` are
|
||||
* removed: whole months by dropping their partition (no scan), the
|
||||
* remainder (default partition, plain tables) by a ranged delete. The
|
||||
* deletion is audited (`read_trail.pruned`) so a gap in the evidence is
|
||||
* always explainable — same principle as `audit.pruned` (#196), but a
|
||||
* deliberately separate period.
|
||||
*/
|
||||
@Injectable()
|
||||
export class ReadTrailMaintenanceService {
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly settings: InstanceSettingsService,
|
||||
private readonly audit: AuditService,
|
||||
private readonly clock: ClockService,
|
||||
private readonly logger: PinoLogger,
|
||||
) {
|
||||
this.logger.setContext(ReadTrailMaintenanceService.name);
|
||||
}
|
||||
|
||||
async run(): Promise<void> {
|
||||
await this.ensurePartitions();
|
||||
await this.pruneExpired();
|
||||
}
|
||||
|
||||
/** True when read_events is a partitioned parent (relkind `p`). */
|
||||
private async isPartitioned(): Promise<boolean> {
|
||||
const rows = await this.prisma.$queryRaw<{ relkind: string }[]>`
|
||||
SELECT relkind::text FROM pg_class
|
||||
WHERE relname = 'read_events' AND relnamespace = 'public'::regnamespace`;
|
||||
return rows[0]?.relkind === 'p';
|
||||
}
|
||||
|
||||
/** `read_events_y2026m08` for 2026-08. */
|
||||
private partitionName(month: Date): string {
|
||||
const y = month.getUTCFullYear();
|
||||
const m = String(month.getUTCMonth() + 1).padStart(2, '0');
|
||||
return `read_events_y${y}m${m}`;
|
||||
}
|
||||
|
||||
private monthStart(base: Date, offsetMonths: number): Date {
|
||||
return new Date(Date.UTC(base.getUTCFullYear(), base.getUTCMonth() + offsetMonths, 1));
|
||||
}
|
||||
|
||||
async ensurePartitions(): Promise<void> {
|
||||
if (!(await this.isPartitioned())) {
|
||||
this.logger.debug('read_events is not partitioned here; skipping partition upkeep');
|
||||
return;
|
||||
}
|
||||
const now = this.clock.now();
|
||||
for (let offset = 0; offset <= MONTHS_AHEAD; offset += 1) {
|
||||
const from = this.monthStart(now, offset);
|
||||
const to = this.monthStart(now, offset + 1);
|
||||
const name = this.partitionName(from);
|
||||
try {
|
||||
await this.prisma.$executeRawUnsafe(
|
||||
`CREATE TABLE IF NOT EXISTS "${name}" PARTITION OF "read_events"
|
||||
FOR VALUES FROM ('${from.toISOString()}') TO ('${to.toISOString()}')`,
|
||||
);
|
||||
await this.prisma.$executeRawUnsafe(
|
||||
`CREATE UNIQUE INDEX IF NOT EXISTS "${name}_dedup_key"
|
||||
ON "${name}" ("dedup_key", "window_bucket")`,
|
||||
);
|
||||
} catch (error) {
|
||||
// Most likely: the DEFAULT partition already holds rows of this month
|
||||
// (the job lagged past a month boundary). Nothing is lost — those
|
||||
// rows live in the default partition and age out through the ranged
|
||||
// delete below; the month just cannot get its own partition anymore.
|
||||
this.logger.warn({ partition: name, err: error }, 'read-trail partition not created');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async pruneExpired(): Promise<number> {
|
||||
const retentionDays = await this.settings.get('readTrail.retentionDays');
|
||||
const cutoff = new Date(this.clock.now().getTime() - retentionDays * MS_PER_DAY);
|
||||
let dropped = 0;
|
||||
|
||||
if (await this.isPartitioned()) {
|
||||
// Whole months strictly before the cutoff month go by DROP — no scan,
|
||||
// and the dropped range is exact (every row in them is < cutoff).
|
||||
const partitions = await this.prisma.$queryRaw<{ relname: string }[]>`
|
||||
SELECT c.relname::text
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = 'read_events'::regclass
|
||||
AND c.relname ~ '^read_events_y[0-9]{4}m[0-9]{2}$'`;
|
||||
const cutoffMonth = this.monthStart(cutoff, 0);
|
||||
for (const { relname } of partitions) {
|
||||
const match = /^read_events_y(\d{4})m(\d{2})$/.exec(relname);
|
||||
if (!match) continue;
|
||||
const monthEnd = new Date(Date.UTC(Number(match[1]), Number(match[2]), 1));
|
||||
if (monthEnd.getTime() > cutoffMonth.getTime()) continue;
|
||||
const counted = await this.prisma.$queryRawUnsafe<{ count: bigint }[]>(
|
||||
`SELECT count(*)::bigint AS count FROM "${relname}"`,
|
||||
);
|
||||
dropped += Number(counted[0]?.count ?? 0n);
|
||||
await this.prisma.$executeRawUnsafe(`DROP TABLE "${relname}"`);
|
||||
}
|
||||
}
|
||||
|
||||
// The remainder: rows before the cutoff inside surviving partitions,
|
||||
// the default partition, or a plain (test) table.
|
||||
const deleted = await this.prisma.readEvent.deleteMany({
|
||||
where: { occurredAt: { lt: cutoff } },
|
||||
});
|
||||
const count = dropped + deleted.count;
|
||||
if (count > 0) {
|
||||
await this.audit.record({
|
||||
action: 'read_trail.pruned',
|
||||
details: { count, cutoff: cutoff.toISOString(), retentionDays },
|
||||
});
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
308
apps/api/src/read-trail/read-trail-storage.e2e.db.test.ts
Normal file
308
apps/api/src/read-trail/read-trail-storage.e2e.db.test.ts
Normal file
@ -0,0 +1,308 @@
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { PinoLogger } from 'nestjs-pino';
|
||||
import request from 'supertest';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { ClockService } from '../common/clock.service';
|
||||
import { InstanceSettingsService } from '../settings/instance-settings.service';
|
||||
import { createTestApp, sessionCookieOf } from '../testing/test-app';
|
||||
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
|
||||
import { UsersService } from '../users/users.service';
|
||||
|
||||
import { ReadTrailMaintenanceService } from './read-trail-maintenance.service';
|
||||
|
||||
const API_ROOT = join(__dirname, '..', '..');
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Read-trail storage (issue #224, ADR 0023): its own retention period with
|
||||
* an audited deletion, the Site-Admin query path, and — against a freshly
|
||||
* migrated database, where the real DDL applies — the monthly RANGE
|
||||
* partitioning with per-partition dedup indexes and DROP-based pruning.
|
||||
*/
|
||||
describe.skipIf(!hasTestDb)('read-trail storage (e2e, issue #224)', () => {
|
||||
const suffix = uniqueSuffix();
|
||||
|
||||
describe('retention and admin query path (shared database)', () => {
|
||||
let app: INestApplication;
|
||||
let prisma: PrismaClient;
|
||||
const password = 'lesetrail speicher 123';
|
||||
|
||||
let adminId: string;
|
||||
let adminCookie: string;
|
||||
let readerCookie: string;
|
||||
// Plain text ids — read_events has no FKs by design (#222).
|
||||
const pondId = `pond-${suffix}`;
|
||||
|
||||
const api = () => request(app.getHttpServer());
|
||||
|
||||
async function makeUser(handle: string, siteAdmin: boolean) {
|
||||
const users = app.get(UsersService);
|
||||
const user = await users.createUser({
|
||||
username: `${handle}-${suffix}`,
|
||||
email: `${handle}-${suffix}@example.test`,
|
||||
displayName: handle,
|
||||
password,
|
||||
locale: 'en',
|
||||
});
|
||||
await users.markEmailVerified(user.id);
|
||||
if (siteAdmin) {
|
||||
await prisma.user.update({ where: { id: user.id }, data: { isSiteAdmin: true } });
|
||||
}
|
||||
const cookie = sessionCookieOf(
|
||||
await api()
|
||||
.post('/api/v1/auth/login')
|
||||
.send({ usernameOrEmail: `${handle}-${suffix}`, password })
|
||||
.expect(200),
|
||||
);
|
||||
return { id: user.id, cookie };
|
||||
}
|
||||
|
||||
function eventRow(overrides: Record<string, unknown>) {
|
||||
return {
|
||||
actorId: null,
|
||||
sessionKey: 'anon',
|
||||
pageId: null,
|
||||
pondId,
|
||||
channel: 'page_view',
|
||||
classification: 'vs_nfd',
|
||||
dedupKey: `k-${suffix}-${Math.random().toString(36).slice(2)}`,
|
||||
windowBucket: BigInt(Math.floor(Math.random() * 1_000_000_000)),
|
||||
windowSeconds: 300,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
prisma = createTestPrisma();
|
||||
await prisma.rateLimit.deleteMany({});
|
||||
app = await createTestApp();
|
||||
const admin = await makeUser('storage-admin', true);
|
||||
adminId = admin.id;
|
||||
adminCookie = admin.cookie;
|
||||
const reader = await makeUser('storage-reader', false);
|
||||
readerCookie = reader.cookie;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await prisma.readEvent.deleteMany({ where: { pondId } });
|
||||
await prisma.auditEntry.deleteMany({ where: { action: 'read_trail.pruned' } });
|
||||
await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.retentionDays' } });
|
||||
await prisma.user.deleteMany({ where: { username: { contains: suffix } } });
|
||||
await prisma.$disconnect();
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it('prunes only events past its own period and audits the deletion', async () => {
|
||||
await app.get(InstanceSettingsService).set('readTrail.retentionDays', 30, adminId);
|
||||
const now = Date.now();
|
||||
await prisma.readEvent.createMany({
|
||||
data: [
|
||||
eventRow({ occurredAt: new Date(now - 40 * MS_PER_DAY) }),
|
||||
eventRow({ occurredAt: new Date(now - 31 * MS_PER_DAY) }),
|
||||
eventRow({ occurredAt: new Date(now - 5 * MS_PER_DAY) }),
|
||||
],
|
||||
});
|
||||
|
||||
const maintenance = app.get(ReadTrailMaintenanceService);
|
||||
const pruned = await maintenance.pruneExpired();
|
||||
expect(pruned).toBeGreaterThanOrEqual(2);
|
||||
|
||||
const remaining = await prisma.readEvent.findMany({ where: { pondId } });
|
||||
expect(remaining).toHaveLength(1);
|
||||
|
||||
const audit = await prisma.auditEntry.findFirst({
|
||||
where: { action: 'read_trail.pruned' },
|
||||
orderBy: { at: 'desc' },
|
||||
});
|
||||
expect(audit).not.toBeNull();
|
||||
expect(audit!.details).toMatchObject({ retentionDays: 30 });
|
||||
});
|
||||
|
||||
it('answers "who read page X" and "what did user Y read" for Site Admins only', async () => {
|
||||
const pageId = `page-${suffix}`;
|
||||
await prisma.readEvent.createMany({
|
||||
data: [
|
||||
eventRow({ pageId, actorId: adminId, channel: 'export' }),
|
||||
eventRow({ pageId: `other-${suffix}`, actorId: null }),
|
||||
],
|
||||
});
|
||||
|
||||
const byPage = await api()
|
||||
.get(`/api/v1/admin/system/read-events?pageId=${pageId}`)
|
||||
.set('Cookie', adminCookie)
|
||||
.expect(200);
|
||||
expect(byPage.body.total).toBe(1);
|
||||
expect(byPage.body.entries[0]).toMatchObject({
|
||||
pageId,
|
||||
channel: 'export',
|
||||
windowSeconds: 300,
|
||||
});
|
||||
expect(byPage.body.entries[0].actor).toMatchObject({ id: adminId });
|
||||
|
||||
const byActor = await api()
|
||||
.get(`/api/v1/admin/system/read-events?actor=storage-admin-${suffix}`)
|
||||
.set('Cookie', adminCookie)
|
||||
.expect(200);
|
||||
expect(byActor.body.total).toBe(1);
|
||||
|
||||
// An unknown username matches nothing rather than everything.
|
||||
const unknown = await api()
|
||||
.get(`/api/v1/admin/system/read-events?actor=nobody-${suffix}`)
|
||||
.set('Cookie', adminCookie)
|
||||
.expect(200);
|
||||
expect(unknown.body.total).toBe(0);
|
||||
|
||||
await api()
|
||||
.get(`/api/v1/admin/system/read-events?pageId=${pageId}`)
|
||||
.set('Cookie', readerCookie)
|
||||
.expect(403);
|
||||
});
|
||||
});
|
||||
|
||||
describe('partitioned shape (fresh database, real migrations)', () => {
|
||||
const baseUrl = process.env.TEST_DATABASE_URL!;
|
||||
const dbName = `dorfteich_trail_${suffix}`;
|
||||
let fresh: PrismaClient;
|
||||
let maintenance: ReadTrailMaintenanceService;
|
||||
const auditRecord = vi.fn().mockResolvedValue(undefined);
|
||||
let retentionDays = 365;
|
||||
|
||||
function freshUrl(): string {
|
||||
const url = new URL(baseUrl);
|
||||
url.pathname = `/${dbName}`;
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
const admin = new PrismaClient({ datasourceUrl: baseUrl });
|
||||
try {
|
||||
await admin.$executeRawUnsafe(`CREATE DATABASE "${dbName}"`);
|
||||
} finally {
|
||||
await admin.$disconnect();
|
||||
}
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[join(API_ROOT, 'node_modules', 'prisma', 'build', 'index.js'), 'migrate', 'deploy'],
|
||||
{ env: { ...process.env, DATABASE_URL: freshUrl() }, stdio: 'pipe', cwd: API_ROOT },
|
||||
);
|
||||
fresh = new PrismaClient({ datasourceUrl: freshUrl() });
|
||||
const settingsStub = {
|
||||
get: async () => retentionDays,
|
||||
} as unknown as InstanceSettingsService;
|
||||
const auditStub = { record: auditRecord } as never;
|
||||
const loggerStub = {
|
||||
setContext: () => undefined,
|
||||
debug: () => undefined,
|
||||
info: () => undefined,
|
||||
warn: () => undefined,
|
||||
error: () => undefined,
|
||||
} as unknown as PinoLogger;
|
||||
maintenance = new ReadTrailMaintenanceService(
|
||||
fresh as never,
|
||||
settingsStub,
|
||||
auditStub,
|
||||
new ClockService(),
|
||||
loggerStub,
|
||||
);
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await fresh.$disconnect();
|
||||
const admin = new PrismaClient({ datasourceUrl: baseUrl });
|
||||
try {
|
||||
await admin.$executeRawUnsafe(`DROP DATABASE IF EXISTS "${dbName}" WITH (FORCE)`);
|
||||
} finally {
|
||||
await admin.$disconnect();
|
||||
}
|
||||
});
|
||||
|
||||
it('migrates read_events to a partitioned parent with current-month coverage', async () => {
|
||||
const kind = await fresh.$queryRaw<{ relkind: string }[]>`
|
||||
SELECT relkind::text FROM pg_class WHERE relname = 'read_events'`;
|
||||
expect(kind[0]!.relkind).toBe('p');
|
||||
|
||||
const partitions = await fresh.$queryRaw<{ relname: string }[]>`
|
||||
SELECT c.relname::text FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = 'read_events'::regclass ORDER BY c.relname`;
|
||||
const names = partitions.map((p) => p.relname);
|
||||
expect(names).toContain('read_events_default');
|
||||
expect(names.some((n) => /^read_events_y\d{4}m\d{2}$/.test(n))).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps the dedup unique pair enforced per partition (P2002 on the duplicate)', async () => {
|
||||
const row = {
|
||||
actorId: null,
|
||||
sessionKey: 'anon',
|
||||
pageId: null,
|
||||
pondId: 'pond-part',
|
||||
channel: 'page_view',
|
||||
classification: 'vs_nfd',
|
||||
dedupKey: `dup-${suffix}`,
|
||||
windowBucket: 42n,
|
||||
windowSeconds: 300,
|
||||
};
|
||||
await fresh.readEvent.create({ data: row });
|
||||
await expect(fresh.readEvent.create({ data: { ...row } })).rejects.toMatchObject({
|
||||
code: 'P2002',
|
||||
});
|
||||
});
|
||||
|
||||
it('creates months ahead and prunes whole expired partitions by DROP, audited', async () => {
|
||||
await maintenance.ensurePartitions();
|
||||
const next = new Date();
|
||||
const nextMonth = new Date(Date.UTC(next.getUTCFullYear(), next.getUTCMonth() + 2, 1));
|
||||
const nextName = `read_events_y${nextMonth.getUTCFullYear()}m${String(
|
||||
nextMonth.getUTCMonth() + 1,
|
||||
).padStart(2, '0')}`;
|
||||
const created = await fresh.$queryRawUnsafe<{ relname: string }[]>(
|
||||
`SELECT relname::text FROM pg_class WHERE relname = '${nextName}'`,
|
||||
);
|
||||
expect(created).toHaveLength(1);
|
||||
// ...and each new partition carries its own dedup unique index.
|
||||
const index = await fresh.$queryRawUnsafe<{ indexname: string }[]>(
|
||||
`SELECT indexname::text FROM pg_indexes WHERE tablename = '${nextName}'
|
||||
AND indexname = '${nextName}_dedup_key'`,
|
||||
);
|
||||
expect(index).toHaveLength(1);
|
||||
|
||||
// An old month: partition + one event well past retention.
|
||||
await fresh.$executeRawUnsafe(
|
||||
`CREATE TABLE "read_events_y2020m01" PARTITION OF "read_events"
|
||||
FOR VALUES FROM ('2020-01-01') TO ('2020-02-01')`,
|
||||
);
|
||||
await fresh.readEvent.create({
|
||||
data: {
|
||||
occurredAt: new Date('2020-01-15T12:00:00Z'),
|
||||
actorId: null,
|
||||
sessionKey: 'anon',
|
||||
pageId: null,
|
||||
pondId: 'pond-part',
|
||||
channel: 'export',
|
||||
classification: 'vs_nfd',
|
||||
dedupKey: `old-${suffix}`,
|
||||
windowBucket: 1n,
|
||||
windowSeconds: 300,
|
||||
},
|
||||
});
|
||||
|
||||
const pruned = await maintenance.pruneExpired();
|
||||
expect(pruned).toBeGreaterThanOrEqual(1);
|
||||
const gone = await fresh.$queryRawUnsafe<{ relname: string }[]>(
|
||||
`SELECT relname::text FROM pg_class WHERE relname = 'read_events_y2020m01'`,
|
||||
);
|
||||
expect(gone).toHaveLength(0);
|
||||
expect(auditRecord).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ action: 'read_trail.pruned' }),
|
||||
);
|
||||
// The current month's events survive.
|
||||
const kept = await fresh.readEvent.count({ where: { dedupKey: `dup-${suffix}` } });
|
||||
expect(kept).toBe(1);
|
||||
retentionDays = 365; // restore for any later use
|
||||
});
|
||||
});
|
||||
});
|
||||
132
apps/api/src/read-trail/read-trail-switch.e2e.db.test.ts
Normal file
132
apps/api/src/read-trail/read-trail-switch.e2e.db.test.ts
Normal file
@ -0,0 +1,132 @@
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { PinoLogger } from 'nestjs-pino';
|
||||
import request from 'supertest';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { InstanceSettingsService } from '../settings/instance-settings.service';
|
||||
import { createTestApp, sessionCookieOf } from '../testing/test-app';
|
||||
import { createTestPrisma, grantOwnerAdmin, hasTestDb, uniqueSuffix } from '../testing/test-db';
|
||||
import { UsersService } from '../users/users.service';
|
||||
|
||||
import { ReadTrailService } from './read-trail.service';
|
||||
|
||||
/**
|
||||
* The read-trail master switch (issue #225, ADR 0023): OFF is the default
|
||||
* and means no event is written ANYWHERE — no row, no stdout line; ON
|
||||
* restores the full #222 semantics. The switch position is announced so a
|
||||
* silent trail is never ambiguous.
|
||||
*/
|
||||
describe.skipIf(!hasTestDb)('read-trail switch (e2e, issue #225)', () => {
|
||||
let app: INestApplication;
|
||||
let prisma: PrismaClient;
|
||||
const suffix = uniqueSuffix();
|
||||
const password = 'schalter zeugen 123';
|
||||
|
||||
let ownerId: string;
|
||||
let ownerCookie: string;
|
||||
let pondId: string;
|
||||
let classifiedId: string;
|
||||
|
||||
const api = () => request(app.getHttpServer());
|
||||
|
||||
beforeAll(async () => {
|
||||
prisma = createTestPrisma();
|
||||
await prisma.rateLimit.deleteMany({});
|
||||
// No leftover switch row: this suite tests the DEFAULT (off).
|
||||
await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.enabled' } });
|
||||
app = await createTestApp();
|
||||
const users = app.get(UsersService);
|
||||
|
||||
const owner = await users.createUser({
|
||||
username: `switch-owner-${suffix}`,
|
||||
email: `switch-owner-${suffix}@example.test`,
|
||||
displayName: 'Switch Owner',
|
||||
password,
|
||||
locale: 'en',
|
||||
});
|
||||
await users.markEmailVerified(owner.id);
|
||||
ownerId = owner.id;
|
||||
ownerCookie = sessionCookieOf(
|
||||
await api()
|
||||
.post('/api/v1/auth/login')
|
||||
.send({ usernameOrEmail: `switch-owner-${suffix}`, password })
|
||||
.expect(200),
|
||||
);
|
||||
|
||||
const pond = await prisma.pond.create({
|
||||
data: { slug: `switch-pond-${suffix}`, name: 'Switch Pond', type: 'SHARED', ownerId },
|
||||
});
|
||||
pondId = pond.id;
|
||||
await grantOwnerAdmin(prisma, pondId, ownerId);
|
||||
const page = await prisma.page.create({
|
||||
data: {
|
||||
pondId,
|
||||
slug: `classified-${suffix}`,
|
||||
title: 'Classified',
|
||||
classification: 'VS_NFD',
|
||||
createdBy: ownerId,
|
||||
sortKey: 'a0',
|
||||
ydocState: new Uint8Array(),
|
||||
contentCache: {
|
||||
create: { plainText: 'x', markdown: 'x', html: '<p>x</p>', outline: [] },
|
||||
},
|
||||
},
|
||||
});
|
||||
classifiedId = page.id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.enabled' } });
|
||||
await prisma.readEvent.deleteMany({ where: { pondId } });
|
||||
await prisma.roleGrant.deleteMany({ where: { pondId } });
|
||||
await prisma.pageContentCache.deleteMany({ where: { page: { pondId } } });
|
||||
await prisma.page.deleteMany({ where: { pondId } });
|
||||
await prisma.pond.deleteMany({ where: { id: pondId } });
|
||||
await prisma.user.deleteMany({ where: { id: ownerId } });
|
||||
await prisma.$disconnect();
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it('is OFF by default: a classified read writes nothing — no row, no stdout line', async () => {
|
||||
const trail = app.get(ReadTrailService);
|
||||
const logger = (trail as unknown as { logger: PinoLogger }).logger;
|
||||
const infoSpy = vi.spyOn(logger, 'info');
|
||||
try {
|
||||
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
|
||||
} finally {
|
||||
infoSpy.mockRestore();
|
||||
}
|
||||
expect(await prisma.readEvent.count({ where: { pondId } })).toBe(0);
|
||||
expect(infoSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('records again the moment the switch turns on', async () => {
|
||||
await app.get(InstanceSettingsService).set('readTrail.enabled', true, ownerId);
|
||||
try {
|
||||
await api().get(`/api/v1/pages/${classifiedId}`).set('Cookie', ownerCookie).expect(200);
|
||||
expect(await prisma.readEvent.count({ where: { pondId } })).toBe(1);
|
||||
} finally {
|
||||
await app.get(InstanceSettingsService).set('readTrail.enabled', false, ownerId);
|
||||
}
|
||||
});
|
||||
|
||||
it('announces the switch position so silence is never ambiguous', async () => {
|
||||
const trail = app.get(ReadTrailService);
|
||||
const logger = (trail as unknown as { logger: PinoLogger }).logger;
|
||||
const warnSpy = vi.spyOn(logger, 'warn');
|
||||
const infoSpy = vi.spyOn(logger, 'info');
|
||||
try {
|
||||
await trail.announceState(); // switch is off after the previous test
|
||||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('NOT evidenced'));
|
||||
|
||||
await app.get(InstanceSettingsService).set('readTrail.enabled', true, ownerId);
|
||||
await trail.announceState();
|
||||
expect(infoSpy).toHaveBeenCalledWith(expect.stringContaining('enabled'));
|
||||
} finally {
|
||||
warnSpy.mockRestore();
|
||||
infoSpy.mockRestore();
|
||||
await app.get(InstanceSettingsService).set('readTrail.enabled', false, ownerId);
|
||||
}
|
||||
});
|
||||
});
|
||||
@ -74,6 +74,9 @@ describe.skipIf(!hasTestDb)('read-access trail (e2e, issue #222)', () => {
|
||||
});
|
||||
await users.markEmailVerified(owner.id);
|
||||
ownerId = owner.id;
|
||||
// The trail ships OFF by default (#225) — these suites test the
|
||||
// instrumented channels, so they run with the switch on.
|
||||
await app.get(InstanceSettingsService).set('readTrail.enabled', true, ownerId);
|
||||
ownerCookie = sessionCookieOf(
|
||||
await api()
|
||||
.post('/api/v1/auth/login')
|
||||
@ -131,7 +134,9 @@ describe.skipIf(!hasTestDb)('read-access trail (e2e, issue #222)', () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await prisma.instanceSetting.deleteMany({ where: { key: 'api.enabled' } });
|
||||
await prisma.instanceSetting.deleteMany({
|
||||
where: { key: { in: ['api.enabled', 'readTrail.enabled'] } },
|
||||
});
|
||||
await prisma.readEvent.deleteMany({ where: { pondId } });
|
||||
await prisma.conversionJob.deleteMany({ where: { ownerId } });
|
||||
await prisma.apiToken.deleteMany({ where: { userId: ownerId } });
|
||||
|
||||
@ -1,7 +1,16 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { Global, Module, OnModuleInit } from '@nestjs/common';
|
||||
|
||||
import { CommonModule } from '../common/common.module';
|
||||
import { SchedulerModule } from '../scheduler/scheduler.module';
|
||||
import { SchedulerService } from '../scheduler/scheduler.service';
|
||||
import { SettingsModule } from '../settings/settings.module';
|
||||
|
||||
import { ReadTrailMaintenanceService } from './read-trail-maintenance.service';
|
||||
import { ReadTrailService } from './read-trail.service';
|
||||
|
||||
/** Daily, per operations.md's maintenance-jobs table (issue #224). */
|
||||
const READ_TRAIL_MAINTENANCE_CADENCE_SECONDS = 24 * 60 * 60;
|
||||
|
||||
/**
|
||||
* Global like AuditModule and for the same reason: the read-access trail
|
||||
* (issue #222, ADR 0023) cuts across every module that serves page content —
|
||||
@ -9,7 +18,27 @@ import { ReadTrailService } from './read-trail.service';
|
||||
*/
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [ReadTrailService],
|
||||
exports: [ReadTrailService],
|
||||
imports: [CommonModule, SchedulerModule, SettingsModule],
|
||||
providers: [ReadTrailService, ReadTrailMaintenanceService],
|
||||
exports: [ReadTrailService, ReadTrailMaintenanceService],
|
||||
})
|
||||
export class ReadTrailModule {}
|
||||
export class ReadTrailModule implements OnModuleInit {
|
||||
constructor(
|
||||
private readonly scheduler: SchedulerService,
|
||||
private readonly maintenance: ReadTrailMaintenanceService,
|
||||
private readonly trail: ReadTrailService,
|
||||
) {}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
this.scheduler.register({
|
||||
name: 'read-trail-maintenance',
|
||||
cadenceSeconds: READ_TRAIL_MAINTENANCE_CADENCE_SECONDS,
|
||||
run: async () => {
|
||||
await this.maintenance.run();
|
||||
},
|
||||
});
|
||||
// One line per boot stating the switch position (issue #225) — a silent
|
||||
// trail must never be ambiguous.
|
||||
await this.trail.announceState();
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,7 +2,9 @@ import { Injectable } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { PinoLogger } from 'nestjs-pino';
|
||||
|
||||
import { ClockService } from '../common/clock.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { InstanceSettingsService } from '../settings/instance-settings.service';
|
||||
|
||||
/**
|
||||
* Every read surface classified content can leave through (issue #222,
|
||||
@ -61,33 +63,83 @@ export interface ReadEventInput extends ReadActor {
|
||||
* swallowed. A lost event is a gap in evidence, so a failed write aborts
|
||||
* the read with the ordinary 500 — the reader retries, the evidence stays
|
||||
* complete (decision recorded in ADR 0023 and security.md).
|
||||
*
|
||||
* Dedup window (issue #223): one event per (session, page, channel) within
|
||||
* an aligned window of `readTrail.dedupWindowMinutes` (default 5). Buckets
|
||||
* are `floor(epoch / windowSeconds)`, and the unique
|
||||
* (dedupKey, windowBucket) pair collapses concurrent duplicates race-free:
|
||||
* the first insert wins, every later one lands on P2002 and is skipped —
|
||||
* a skipped DUPLICATE is not a gap, so it must not abort the read. The row
|
||||
* carries `windowSeconds`, so each event states that it represents up to
|
||||
* that many seconds of access, not a single request.
|
||||
*/
|
||||
@Injectable()
|
||||
export class ReadTrailService {
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly settings: InstanceSettingsService,
|
||||
private readonly clock: ClockService,
|
||||
private readonly logger: PinoLogger,
|
||||
) {
|
||||
this.logger.setContext(ReadTrailService.name);
|
||||
}
|
||||
|
||||
/** States the switch position once at startup (issue #225): a trail
|
||||
* without events must be distinguishable from a disabled trail — the log
|
||||
* line makes the gap explainable either way. Must never fail the boot:
|
||||
* healthz-only environments come up without a reachable database. */
|
||||
async announceState(): Promise<void> {
|
||||
let enabled: boolean;
|
||||
try {
|
||||
enabled = await this.settings.get('readTrail.enabled');
|
||||
} catch {
|
||||
this.logger.warn('read_trail: switch position unknown at startup (settings unavailable)');
|
||||
return;
|
||||
}
|
||||
if (enabled) {
|
||||
this.logger.info('read_trail: enabled — reads of classified pages are recorded');
|
||||
} else {
|
||||
this.logger.warn(
|
||||
'read_trail: disabled — reads of classified pages are NOT evidenced (readTrail.enabled)',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async record(event: ReadEventInput): Promise<void> {
|
||||
// The master switch (issue #225): off means nothing is written anywhere
|
||||
// — no row, no stdout line. The startup announcement above is what keeps
|
||||
// the resulting silence unambiguous.
|
||||
if (!(await this.settings.get('readTrail.enabled'))) return;
|
||||
const { actorId, sessionKey, pageId, pondId, channel, details } = event;
|
||||
await this.prisma.readEvent.create({
|
||||
data: {
|
||||
actorId,
|
||||
sessionKey,
|
||||
pageId,
|
||||
pondId,
|
||||
channel,
|
||||
classification: 'vs_nfd',
|
||||
details: details ? (details as Prisma.InputJsonObject) : undefined,
|
||||
},
|
||||
});
|
||||
const windowSeconds = (await this.settings.get('readTrail.dedupWindowMinutes')) * 60;
|
||||
const dedupKey = `${sessionKey}:${pageId ?? '-'}:${channel}`;
|
||||
const windowBucket = BigInt(Math.floor(this.clock.now().getTime() / 1000 / windowSeconds));
|
||||
try {
|
||||
await this.prisma.readEvent.create({
|
||||
data: {
|
||||
actorId,
|
||||
sessionKey,
|
||||
pageId,
|
||||
pondId,
|
||||
channel,
|
||||
classification: 'vs_nfd',
|
||||
details: details ? (details as Prisma.InputJsonObject) : undefined,
|
||||
dedupKey,
|
||||
windowBucket,
|
||||
windowSeconds,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
const isDuplicate =
|
||||
error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002';
|
||||
if (!isDuplicate) throw error;
|
||||
this.logger.debug({ dedupKey, channel }, 'read_trail: deduped within window');
|
||||
return;
|
||||
}
|
||||
// The stdout line mirrors the row (SIEM forwarding beyond this is out of
|
||||
// scope, #224); it fires only after the row is safely persisted.
|
||||
this.logger.info(
|
||||
{ actor: actorId, sessionKey, pageId, pondId, channel },
|
||||
{ actor: actorId, sessionKey, pageId, pondId, channel, windowSeconds },
|
||||
'read_trail: classified page read',
|
||||
);
|
||||
}
|
||||
|
||||
@ -51,6 +51,26 @@ export const INSTANCE_SETTINGS = {
|
||||
// bounded. PENDING rows — including failed-but-retryable ones — are
|
||||
// never touched; the retry loop owns them.
|
||||
'mail.outboxRetentionDays': z.number().int().min(1).default(30),
|
||||
// Read-trail master switch (issue #225, ADR 0023). Default OFF: read
|
||||
// logging is employee monitoring in a works council's eyes — an ordinary
|
||||
// instance must not surveil reads. The VS-NfD reference configuration
|
||||
// (#227) turns it on together with the written purpose limitation
|
||||
// (60-sicherheitsdokumentation.md §7). Off means NO event is written
|
||||
// anywhere, including stdout; the api states the switch position once at
|
||||
// startup, so a gap in the evidence is never ambiguous.
|
||||
'readTrail.enabled': z.boolean().default(false),
|
||||
// Read-trail dedup window (issue #223, ADR 0023): one event per
|
||||
// (session, page, channel) within an aligned window of this many minutes.
|
||||
// 5 minutes keeps a live Yjs session (collab tokens every 60 s) at a
|
||||
// bounded ~12 events/hour/page while still evidencing distinct visits.
|
||||
'readTrail.dedupWindowMinutes': z.number().int().min(1).default(5),
|
||||
// Read-trail retention (issue #224): days a read event is kept before the
|
||||
// daily maintenance job removes it — deliberately independent of
|
||||
// `audit.retentionDays` (#196), because volume, purpose and legal basis
|
||||
// differ. The deletion itself is audited (`read_trail.pruned`), so a gap
|
||||
// is always explainable. One year mirrors the audit default; shortening
|
||||
// it is an operator decision under the purpose limitation (#225).
|
||||
'readTrail.retentionDays': z.number().int().min(1).default(365),
|
||||
// Default VS-NfD classification for newly created pages (ADR 0022,
|
||||
// issue #204). An instance operated inside a classified environment sets
|
||||
// this to `vs_nfd` so nothing starts unmarked; inheritance from the
|
||||
|
||||
@ -20,8 +20,9 @@ test('lists maintenance jobs and triggers one manually', async ({ browser }) =>
|
||||
// Keep in sync with the scheduler registrations: trash-purge,
|
||||
// version-thinning, page-compaction, data-export-purge,
|
||||
// notification-digest, orphan-file-sweep (#194), audit-retention (#196),
|
||||
// conversion-payload-prune (#233), mail-outbox-retention (#234).
|
||||
await expect(jobsTable.locator('tbody tr')).toHaveCount(9);
|
||||
// conversion-payload-prune (#233), mail-outbox-retention (#234),
|
||||
// read-trail-maintenance (#224).
|
||||
await expect(jobsTable.locator('tbody tr')).toHaveCount(10);
|
||||
|
||||
const firstRow = jobsTable.locator('tbody tr').first();
|
||||
await firstRow.getByRole('button').click();
|
||||
|
||||
@ -67,6 +67,61 @@ traffic per open document.
|
||||
also the MCP `read_page` path), `job:<id>` (background builds such as the
|
||||
account data export), `anon` (anonymous reader on a public grant).
|
||||
|
||||
## Decisions taken in #223
|
||||
|
||||
- **Aligned dedup windows.** One event per (session, page, channel) within
|
||||
an aligned window of `readTrail.dedupWindowMinutes` (default 5):
|
||||
buckets are `floor(epoch / windowSeconds)`, and a unique
|
||||
(dedupKey, windowBucket) pair collapses concurrent duplicates race-free
|
||||
at insert time. Sliding windows (measured from the first event) were
|
||||
rejected — they need a read-before-write and lose the race-safety of the
|
||||
plain unique insert. Consequence: two reads just across a bucket
|
||||
boundary yield two events; over-recording is acceptable, gaps are not.
|
||||
- **A skipped duplicate is not a gap** — the unique-violation path returns
|
||||
quietly (debug log), only real write failures abort the read.
|
||||
- **Each row states its window** (`windowSeconds`), so the evidence reads
|
||||
as "accessed at least once in these N minutes", never as a request
|
||||
count. What the trail can prove about a live session: the per-minute
|
||||
collab-token renewals collapse to one `collab_join` event per window —
|
||||
presence during the window, not activity within it.
|
||||
- **Anonymous readers share one `anon` key**: all anonymous reads of a page
|
||||
through one channel inside a window are one event. Deliberate — telling
|
||||
anonymous browsers apart would require fingerprinting (IP/UA), which the
|
||||
purpose limitation (#225) rules out.
|
||||
|
||||
## Decisions taken in #224
|
||||
|
||||
- **Monthly RANGE partitions** on `occurred_at`, plus a DEFAULT partition
|
||||
as a safety net: a lagging maintenance job must never make classified
|
||||
reads fail (the trail write is hard-failing — an ops miss must not
|
||||
become an outage). Retention DROPs whole expired months without
|
||||
scanning; the pruning run is audited (`read_trail.pruned`) under the
|
||||
trail's own `readTrail.retentionDays` (default 365).
|
||||
- **The dedup unique pair lives per partition** (PostgreSQL cannot carry
|
||||
it on the parent without the partition key). A dedup bucket spanning a
|
||||
month boundary can therefore record one duplicate — over-recording is
|
||||
acceptable, gaps are not.
|
||||
- **Query path is API-only** (`GET /admin/system/read-events`, Site
|
||||
Admin): the trail is an examiner's tool, not a daily screen. Evidence
|
||||
nobody can read is not evidence — the path exists; a panel does not.
|
||||
- **Growth measured**: ~1 MB per 1000 events including indexes.
|
||||
|
||||
## Decisions taken in #225
|
||||
|
||||
- **Default OFF** (`readTrail.enabled`): read logging is employee
|
||||
monitoring in a works council's eyes — an ordinary instance must not
|
||||
surveil reads. The VS-NfD reference configuration turns it on together
|
||||
with the written purpose limitation
|
||||
(`docs/vs-nfd/60-sicherheitsdokumentation.md` §7, referenced from the
|
||||
hardening guide).
|
||||
- **Off means silence everywhere** — no row, no stdout line; the api
|
||||
announces the switch position once per boot, so an eventless trail is
|
||||
never ambiguous (a gap reads as "was off", not "was lost").
|
||||
- The purpose limitation names what is recorded, why, who may read it
|
||||
(Site Admin, API-only), for how long, and what it may NOT be used for
|
||||
(no performance or behaviour monitoring). Its technical anchor is the
|
||||
variant-A scope: unmarked content produces no trace.
|
||||
|
||||
## Consequences
|
||||
|
||||
- The scope limit is the feature's strongest argument in the works-council
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# Audit event catalogue
|
||||
|
||||
**Catalogue version 1.1 (2026-07-31; 1.1 adds `page.classification_*`,
|
||||
issue #205).**
|
||||
**Catalogue version 1.2 (2026-07-31; 1.2 adds `read_trail.pruned`,
|
||||
issue #224; 1.1 added `page.classification_*`, issue #205).**
|
||||
|
||||
This is the operator-facing contract for the audit trail: every event id
|
||||
the application can emit, with its trigger, severity, actor/target
|
||||
@ -103,11 +103,12 @@ failure), `warning` = feeds detection (suspicious or destructive),
|
||||
|
||||
### Content integrity & lifecycle (`file.*`, `pond.*`, `audit.*`)
|
||||
|
||||
| Id | Trigger | Severity | Actor | Target | Fields |
|
||||
| ----------------------- | ------------------------------------------------------------------- | -------- | ---------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `file.integrity_failed` | Attachment download hash mismatch — fail-closed (issue #199) | critical | `null` (any downloader; detection) | `attachment` | `pondId`, `expected` (stored sha256), `actual` (computed sha256) |
|
||||
| `pond.purged` | Pond irreversibly destroyed (manual or trash retention, issue #193) | notice | admin, `null` when retention-run | `pond` | `trigger` (`manual` \| `retention`) plus per-object-type deletion counts (e.g. `pages`, `attachments`, … — informational, keys may grow) |
|
||||
| `audit.pruned` | Audit retention deleted rows past the period (issue #196) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` |
|
||||
| Id | Trigger | Severity | Actor | Target | Fields |
|
||||
| ----------------------- | -------------------------------------------------------------------- | -------- | ---------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `file.integrity_failed` | Attachment download hash mismatch — fail-closed (issue #199) | critical | `null` (any downloader; detection) | `attachment` | `pondId`, `expected` (stored sha256), `actual` (computed sha256) |
|
||||
| `pond.purged` | Pond irreversibly destroyed (manual or trash retention, issue #193) | notice | admin, `null` when retention-run | `pond` | `trigger` (`manual` \| `retention`) plus per-object-type deletion counts (e.g. `pages`, `attachments`, … — informational, keys may grow) |
|
||||
| `audit.pruned` | Audit retention deleted rows past the period (issue #196) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` |
|
||||
| `read_trail.pruned` | Read-trail retention removed events past its own period (issue #224) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` |
|
||||
|
||||
### Public API (`api.*`)
|
||||
|
||||
|
||||
@ -186,6 +186,40 @@ Cached counters per pond: `storage_bytes_used`, `editor_count`,
|
||||
- `jobs`: conversion jobs for import/export (ADR 0009) and maintenance jobs
|
||||
(compaction, trash purge, quota reconciliation) with status + timestamps.
|
||||
|
||||
## Read-access trail (`read_events`, issues #222–#224, ADR 0023)
|
||||
|
||||
One row per read of a `classification = vs_nfd` page per channel and dedup
|
||||
window — separate from `audit_log` because volume, purpose and legal basis
|
||||
all differ.
|
||||
|
||||
| Column | Notes |
|
||||
| ---------------------------- | ------------------------------------------------------------------------------------ |
|
||||
| `id`, `occurred_at` | composite PK (the partition key must be part of it) |
|
||||
| `actor_id` | reader, null = anonymous; **no FK** — evidence outlives accounts |
|
||||
| `session_key` | `session:<id>` / `token:<id>` / `job:<id>` / `anon` |
|
||||
| `page_id`, `pond_id` | plain ids, **no FK** — evidence outlives purges |
|
||||
| `channel` | `page_view` / `no_js_shell` / `public_api` / `attachment` / `export` / `collab_join` |
|
||||
| `classification` | at read time; reclassification never rewrites history |
|
||||
| `dedup_key`, `window_bucket` | unique pair per partition — race-free dedup (#223) |
|
||||
| `window_seconds` | the row represents up to this many seconds of access |
|
||||
| `details` | small context (export format, attachment id) — never content |
|
||||
|
||||
Storage shape (issue #224): RANGE-partitioned by month on `occurred_at`,
|
||||
with a DEFAULT partition as safety net (a lagging maintenance job must
|
||||
never make classified reads fail — the trail write is hard-failing by
|
||||
design). The daily `read-trail-maintenance` job creates months ahead
|
||||
(each with its per-partition dedup index) and applies the trail's own
|
||||
retention `readTrail.retentionDays` (default 365): whole expired months
|
||||
are DROPped without scanning, remainders deleted by range; every pruning
|
||||
run is audited as `read_trail.pruned`. Growth, measured on PG 17:
|
||||
**~1 MB per 1000 events** including indexes (≈970 bytes/row) — a
|
||||
100-person instance at hundreds of classified reads/day stays in the tens
|
||||
of MB per year.
|
||||
|
||||
Query path: `GET /admin/system/read-events` (Site Admin, filters
|
||||
`pageId`, `actor`, `channel`, `from`/`to`) — deliberately API-only, no
|
||||
panel: the trail is an examiner's tool, not a daily screen.
|
||||
|
||||
## Comments & notifications (later milestone)
|
||||
|
||||
- `comments`: `page_id`, `author_id`, `body` (Markdown), `anchor`
|
||||
|
||||
@ -109,6 +109,7 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack.
|
||||
| conversion payload prune | daily | null finished conversion jobs' raw bytes (#233) |
|
||||
| mail outbox retention | daily | delete sent/failed outbox rows past period (#234) |
|
||||
| mail outbox retry | every minute | e-mail delivery with backoff |
|
||||
| read-trail maintenance | daily | partition months ahead + prune `read_events` (#224) |
|
||||
|
||||
Job outcomes are visible in the Site Admin UI (last run, status) — that
|
||||
panel is the operator's single glance for instance health.
|
||||
|
||||
@ -214,6 +214,11 @@ scan docker-archive:/image.tar -o cyclonedx-json` respectively
|
||||
- **Read-access trail** (issue #222, ADR 0023): reads of pages with
|
||||
`classification = vs_nfd` land as `read_events` rows — only classified
|
||||
pages, which is what keeps the purpose limitation defensible (variant A).
|
||||
Master switch `readTrail.enabled`, **default off** (#225): off means no
|
||||
event is written anywhere, including stdout, and the api announces the
|
||||
switch position once per boot so an eventless trail is never ambiguous.
|
||||
The written purpose limitation lives in
|
||||
`docs/vs-nfd/60-sicherheitsdokumentation.md` §7.
|
||||
The instrumented channels, and the emission point of each:
|
||||
- `page_view` — authenticated SPA state fetch (`GET /pages/:id`, the
|
||||
by-slug variant), the rendered read view (`/read/...`), the public JSON
|
||||
|
||||
@ -134,10 +134,10 @@ Protokolliert werden Lesezugriffe **ausschließlich** auf Seiten mit
|
||||
|
||||
- [x] Instrumentierung der Lesepfade: Seitenansicht, Public-API-GET,
|
||||
Attachment-Download, Export, No-JS-Shell, Collab-WS-Join · 4 AT · #222
|
||||
- [ ] Dedup-Fenster (eine Sitzung + eine Seite innerhalb N Minuten = ein
|
||||
- [x] Dedup-Fenster (eine Sitzung + eine Seite innerhalb N Minuten = ein
|
||||
Ereignis), sonst erzeugt Yjs-Sync eine Ereignisflut · 2 AT · #223
|
||||
- [ ] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224
|
||||
- [ ] Abschaltbar, Zweckbindung dokumentiert · 1–2 AT · #225
|
||||
- [x] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224
|
||||
- [x] Abschaltbar, Zweckbindung dokumentiert · 1–2 AT · #225
|
||||
|
||||
Vorteil über den Aufwand hinaus: Die Zweckbindung ist sauber begründbar
|
||||
(„nur eingestufte Inhalte"), was die Personalrats-Diskussion beim Kunden
|
||||
|
||||
@ -39,6 +39,9 @@ Settings-Cache ist in-process (operations.md).
|
||||
| `upload.allowedExtensions` | nur das dienstlich Nötige (z. B. `pdf`) | Standardliste | jede zusätzliche Endung vergrößert die Menge nicht prüfbarer Binärformate im Bestand. Bilder sind davon unabhängig immer erlaubt (Magic-Byte-geprüft). |
|
||||
| `backup.nextcloud.enabled` | `false` | `false` | „Backup nur lokal": kein Anwendungs-Upload von Restore-Sets zu Drittdiensten. Fernspiegel regelt ausschließlich die Deploy-Allowlist (1.2). |
|
||||
| `trash.retentionDays`, `audit.retentionDays`, `conversion.payloadRetentionDays`, `mail.outboxRetentionDays` | Defaults (30/365/30/30) | ebd. | Aufbewahrung bewusst begrenzt; Verkürzung nach Betreiber-Löschkonzept zulässig (Betriebshandbuch §5). |
|
||||
| `readTrail.enabled` | `true` | `false` | **explizit setzen** — der Lesetrail (#222–#225) evidenziert Lesezugriffe auf eingestufte Seiten; Default aus, weil Lesebeobachtung mitbestimmungsrelevant ist. Einschalten NUR zusammen mit der Zweckbindung (Sicherheitsdokumentation §7); die api meldet die Schalterstellung beim Start. |
|
||||
| `readTrail.dedupWindowMinutes` | Default (5) | `5` | Dedup-Fenster des Lesetrails (#223): je (Sitzung, Seite, Kanal) ein Ereignis pro Fenster — begrenzt die Ereignisflut einer Live-Sitzung auf ~12/h. Kleiner = feineres Protokoll und mehr Zeilen; Änderung mit dem Zweckbindungs-Dokument (#225) abstimmen. |
|
||||
| `readTrail.retentionDays` | Default (365) | `365` | eigene Aufbewahrung des Lesetrails (#224), bewusst getrennt von `audit.retentionDays`; Löschläufe sind selbst auditiert (`read_trail.pruned`). Dauer mit der Zweckbindung (#225) und dem Betreiber-Löschkonzept abstimmen. |
|
||||
| `legal.imprint`, `legal.privacyPolicy` | befüllt | leer | Betreiberpflicht; leere Seiten zeigen einen Warnbanner. |
|
||||
|
||||
### 1.2 Deploy-Konfiguration (`.env` / Compose — nur Plattformzugriff, bewusst nicht per Admin-UI)
|
||||
|
||||
@ -181,7 +181,14 @@ Datenträger- und Transportschutz ist Plattformsache
|
||||
### 3.5 Lesekanäle
|
||||
|
||||
Jeder Kanal, über den Seiteninhalt die Anwendung verlässt (zugleich die
|
||||
Instrumentierungsliste für den Lesetrail, #222):
|
||||
Instrumentierungsliste für den Lesetrail, #222). Der Lesetrail
|
||||
protokolliert je (Sitzung, Seite, Kanal) **ein Ereignis pro
|
||||
Dedup-Fenster** (`readTrail.dedupWindowMinutes`, Default 5 Minuten,
|
||||
#223); jede Zeile trägt die Fensterlänge. Beweiswert damit: „mindestens
|
||||
ein Zugriff in diesem Fenster" — **nicht** eine Zugriffszählung, und
|
||||
beim Kanal Collab „Live-Verbindung bestand während des Fensters", nicht
|
||||
einzelne Sync-Frames. Anonyme Leser teilen sich den Marker `anon`
|
||||
(keine Fingerprinting-Unterscheidung, Zweckbindung #225):
|
||||
|
||||
| Kanal | Pfad | Rechteprüfung | Kennzeichnung (M26) |
|
||||
| -------------------------- | -------------------------------- | -------------------------------------- | ----------------------------------- |
|
||||
@ -275,3 +282,42 @@ HKDF-Zweckableitung (#188), SHA-256-Integritätshashes für Attachments
|
||||
(#199, fail-closed beim Download). Keine Inhalts- oder
|
||||
Backup-Verschlüsselung, kein eigenes Schlüsselmanagement — bewusst
|
||||
(§52 VSA).
|
||||
|
||||
## 7 Lesetrail: Zweckbindung (Issue #225, ADR 0023)
|
||||
|
||||
Der Lesetrail ist abschaltbar (`readTrail.enabled`, **Default: aus**);
|
||||
die Referenzkonfiguration (Härtungsleitfaden §1.1) schaltet ihn ein.
|
||||
Aus heißt: es wird **nirgends** ein Ereignis geschrieben — keine
|
||||
Tabellenzeile, keine Logzeile; die api meldet die Schalterstellung
|
||||
einmal beim Start, damit ein ereignisloser Trail nie zweideutig ist.
|
||||
Diese Zweckbindung ist Teil der Betriebsdokumentation und gegenüber
|
||||
der Personalvertretung offenzulegen.
|
||||
|
||||
**Was aufgezeichnet wird:** je (Sitzung, Seite, Kanal) und Dedup-Fenster
|
||||
(§3.5) ein Ereignis mit Zeitstempel, Konto-Id (oder dem Marker `anon`),
|
||||
Sitzungsschlüssel, Seiten- und Teich-Id, Kanal und der Einstufung zum
|
||||
Lesezeitpunkt. **Kein** Seiteninhalt, **keine** Titel, **keine**
|
||||
IP-Adressen, **kein** Fingerprinting anonymer Leser.
|
||||
|
||||
**Warum:** ausschließlich Beweissicherung für Lesezugriffe auf **als
|
||||
VS-NfD gekennzeichnete** Inhalte (§52-konforme Nachvollziehbarkeit:
|
||||
„welche eingestufte Seite wurde wann über welchen Kanal gelesen") —
|
||||
Plattform-Logs kennen URLs, nicht Einstufungen. Zugriffe auf nicht
|
||||
eingestufte Inhalte werden **nie** erfasst (Variante A, ADR 0023).
|
||||
|
||||
**Wer lesen darf:** ausschließlich Site-Admins über
|
||||
`GET /admin/system/read-events` (kein UI-Panel — Prüfwerkzeug, kein
|
||||
Alltagsbildschirm). Organisatorische Beschränkung auf den
|
||||
Sicherheitsbeauftragten: Betreibersache (Betriebshandbuch §6
|
||||
Rollentrennung).
|
||||
|
||||
**Wie lange:** `readTrail.retentionDays` (Default 365 Tage); Löschläufe
|
||||
sind selbst auditiert (`read_trail.pruned`), sodass jede Lücke erklärbar
|
||||
ist.
|
||||
|
||||
**Wofür NICHT:** keine Leistungs- oder Verhaltenskontrolle der
|
||||
Beschäftigten, keine Auswertung von Arbeitsmustern, keine
|
||||
Anwesenheitskontrolle, keine Weitergabe außerhalb des
|
||||
Sicherheitsvorfalls- bzw. Prüfkontexts. Die Beschränkung auf
|
||||
eingestufte Inhalte ist die technische Absicherung dieser Zusage: was
|
||||
nicht gekennzeichnet ist, erzeugt keine Spur.
|
||||
|
||||
@ -164,6 +164,49 @@ export interface AuditListView {
|
||||
total: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Site-Admin query path over the read-access trail (issue #224, ADR 0023) —
|
||||
* evidence nobody can read is not evidence. Answers the two expected
|
||||
* questions: "who read page X" (pageId) and "what did user Y read" (actor),
|
||||
* both within a period.
|
||||
*/
|
||||
export const READ_EVENT_PAGE_SIZE = 50;
|
||||
|
||||
export const readEventListQuerySchema = z.object({
|
||||
/** Page id the events belong to. */
|
||||
pageId: z.string().trim().min(1).optional(),
|
||||
/** Exact username of the reading user. */
|
||||
actor: z.string().trim().min(1).optional(),
|
||||
channel: z.string().trim().min(1).optional(),
|
||||
from: z.coerce.date().optional(),
|
||||
to: z.coerce.date().optional(),
|
||||
page: z.coerce.number().int().min(1).default(1),
|
||||
});
|
||||
|
||||
export type ReadEventListQuery = z.infer<typeof readEventListQuerySchema>;
|
||||
|
||||
export interface ReadEventView {
|
||||
id: string;
|
||||
occurredAt: string;
|
||||
actor: AuditActorView | null;
|
||||
/** Null for pond-level attachments (no page); the details name the file. */
|
||||
pageId: string | null;
|
||||
pondId: string;
|
||||
channel: string;
|
||||
classification: string;
|
||||
/** Seconds the event's dedup window spans (#223) — the row represents up
|
||||
* to this much access time, not a single request. */
|
||||
windowSeconds: number;
|
||||
details: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
export interface ReadEventListView {
|
||||
entries: ReadEventView[];
|
||||
page: number;
|
||||
pageCount: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface StoragePondView {
|
||||
pondId: string;
|
||||
name: string;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user