#224: read-trail storage — partitioning, retention, admin query path
Convert read_events to monthly RANGE partitions on occurred_at, with a DEFAULT partition as safety net: a lagging maintenance job must never turn the trail's hard-failure semantics into an outage for classified reads. The dedup unique pair (#223) moves to per-partition indexes (PostgreSQL cannot carry it on the parent); a bucket spanning a month boundary may record one duplicate — over-recording is acceptable, gaps are not. New daily job read-trail-maintenance (job-count fence 9 -> 10) creates months ahead — each with its dedup index — and applies the trail's own retention readTrail.retentionDays (default 365, deliberately independent of audit.retentionDays): whole expired months are DROPped without scanning, remainders deleted by range, every run audited as read_trail.pruned (catalogue v1.2; the fence regex now admits an underscore namespace). Site-Admin query path GET /admin/system/read-events answers "who read page X" and "what did user Y read" within a period — API-only by design, documented. Growth measured and documented in data-model.md: ~1 MB per 1000 events including indexes. Tests: retention pruning + audited deletion + admin queries on the shared database; the partitioned shape, per-partition P2002 dedup, months-ahead creation and DROP-based pruning against a fresh database built by the real migration chain. Refs #224. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AUtYMxwTCMHG9mVHnwbFg8
This commit is contained in:
parent
fd4fd60c99
commit
2bdb0ec2cf
@ -0,0 +1,76 @@
|
|||||||
|
-- #224 (ADR 0023): convert read_events to monthly RANGE partitions on
|
||||||
|
-- occurred_at. Volume grows unbounded with use; retention then DROPs whole
|
||||||
|
-- expired partitions instead of scanning deletes. The primary key gains the
|
||||||
|
-- partition column (PostgreSQL requirement); the dedup unique pair
|
||||||
|
-- (dedup_key, window_bucket) moves to PER-PARTITION unique indexes — a
|
||||||
|
-- partitioned parent cannot carry it without the partition key. A bucket
|
||||||
|
-- spanning a month boundary can therefore record one duplicate; documented
|
||||||
|
-- in ADR 0023, over-recording is acceptable, gaps are not.
|
||||||
|
--
|
||||||
|
-- A DEFAULT partition catches rows outside every maintained range, so a
|
||||||
|
-- lagging maintenance job can never make classified reads fail (the trail's
|
||||||
|
-- hard-failure semantics would otherwise turn an ops miss into an outage).
|
||||||
|
|
||||||
|
ALTER TABLE "read_events" RENAME TO "read_events_old";
|
||||||
|
ALTER INDEX "read_events_pkey" RENAME TO "read_events_old_pkey";
|
||||||
|
ALTER INDEX "read_events_dedup_key_window_bucket_key" RENAME TO "read_events_old_dedup_key";
|
||||||
|
ALTER INDEX "read_events_page_id_occurred_at_idx" RENAME TO "read_events_old_page_idx";
|
||||||
|
ALTER INDEX "read_events_actor_id_occurred_at_idx" RENAME TO "read_events_old_actor_idx";
|
||||||
|
ALTER INDEX "read_events_occurred_at_idx" RENAME TO "read_events_old_at_idx";
|
||||||
|
|
||||||
|
CREATE TABLE "read_events" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"occurred_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"actor_id" TEXT,
|
||||||
|
"session_key" TEXT NOT NULL,
|
||||||
|
"page_id" TEXT,
|
||||||
|
"pond_id" TEXT NOT NULL,
|
||||||
|
"channel" TEXT NOT NULL,
|
||||||
|
"classification" TEXT NOT NULL,
|
||||||
|
"details" JSONB,
|
||||||
|
"dedup_key" TEXT NOT NULL,
|
||||||
|
"window_bucket" BIGINT NOT NULL,
|
||||||
|
"window_seconds" INTEGER NOT NULL,
|
||||||
|
|
||||||
|
CONSTRAINT "read_events_pkey" PRIMARY KEY ("id", "occurred_at")
|
||||||
|
) PARTITION BY RANGE ("occurred_at");
|
||||||
|
|
||||||
|
-- Non-unique parent indexes propagate to every partition automatically.
|
||||||
|
CREATE INDEX "read_events_page_id_occurred_at_idx" ON "read_events"("page_id", "occurred_at");
|
||||||
|
CREATE INDEX "read_events_actor_id_occurred_at_idx" ON "read_events"("actor_id", "occurred_at");
|
||||||
|
CREATE INDEX "read_events_occurred_at_idx" ON "read_events"("occurred_at");
|
||||||
|
|
||||||
|
-- The safety-net partition, plus the current and the next month — the daily
|
||||||
|
-- maintenance job (read-trail-maintenance) keeps creating months ahead and
|
||||||
|
-- adds the same per-partition dedup index to each new one.
|
||||||
|
CREATE TABLE "read_events_default" PARTITION OF "read_events" DEFAULT;
|
||||||
|
CREATE UNIQUE INDEX "read_events_default_dedup_key"
|
||||||
|
ON "read_events_default"("dedup_key", "window_bucket");
|
||||||
|
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
m DATE;
|
||||||
|
part TEXT;
|
||||||
|
BEGIN
|
||||||
|
FOR i IN 0..1 LOOP
|
||||||
|
m := date_trunc('month', now())::date + (i || ' month')::interval;
|
||||||
|
part := 'read_events_y' || to_char(m, 'YYYY') || 'm' || to_char(m, 'MM');
|
||||||
|
EXECUTE format(
|
||||||
|
'CREATE TABLE %I PARTITION OF "read_events" FOR VALUES FROM (%L) TO (%L)',
|
||||||
|
part, m, m + interval '1 month');
|
||||||
|
EXECUTE format(
|
||||||
|
'CREATE UNIQUE INDEX %I ON %I ("dedup_key", "window_bucket")',
|
||||||
|
part || '_dedup_key', part);
|
||||||
|
END LOOP;
|
||||||
|
END $$;
|
||||||
|
|
||||||
|
INSERT INTO "read_events"
|
||||||
|
("id", "occurred_at", "actor_id", "session_key", "page_id", "pond_id",
|
||||||
|
"channel", "classification", "details", "dedup_key", "window_bucket",
|
||||||
|
"window_seconds")
|
||||||
|
SELECT "id", "occurred_at", "actor_id", "session_key", "page_id", "pond_id",
|
||||||
|
"channel", "classification", "details", "dedup_key", "window_bucket",
|
||||||
|
"window_seconds"
|
||||||
|
FROM "read_events_old";
|
||||||
|
|
||||||
|
DROP TABLE "read_events_old";
|
||||||
@ -96,8 +96,13 @@ model AuditEntry {
|
|||||||
/// volume, purpose and legal basis all differ. Deliberately WITHOUT foreign
|
/// volume, purpose and legal basis all differ. Deliberately WITHOUT foreign
|
||||||
/// keys: evidence must survive a page purge and a hard user deletion — the
|
/// keys: evidence must survive a page purge and a hard user deletion — the
|
||||||
/// ids stay as recorded (pseudonymous uuids), history is never rewritten.
|
/// ids stay as recorded (pseudonymous uuids), history is never rewritten.
|
||||||
|
///
|
||||||
|
/// In migrated databases the table is RANGE-partitioned by `occurred_at`
|
||||||
|
/// (monthly, issue #224) — hence the composite id. The dedup unique pair
|
||||||
|
/// lives per partition there (a partitioned parent cannot carry it without
|
||||||
|
/// the partition key); `db push` test databases get it on the plain table.
|
||||||
model ReadEvent {
|
model ReadEvent {
|
||||||
id String @id @default(uuid())
|
id String @default(uuid())
|
||||||
occurredAt DateTime @default(now()) @map("occurred_at")
|
occurredAt DateTime @default(now()) @map("occurred_at")
|
||||||
/// Null = anonymous reader (public grant); `sessionKey` still names the
|
/// Null = anonymous reader (public grant); `sessionKey` still names the
|
||||||
/// browsing session, so the anonymous marker is explicit, not an accident.
|
/// browsing session, so the anonymous marker is explicit, not an accident.
|
||||||
@ -124,6 +129,7 @@ model ReadEvent {
|
|||||||
/// represents up to this many seconds, so the evidence is not overread.
|
/// represents up to this many seconds, so the evidence is not overread.
|
||||||
windowSeconds Int @map("window_seconds")
|
windowSeconds Int @map("window_seconds")
|
||||||
|
|
||||||
|
@@id([id, occurredAt])
|
||||||
@@unique([dedupKey, windowBucket])
|
@@unique([dedupKey, windowBucket])
|
||||||
@@index([pageId, occurredAt])
|
@@index([pageId, occurredAt])
|
||||||
@@index([actorId, occurredAt])
|
@@index([actorId, occurredAt])
|
||||||
|
|||||||
@ -1,9 +1,12 @@
|
|||||||
import { Controller, Get, Param, Post, Query, Req, UseGuards } from '@nestjs/common';
|
import { Controller, Get, Param, Post, Query, Req, UseGuards } from '@nestjs/common';
|
||||||
import {
|
import {
|
||||||
auditListQuerySchema,
|
auditListQuerySchema,
|
||||||
|
readEventListQuerySchema,
|
||||||
type AuditListQuery,
|
type AuditListQuery,
|
||||||
type AuditListView,
|
type AuditListView,
|
||||||
type JobTriggerResult,
|
type JobTriggerResult,
|
||||||
|
type ReadEventListQuery,
|
||||||
|
type ReadEventListView,
|
||||||
type StorageOverviewView,
|
type StorageOverviewView,
|
||||||
type SystemBackupView,
|
type SystemBackupView,
|
||||||
type SystemJobView,
|
type SystemJobView,
|
||||||
@ -45,6 +48,15 @@ export class SystemAdminController {
|
|||||||
return this.system.auditLog(query);
|
return this.system.auditLog(query);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Read-access trail queries (issue #224): "who read page X", "what did
|
||||||
|
* user Y read" — Site-Admin only, like the audit viewer above. */
|
||||||
|
@Get('read-events')
|
||||||
|
async readEvents(
|
||||||
|
@Query(new ZodValidationPipe(readEventListQuerySchema)) query: ReadEventListQuery,
|
||||||
|
): Promise<ReadEventListView> {
|
||||||
|
return this.system.readEvents(query);
|
||||||
|
}
|
||||||
|
|
||||||
@Get('storage')
|
@Get('storage')
|
||||||
async storage(): Promise<StorageOverviewView> {
|
async storage(): Promise<StorageOverviewView> {
|
||||||
return this.system.storage();
|
return this.system.storage();
|
||||||
|
|||||||
@ -7,10 +7,13 @@ import {
|
|||||||
AUDIT_PAGE_SIZE,
|
AUDIT_PAGE_SIZE,
|
||||||
BACKUP_FRESH_MAX_AGE_HOURS,
|
BACKUP_FRESH_MAX_AGE_HOURS,
|
||||||
BACKUP_STATUS_FILE,
|
BACKUP_STATUS_FILE,
|
||||||
|
READ_EVENT_PAGE_SIZE,
|
||||||
type AuditListQuery,
|
type AuditListQuery,
|
||||||
type AuditListView,
|
type AuditListView,
|
||||||
type BackupStatus,
|
type BackupStatus,
|
||||||
type JobTriggerResult,
|
type JobTriggerResult,
|
||||||
|
type ReadEventListQuery,
|
||||||
|
type ReadEventListView,
|
||||||
type StorageOverviewView,
|
type StorageOverviewView,
|
||||||
type SystemBackupView,
|
type SystemBackupView,
|
||||||
type SystemJobView,
|
type SystemJobView,
|
||||||
@ -160,6 +163,66 @@ export class SystemAdminService {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Site-Admin query path over the read-access trail (issue #224,
|
||||||
|
* ADR 0023) — evidence nobody can read is not evidence. Answers "who read
|
||||||
|
* page X" and "what did user Y read" within a period. API-only by design
|
||||||
|
* (no panel yet): the trail is an examiner's tool, not a daily screen —
|
||||||
|
* documented in data-model.md §read_events.
|
||||||
|
*/
|
||||||
|
async readEvents(query: ReadEventListQuery): Promise<ReadEventListView> {
|
||||||
|
const where: Prisma.ReadEventWhereInput = {};
|
||||||
|
if (query.pageId) where.pageId = query.pageId;
|
||||||
|
if (query.actor) {
|
||||||
|
const actor = await this.prisma.user.findUnique({ where: { username: query.actor } });
|
||||||
|
// An unknown username matches nothing rather than everything.
|
||||||
|
where.actorId = actor?.id ?? '00000000-0000-0000-0000-000000000000';
|
||||||
|
}
|
||||||
|
if (query.channel) where.channel = query.channel;
|
||||||
|
if (query.from || query.to) {
|
||||||
|
where.occurredAt = {
|
||||||
|
...(query.from ? { gte: query.from } : {}),
|
||||||
|
...(query.to ? { lte: query.to } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const total = await this.prisma.readEvent.count({ where });
|
||||||
|
const pageCount = Math.max(1, Math.ceil(total / READ_EVENT_PAGE_SIZE));
|
||||||
|
const page = Math.min(query.page, pageCount);
|
||||||
|
const events = await this.prisma.readEvent.findMany({
|
||||||
|
where,
|
||||||
|
orderBy: { occurredAt: 'desc' },
|
||||||
|
skip: (page - 1) * READ_EVENT_PAGE_SIZE,
|
||||||
|
take: READ_EVENT_PAGE_SIZE,
|
||||||
|
});
|
||||||
|
// No FK on actor_id (evidence outlives accounts) — resolve what still
|
||||||
|
// exists in one query, show the bare id otherwise.
|
||||||
|
const actorIds = [...new Set(events.map((e) => e.actorId).filter((id): id is string => !!id))];
|
||||||
|
const actors = actorIds.length
|
||||||
|
? await this.prisma.user.findMany({
|
||||||
|
where: { id: { in: actorIds } },
|
||||||
|
select: { id: true, username: true, displayName: true },
|
||||||
|
})
|
||||||
|
: [];
|
||||||
|
const actorById = new Map(actors.map((a) => [a.id, a]));
|
||||||
|
return {
|
||||||
|
entries: events.map((event) => ({
|
||||||
|
id: event.id,
|
||||||
|
occurredAt: event.occurredAt.toISOString(),
|
||||||
|
actor: event.actorId ? (actorById.get(event.actorId) ?? null) : null,
|
||||||
|
pageId: event.pageId,
|
||||||
|
pondId: event.pondId,
|
||||||
|
channel: event.channel,
|
||||||
|
classification: event.classification,
|
||||||
|
windowSeconds: event.windowSeconds,
|
||||||
|
details: (event.details as Record<string, unknown> | null) ?? null,
|
||||||
|
})),
|
||||||
|
page,
|
||||||
|
pageCount,
|
||||||
|
total,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async storage(): Promise<StorageOverviewView> {
|
async storage(): Promise<StorageOverviewView> {
|
||||||
const usages = await this.prisma.pondUsage.findMany({
|
const usages = await this.prisma.pondUsage.findMany({
|
||||||
where: { pond: { deletedAt: null } },
|
where: { pond: { deletedAt: null } },
|
||||||
|
|||||||
@ -40,6 +40,7 @@ export const AUDIT_EVENTS = {
|
|||||||
'pond.purged': { severity: 'notice' },
|
'pond.purged': { severity: 'notice' },
|
||||||
'quota.override_cleared': { severity: 'notice' },
|
'quota.override_cleared': { severity: 'notice' },
|
||||||
'quota.override_set': { severity: 'notice' },
|
'quota.override_set': { severity: 'notice' },
|
||||||
|
'read_trail.pruned': { severity: 'info' },
|
||||||
'settings.changed': { severity: 'notice' },
|
'settings.changed': { severity: 'notice' },
|
||||||
'setup.admin_created': { severity: 'notice' },
|
'setup.admin_created': { severity: 'notice' },
|
||||||
'setup.completed': { severity: 'info' },
|
'setup.completed': { severity: 'info' },
|
||||||
|
|||||||
@ -18,11 +18,12 @@ import { AUDIT_EVENTS } from './audit-actions';
|
|||||||
const doc = readFileSync(join(__dirname, '../../../../docs/architecture/audit-events.md'), 'utf8');
|
const doc = readFileSync(join(__dirname, '../../../../docs/architecture/audit-events.md'), 'utf8');
|
||||||
|
|
||||||
/** Event rows are `| \`ns.event\` | trigger | severity | …` — the dot in the
|
/** Event rows are `| \`ns.event\` | trigger | severity | …` — the dot in the
|
||||||
* id keeps field-set rows (`msg`, `severity`, …) out of the match. */
|
* id keeps field-set rows (`msg`, `severity`, …) out of the match. The
|
||||||
|
* namespace may carry an underscore since `read_trail.*` (issue #224). */
|
||||||
function documentedEvents(): Map<string, string> {
|
function documentedEvents(): Map<string, string> {
|
||||||
const events = new Map<string, string>();
|
const events = new Map<string, string>();
|
||||||
for (const line of doc.split('\n')) {
|
for (const line of doc.split('\n')) {
|
||||||
const id = /^\| `([a-z]+\.[a-z_]+)` +\|/.exec(line)?.[1];
|
const id = /^\| `([a-z_]+\.[a-z_]+)` +\|/.exec(line)?.[1];
|
||||||
if (!id) continue;
|
if (!id) continue;
|
||||||
const cells = line.split('|').map((cell) => cell.trim());
|
const cells = line.split('|').map((cell) => cell.trim());
|
||||||
// cells[0] is the empty string before the leading pipe.
|
// cells[0] is the empty string before the leading pipe.
|
||||||
|
|||||||
139
apps/api/src/read-trail/read-trail-maintenance.service.ts
Normal file
139
apps/api/src/read-trail/read-trail-maintenance.service.ts
Normal file
@ -0,0 +1,139 @@
|
|||||||
|
import { Injectable } from '@nestjs/common';
|
||||||
|
import { PinoLogger } from 'nestjs-pino';
|
||||||
|
|
||||||
|
import { AuditService } from '../audit/audit.service';
|
||||||
|
import { ClockService } from '../common/clock.service';
|
||||||
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
|
import { InstanceSettingsService } from '../settings/instance-settings.service';
|
||||||
|
|
||||||
|
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
/** How many months ahead of "now" a partition must exist. Two keeps a
|
||||||
|
* multi-week job outage from ever reaching an uncovered month (the DEFAULT
|
||||||
|
* partition would still catch it — reads never fail on a missing month). */
|
||||||
|
const MONTHS_AHEAD = 2;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read-trail storage maintenance (issue #224, ADR 0023), one daily job with
|
||||||
|
* two duties:
|
||||||
|
*
|
||||||
|
* 1. **Partition upkeep** — `read_events` is RANGE-partitioned by month
|
||||||
|
* (migration `20260731170000`); this creates the next {@link MONTHS_AHEAD}
|
||||||
|
* monthly partitions, each with its per-partition dedup unique index
|
||||||
|
* (#223 — the partitioned parent cannot carry it). A `db push` database
|
||||||
|
* (tests) has a plain table; partition work skips itself there.
|
||||||
|
* 2. **Retention** — events older than `readTrail.retentionDays` are
|
||||||
|
* removed: whole months by dropping their partition (no scan), the
|
||||||
|
* remainder (default partition, plain tables) by a ranged delete. The
|
||||||
|
* deletion is audited (`read_trail.pruned`) so a gap in the evidence is
|
||||||
|
* always explainable — same principle as `audit.pruned` (#196), but a
|
||||||
|
* deliberately separate period.
|
||||||
|
*/
|
||||||
|
@Injectable()
|
||||||
|
export class ReadTrailMaintenanceService {
|
||||||
|
constructor(
|
||||||
|
private readonly prisma: PrismaService,
|
||||||
|
private readonly settings: InstanceSettingsService,
|
||||||
|
private readonly audit: AuditService,
|
||||||
|
private readonly clock: ClockService,
|
||||||
|
private readonly logger: PinoLogger,
|
||||||
|
) {
|
||||||
|
this.logger.setContext(ReadTrailMaintenanceService.name);
|
||||||
|
}
|
||||||
|
|
||||||
|
async run(): Promise<void> {
|
||||||
|
await this.ensurePartitions();
|
||||||
|
await this.pruneExpired();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** True when read_events is a partitioned parent (relkind `p`). */
|
||||||
|
private async isPartitioned(): Promise<boolean> {
|
||||||
|
const rows = await this.prisma.$queryRaw<{ relkind: string }[]>`
|
||||||
|
SELECT relkind::text FROM pg_class
|
||||||
|
WHERE relname = 'read_events' AND relnamespace = 'public'::regnamespace`;
|
||||||
|
return rows[0]?.relkind === 'p';
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `read_events_y2026m08` for 2026-08. */
|
||||||
|
private partitionName(month: Date): string {
|
||||||
|
const y = month.getUTCFullYear();
|
||||||
|
const m = String(month.getUTCMonth() + 1).padStart(2, '0');
|
||||||
|
return `read_events_y${y}m${m}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private monthStart(base: Date, offsetMonths: number): Date {
|
||||||
|
return new Date(Date.UTC(base.getUTCFullYear(), base.getUTCMonth() + offsetMonths, 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
async ensurePartitions(): Promise<void> {
|
||||||
|
if (!(await this.isPartitioned())) {
|
||||||
|
this.logger.debug('read_events is not partitioned here; skipping partition upkeep');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const now = this.clock.now();
|
||||||
|
for (let offset = 0; offset <= MONTHS_AHEAD; offset += 1) {
|
||||||
|
const from = this.monthStart(now, offset);
|
||||||
|
const to = this.monthStart(now, offset + 1);
|
||||||
|
const name = this.partitionName(from);
|
||||||
|
try {
|
||||||
|
await this.prisma.$executeRawUnsafe(
|
||||||
|
`CREATE TABLE IF NOT EXISTS "${name}" PARTITION OF "read_events"
|
||||||
|
FOR VALUES FROM ('${from.toISOString()}') TO ('${to.toISOString()}')`,
|
||||||
|
);
|
||||||
|
await this.prisma.$executeRawUnsafe(
|
||||||
|
`CREATE UNIQUE INDEX IF NOT EXISTS "${name}_dedup_key"
|
||||||
|
ON "${name}" ("dedup_key", "window_bucket")`,
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
// Most likely: the DEFAULT partition already holds rows of this month
|
||||||
|
// (the job lagged past a month boundary). Nothing is lost — those
|
||||||
|
// rows live in the default partition and age out through the ranged
|
||||||
|
// delete below; the month just cannot get its own partition anymore.
|
||||||
|
this.logger.warn({ partition: name, err: error }, 'read-trail partition not created');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async pruneExpired(): Promise<number> {
|
||||||
|
const retentionDays = await this.settings.get('readTrail.retentionDays');
|
||||||
|
const cutoff = new Date(this.clock.now().getTime() - retentionDays * MS_PER_DAY);
|
||||||
|
let dropped = 0;
|
||||||
|
|
||||||
|
if (await this.isPartitioned()) {
|
||||||
|
// Whole months strictly before the cutoff month go by DROP — no scan,
|
||||||
|
// and the dropped range is exact (every row in them is < cutoff).
|
||||||
|
const partitions = await this.prisma.$queryRaw<{ relname: string }[]>`
|
||||||
|
SELECT c.relname::text
|
||||||
|
FROM pg_inherits i
|
||||||
|
JOIN pg_class c ON c.oid = i.inhrelid
|
||||||
|
WHERE i.inhparent = 'read_events'::regclass
|
||||||
|
AND c.relname ~ '^read_events_y[0-9]{4}m[0-9]{2}$'`;
|
||||||
|
const cutoffMonth = this.monthStart(cutoff, 0);
|
||||||
|
for (const { relname } of partitions) {
|
||||||
|
const match = /^read_events_y(\d{4})m(\d{2})$/.exec(relname);
|
||||||
|
if (!match) continue;
|
||||||
|
const monthEnd = new Date(Date.UTC(Number(match[1]), Number(match[2]), 1));
|
||||||
|
if (monthEnd.getTime() > cutoffMonth.getTime()) continue;
|
||||||
|
const counted = await this.prisma.$queryRawUnsafe<{ count: bigint }[]>(
|
||||||
|
`SELECT count(*)::bigint AS count FROM "${relname}"`,
|
||||||
|
);
|
||||||
|
dropped += Number(counted[0]?.count ?? 0n);
|
||||||
|
await this.prisma.$executeRawUnsafe(`DROP TABLE "${relname}"`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The remainder: rows before the cutoff inside surviving partitions,
|
||||||
|
// the default partition, or a plain (test) table.
|
||||||
|
const deleted = await this.prisma.readEvent.deleteMany({
|
||||||
|
where: { occurredAt: { lt: cutoff } },
|
||||||
|
});
|
||||||
|
const count = dropped + deleted.count;
|
||||||
|
if (count > 0) {
|
||||||
|
await this.audit.record({
|
||||||
|
action: 'read_trail.pruned',
|
||||||
|
details: { count, cutoff: cutoff.toISOString(), retentionDays },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
}
|
||||||
308
apps/api/src/read-trail/read-trail-storage.e2e.db.test.ts
Normal file
308
apps/api/src/read-trail/read-trail-storage.e2e.db.test.ts
Normal file
@ -0,0 +1,308 @@
|
|||||||
|
import { execFileSync } from 'node:child_process';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
|
||||||
|
import { INestApplication } from '@nestjs/common';
|
||||||
|
import { PrismaClient } from '@prisma/client';
|
||||||
|
import { PinoLogger } from 'nestjs-pino';
|
||||||
|
import request from 'supertest';
|
||||||
|
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
import { ClockService } from '../common/clock.service';
|
||||||
|
import { InstanceSettingsService } from '../settings/instance-settings.service';
|
||||||
|
import { createTestApp, sessionCookieOf } from '../testing/test-app';
|
||||||
|
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
|
||||||
|
import { UsersService } from '../users/users.service';
|
||||||
|
|
||||||
|
import { ReadTrailMaintenanceService } from './read-trail-maintenance.service';
|
||||||
|
|
||||||
|
const API_ROOT = join(__dirname, '..', '..');
|
||||||
|
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read-trail storage (issue #224, ADR 0023): its own retention period with
|
||||||
|
* an audited deletion, the Site-Admin query path, and — against a freshly
|
||||||
|
* migrated database, where the real DDL applies — the monthly RANGE
|
||||||
|
* partitioning with per-partition dedup indexes and DROP-based pruning.
|
||||||
|
*/
|
||||||
|
describe.skipIf(!hasTestDb)('read-trail storage (e2e, issue #224)', () => {
|
||||||
|
const suffix = uniqueSuffix();
|
||||||
|
|
||||||
|
describe('retention and admin query path (shared database)', () => {
|
||||||
|
let app: INestApplication;
|
||||||
|
let prisma: PrismaClient;
|
||||||
|
const password = 'lesetrail speicher 123';
|
||||||
|
|
||||||
|
let adminId: string;
|
||||||
|
let adminCookie: string;
|
||||||
|
let readerCookie: string;
|
||||||
|
// Plain text ids — read_events has no FKs by design (#222).
|
||||||
|
const pondId = `pond-${suffix}`;
|
||||||
|
|
||||||
|
const api = () => request(app.getHttpServer());
|
||||||
|
|
||||||
|
async function makeUser(handle: string, siteAdmin: boolean) {
|
||||||
|
const users = app.get(UsersService);
|
||||||
|
const user = await users.createUser({
|
||||||
|
username: `${handle}-${suffix}`,
|
||||||
|
email: `${handle}-${suffix}@example.test`,
|
||||||
|
displayName: handle,
|
||||||
|
password,
|
||||||
|
locale: 'en',
|
||||||
|
});
|
||||||
|
await users.markEmailVerified(user.id);
|
||||||
|
if (siteAdmin) {
|
||||||
|
await prisma.user.update({ where: { id: user.id }, data: { isSiteAdmin: true } });
|
||||||
|
}
|
||||||
|
const cookie = sessionCookieOf(
|
||||||
|
await api()
|
||||||
|
.post('/api/v1/auth/login')
|
||||||
|
.send({ usernameOrEmail: `${handle}-${suffix}`, password })
|
||||||
|
.expect(200),
|
||||||
|
);
|
||||||
|
return { id: user.id, cookie };
|
||||||
|
}
|
||||||
|
|
||||||
|
function eventRow(overrides: Record<string, unknown>) {
|
||||||
|
return {
|
||||||
|
actorId: null,
|
||||||
|
sessionKey: 'anon',
|
||||||
|
pageId: null,
|
||||||
|
pondId,
|
||||||
|
channel: 'page_view',
|
||||||
|
classification: 'vs_nfd',
|
||||||
|
dedupKey: `k-${suffix}-${Math.random().toString(36).slice(2)}`,
|
||||||
|
windowBucket: BigInt(Math.floor(Math.random() * 1_000_000_000)),
|
||||||
|
windowSeconds: 300,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
prisma = createTestPrisma();
|
||||||
|
await prisma.rateLimit.deleteMany({});
|
||||||
|
app = await createTestApp();
|
||||||
|
const admin = await makeUser('storage-admin', true);
|
||||||
|
adminId = admin.id;
|
||||||
|
adminCookie = admin.cookie;
|
||||||
|
const reader = await makeUser('storage-reader', false);
|
||||||
|
readerCookie = reader.cookie;
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await prisma.readEvent.deleteMany({ where: { pondId } });
|
||||||
|
await prisma.auditEntry.deleteMany({ where: { action: 'read_trail.pruned' } });
|
||||||
|
await prisma.instanceSetting.deleteMany({ where: { key: 'readTrail.retentionDays' } });
|
||||||
|
await prisma.user.deleteMany({ where: { username: { contains: suffix } } });
|
||||||
|
await prisma.$disconnect();
|
||||||
|
await app.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('prunes only events past its own period and audits the deletion', async () => {
|
||||||
|
await app.get(InstanceSettingsService).set('readTrail.retentionDays', 30, adminId);
|
||||||
|
const now = Date.now();
|
||||||
|
await prisma.readEvent.createMany({
|
||||||
|
data: [
|
||||||
|
eventRow({ occurredAt: new Date(now - 40 * MS_PER_DAY) }),
|
||||||
|
eventRow({ occurredAt: new Date(now - 31 * MS_PER_DAY) }),
|
||||||
|
eventRow({ occurredAt: new Date(now - 5 * MS_PER_DAY) }),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const maintenance = app.get(ReadTrailMaintenanceService);
|
||||||
|
const pruned = await maintenance.pruneExpired();
|
||||||
|
expect(pruned).toBeGreaterThanOrEqual(2);
|
||||||
|
|
||||||
|
const remaining = await prisma.readEvent.findMany({ where: { pondId } });
|
||||||
|
expect(remaining).toHaveLength(1);
|
||||||
|
|
||||||
|
const audit = await prisma.auditEntry.findFirst({
|
||||||
|
where: { action: 'read_trail.pruned' },
|
||||||
|
orderBy: { at: 'desc' },
|
||||||
|
});
|
||||||
|
expect(audit).not.toBeNull();
|
||||||
|
expect(audit!.details).toMatchObject({ retentionDays: 30 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('answers "who read page X" and "what did user Y read" for Site Admins only', async () => {
|
||||||
|
const pageId = `page-${suffix}`;
|
||||||
|
await prisma.readEvent.createMany({
|
||||||
|
data: [
|
||||||
|
eventRow({ pageId, actorId: adminId, channel: 'export' }),
|
||||||
|
eventRow({ pageId: `other-${suffix}`, actorId: null }),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const byPage = await api()
|
||||||
|
.get(`/api/v1/admin/system/read-events?pageId=${pageId}`)
|
||||||
|
.set('Cookie', adminCookie)
|
||||||
|
.expect(200);
|
||||||
|
expect(byPage.body.total).toBe(1);
|
||||||
|
expect(byPage.body.entries[0]).toMatchObject({
|
||||||
|
pageId,
|
||||||
|
channel: 'export',
|
||||||
|
windowSeconds: 300,
|
||||||
|
});
|
||||||
|
expect(byPage.body.entries[0].actor).toMatchObject({ id: adminId });
|
||||||
|
|
||||||
|
const byActor = await api()
|
||||||
|
.get(`/api/v1/admin/system/read-events?actor=storage-admin-${suffix}`)
|
||||||
|
.set('Cookie', adminCookie)
|
||||||
|
.expect(200);
|
||||||
|
expect(byActor.body.total).toBe(1);
|
||||||
|
|
||||||
|
// An unknown username matches nothing rather than everything.
|
||||||
|
const unknown = await api()
|
||||||
|
.get(`/api/v1/admin/system/read-events?actor=nobody-${suffix}`)
|
||||||
|
.set('Cookie', adminCookie)
|
||||||
|
.expect(200);
|
||||||
|
expect(unknown.body.total).toBe(0);
|
||||||
|
|
||||||
|
await api()
|
||||||
|
.get(`/api/v1/admin/system/read-events?pageId=${pageId}`)
|
||||||
|
.set('Cookie', readerCookie)
|
||||||
|
.expect(403);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('partitioned shape (fresh database, real migrations)', () => {
|
||||||
|
const baseUrl = process.env.TEST_DATABASE_URL!;
|
||||||
|
const dbName = `dorfteich_trail_${suffix}`;
|
||||||
|
let fresh: PrismaClient;
|
||||||
|
let maintenance: ReadTrailMaintenanceService;
|
||||||
|
const auditRecord = vi.fn().mockResolvedValue(undefined);
|
||||||
|
let retentionDays = 365;
|
||||||
|
|
||||||
|
function freshUrl(): string {
|
||||||
|
const url = new URL(baseUrl);
|
||||||
|
url.pathname = `/${dbName}`;
|
||||||
|
return url.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
const admin = new PrismaClient({ datasourceUrl: baseUrl });
|
||||||
|
try {
|
||||||
|
await admin.$executeRawUnsafe(`CREATE DATABASE "${dbName}"`);
|
||||||
|
} finally {
|
||||||
|
await admin.$disconnect();
|
||||||
|
}
|
||||||
|
execFileSync(
|
||||||
|
process.execPath,
|
||||||
|
[join(API_ROOT, 'node_modules', 'prisma', 'build', 'index.js'), 'migrate', 'deploy'],
|
||||||
|
{ env: { ...process.env, DATABASE_URL: freshUrl() }, stdio: 'pipe', cwd: API_ROOT },
|
||||||
|
);
|
||||||
|
fresh = new PrismaClient({ datasourceUrl: freshUrl() });
|
||||||
|
const settingsStub = {
|
||||||
|
get: async () => retentionDays,
|
||||||
|
} as unknown as InstanceSettingsService;
|
||||||
|
const auditStub = { record: auditRecord } as never;
|
||||||
|
const loggerStub = {
|
||||||
|
setContext: () => undefined,
|
||||||
|
debug: () => undefined,
|
||||||
|
info: () => undefined,
|
||||||
|
warn: () => undefined,
|
||||||
|
error: () => undefined,
|
||||||
|
} as unknown as PinoLogger;
|
||||||
|
maintenance = new ReadTrailMaintenanceService(
|
||||||
|
fresh as never,
|
||||||
|
settingsStub,
|
||||||
|
auditStub,
|
||||||
|
new ClockService(),
|
||||||
|
loggerStub,
|
||||||
|
);
|
||||||
|
}, 120_000);
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await fresh.$disconnect();
|
||||||
|
const admin = new PrismaClient({ datasourceUrl: baseUrl });
|
||||||
|
try {
|
||||||
|
await admin.$executeRawUnsafe(`DROP DATABASE IF EXISTS "${dbName}" WITH (FORCE)`);
|
||||||
|
} finally {
|
||||||
|
await admin.$disconnect();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('migrates read_events to a partitioned parent with current-month coverage', async () => {
|
||||||
|
const kind = await fresh.$queryRaw<{ relkind: string }[]>`
|
||||||
|
SELECT relkind::text FROM pg_class WHERE relname = 'read_events'`;
|
||||||
|
expect(kind[0]!.relkind).toBe('p');
|
||||||
|
|
||||||
|
const partitions = await fresh.$queryRaw<{ relname: string }[]>`
|
||||||
|
SELECT c.relname::text FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
|
||||||
|
WHERE i.inhparent = 'read_events'::regclass ORDER BY c.relname`;
|
||||||
|
const names = partitions.map((p) => p.relname);
|
||||||
|
expect(names).toContain('read_events_default');
|
||||||
|
expect(names.some((n) => /^read_events_y\d{4}m\d{2}$/.test(n))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps the dedup unique pair enforced per partition (P2002 on the duplicate)', async () => {
|
||||||
|
const row = {
|
||||||
|
actorId: null,
|
||||||
|
sessionKey: 'anon',
|
||||||
|
pageId: null,
|
||||||
|
pondId: 'pond-part',
|
||||||
|
channel: 'page_view',
|
||||||
|
classification: 'vs_nfd',
|
||||||
|
dedupKey: `dup-${suffix}`,
|
||||||
|
windowBucket: 42n,
|
||||||
|
windowSeconds: 300,
|
||||||
|
};
|
||||||
|
await fresh.readEvent.create({ data: row });
|
||||||
|
await expect(fresh.readEvent.create({ data: { ...row } })).rejects.toMatchObject({
|
||||||
|
code: 'P2002',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('creates months ahead and prunes whole expired partitions by DROP, audited', async () => {
|
||||||
|
await maintenance.ensurePartitions();
|
||||||
|
const next = new Date();
|
||||||
|
const nextMonth = new Date(Date.UTC(next.getUTCFullYear(), next.getUTCMonth() + 2, 1));
|
||||||
|
const nextName = `read_events_y${nextMonth.getUTCFullYear()}m${String(
|
||||||
|
nextMonth.getUTCMonth() + 1,
|
||||||
|
).padStart(2, '0')}`;
|
||||||
|
const created = await fresh.$queryRawUnsafe<{ relname: string }[]>(
|
||||||
|
`SELECT relname::text FROM pg_class WHERE relname = '${nextName}'`,
|
||||||
|
);
|
||||||
|
expect(created).toHaveLength(1);
|
||||||
|
// ...and each new partition carries its own dedup unique index.
|
||||||
|
const index = await fresh.$queryRawUnsafe<{ indexname: string }[]>(
|
||||||
|
`SELECT indexname::text FROM pg_indexes WHERE tablename = '${nextName}'
|
||||||
|
AND indexname = '${nextName}_dedup_key'`,
|
||||||
|
);
|
||||||
|
expect(index).toHaveLength(1);
|
||||||
|
|
||||||
|
// An old month: partition + one event well past retention.
|
||||||
|
await fresh.$executeRawUnsafe(
|
||||||
|
`CREATE TABLE "read_events_y2020m01" PARTITION OF "read_events"
|
||||||
|
FOR VALUES FROM ('2020-01-01') TO ('2020-02-01')`,
|
||||||
|
);
|
||||||
|
await fresh.readEvent.create({
|
||||||
|
data: {
|
||||||
|
occurredAt: new Date('2020-01-15T12:00:00Z'),
|
||||||
|
actorId: null,
|
||||||
|
sessionKey: 'anon',
|
||||||
|
pageId: null,
|
||||||
|
pondId: 'pond-part',
|
||||||
|
channel: 'export',
|
||||||
|
classification: 'vs_nfd',
|
||||||
|
dedupKey: `old-${suffix}`,
|
||||||
|
windowBucket: 1n,
|
||||||
|
windowSeconds: 300,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const pruned = await maintenance.pruneExpired();
|
||||||
|
expect(pruned).toBeGreaterThanOrEqual(1);
|
||||||
|
const gone = await fresh.$queryRawUnsafe<{ relname: string }[]>(
|
||||||
|
`SELECT relname::text FROM pg_class WHERE relname = 'read_events_y2020m01'`,
|
||||||
|
);
|
||||||
|
expect(gone).toHaveLength(0);
|
||||||
|
expect(auditRecord).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({ action: 'read_trail.pruned' }),
|
||||||
|
);
|
||||||
|
// The current month's events survive.
|
||||||
|
const kept = await fresh.readEvent.count({ where: { dedupKey: `dup-${suffix}` } });
|
||||||
|
expect(kept).toBe(1);
|
||||||
|
retentionDays = 365; // restore for any later use
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -1,10 +1,16 @@
|
|||||||
import { Global, Module } from '@nestjs/common';
|
import { Global, Module, OnModuleInit } from '@nestjs/common';
|
||||||
|
|
||||||
import { CommonModule } from '../common/common.module';
|
import { CommonModule } from '../common/common.module';
|
||||||
|
import { SchedulerModule } from '../scheduler/scheduler.module';
|
||||||
|
import { SchedulerService } from '../scheduler/scheduler.service';
|
||||||
import { SettingsModule } from '../settings/settings.module';
|
import { SettingsModule } from '../settings/settings.module';
|
||||||
|
|
||||||
|
import { ReadTrailMaintenanceService } from './read-trail-maintenance.service';
|
||||||
import { ReadTrailService } from './read-trail.service';
|
import { ReadTrailService } from './read-trail.service';
|
||||||
|
|
||||||
|
/** Daily, per operations.md's maintenance-jobs table (issue #224). */
|
||||||
|
const READ_TRAIL_MAINTENANCE_CADENCE_SECONDS = 24 * 60 * 60;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Global like AuditModule and for the same reason: the read-access trail
|
* Global like AuditModule and for the same reason: the read-access trail
|
||||||
* (issue #222, ADR 0023) cuts across every module that serves page content —
|
* (issue #222, ADR 0023) cuts across every module that serves page content —
|
||||||
@ -12,8 +18,23 @@ import { ReadTrailService } from './read-trail.service';
|
|||||||
*/
|
*/
|
||||||
@Global()
|
@Global()
|
||||||
@Module({
|
@Module({
|
||||||
imports: [CommonModule, SettingsModule],
|
imports: [CommonModule, SchedulerModule, SettingsModule],
|
||||||
providers: [ReadTrailService],
|
providers: [ReadTrailService, ReadTrailMaintenanceService],
|
||||||
exports: [ReadTrailService],
|
exports: [ReadTrailService, ReadTrailMaintenanceService],
|
||||||
})
|
})
|
||||||
export class ReadTrailModule {}
|
export class ReadTrailModule implements OnModuleInit {
|
||||||
|
constructor(
|
||||||
|
private readonly scheduler: SchedulerService,
|
||||||
|
private readonly maintenance: ReadTrailMaintenanceService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
onModuleInit(): void {
|
||||||
|
this.scheduler.register({
|
||||||
|
name: 'read-trail-maintenance',
|
||||||
|
cadenceSeconds: READ_TRAIL_MAINTENANCE_CADENCE_SECONDS,
|
||||||
|
run: async () => {
|
||||||
|
await this.maintenance.run();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -56,6 +56,13 @@ export const INSTANCE_SETTINGS = {
|
|||||||
// 5 minutes keeps a live Yjs session (collab tokens every 60 s) at a
|
// 5 minutes keeps a live Yjs session (collab tokens every 60 s) at a
|
||||||
// bounded ~12 events/hour/page while still evidencing distinct visits.
|
// bounded ~12 events/hour/page while still evidencing distinct visits.
|
||||||
'readTrail.dedupWindowMinutes': z.number().int().min(1).default(5),
|
'readTrail.dedupWindowMinutes': z.number().int().min(1).default(5),
|
||||||
|
// Read-trail retention (issue #224): days a read event is kept before the
|
||||||
|
// daily maintenance job removes it — deliberately independent of
|
||||||
|
// `audit.retentionDays` (#196), because volume, purpose and legal basis
|
||||||
|
// differ. The deletion itself is audited (`read_trail.pruned`), so a gap
|
||||||
|
// is always explainable. One year mirrors the audit default; shortening
|
||||||
|
// it is an operator decision under the purpose limitation (#225).
|
||||||
|
'readTrail.retentionDays': z.number().int().min(1).default(365),
|
||||||
// Default VS-NfD classification for newly created pages (ADR 0022,
|
// Default VS-NfD classification for newly created pages (ADR 0022,
|
||||||
// issue #204). An instance operated inside a classified environment sets
|
// issue #204). An instance operated inside a classified environment sets
|
||||||
// this to `vs_nfd` so nothing starts unmarked; inheritance from the
|
// this to `vs_nfd` so nothing starts unmarked; inheritance from the
|
||||||
|
|||||||
@ -20,8 +20,9 @@ test('lists maintenance jobs and triggers one manually', async ({ browser }) =>
|
|||||||
// Keep in sync with the scheduler registrations: trash-purge,
|
// Keep in sync with the scheduler registrations: trash-purge,
|
||||||
// version-thinning, page-compaction, data-export-purge,
|
// version-thinning, page-compaction, data-export-purge,
|
||||||
// notification-digest, orphan-file-sweep (#194), audit-retention (#196),
|
// notification-digest, orphan-file-sweep (#194), audit-retention (#196),
|
||||||
// conversion-payload-prune (#233), mail-outbox-retention (#234).
|
// conversion-payload-prune (#233), mail-outbox-retention (#234),
|
||||||
await expect(jobsTable.locator('tbody tr')).toHaveCount(9);
|
// read-trail-maintenance (#224).
|
||||||
|
await expect(jobsTable.locator('tbody tr')).toHaveCount(10);
|
||||||
|
|
||||||
const firstRow = jobsTable.locator('tbody tr').first();
|
const firstRow = jobsTable.locator('tbody tr').first();
|
||||||
await firstRow.getByRole('button').click();
|
await firstRow.getByRole('button').click();
|
||||||
|
|||||||
@ -89,6 +89,23 @@ traffic per open document.
|
|||||||
anonymous browsers apart would require fingerprinting (IP/UA), which the
|
anonymous browsers apart would require fingerprinting (IP/UA), which the
|
||||||
purpose limitation (#225) rules out.
|
purpose limitation (#225) rules out.
|
||||||
|
|
||||||
|
## Decisions taken in #224
|
||||||
|
|
||||||
|
- **Monthly RANGE partitions** on `occurred_at`, plus a DEFAULT partition
|
||||||
|
as a safety net: a lagging maintenance job must never make classified
|
||||||
|
reads fail (the trail write is hard-failing — an ops miss must not
|
||||||
|
become an outage). Retention DROPs whole expired months without
|
||||||
|
scanning; the pruning run is audited (`read_trail.pruned`) under the
|
||||||
|
trail's own `readTrail.retentionDays` (default 365).
|
||||||
|
- **The dedup unique pair lives per partition** (PostgreSQL cannot carry
|
||||||
|
it on the parent without the partition key). A dedup bucket spanning a
|
||||||
|
month boundary can therefore record one duplicate — over-recording is
|
||||||
|
acceptable, gaps are not.
|
||||||
|
- **Query path is API-only** (`GET /admin/system/read-events`, Site
|
||||||
|
Admin): the trail is an examiner's tool, not a daily screen. Evidence
|
||||||
|
nobody can read is not evidence — the path exists; a panel does not.
|
||||||
|
- **Growth measured**: ~1 MB per 1000 events including indexes.
|
||||||
|
|
||||||
## Consequences
|
## Consequences
|
||||||
|
|
||||||
- The scope limit is the feature's strongest argument in the works-council
|
- The scope limit is the feature's strongest argument in the works-council
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
# Audit event catalogue
|
# Audit event catalogue
|
||||||
|
|
||||||
**Catalogue version 1.1 (2026-07-31; 1.1 adds `page.classification_*`,
|
**Catalogue version 1.2 (2026-07-31; 1.2 adds `read_trail.pruned`,
|
||||||
issue #205).**
|
issue #224; 1.1 added `page.classification_*`, issue #205).**
|
||||||
|
|
||||||
This is the operator-facing contract for the audit trail: every event id
|
This is the operator-facing contract for the audit trail: every event id
|
||||||
the application can emit, with its trigger, severity, actor/target
|
the application can emit, with its trigger, severity, actor/target
|
||||||
@ -103,11 +103,12 @@ failure), `warning` = feeds detection (suspicious or destructive),
|
|||||||
|
|
||||||
### Content integrity & lifecycle (`file.*`, `pond.*`, `audit.*`)
|
### Content integrity & lifecycle (`file.*`, `pond.*`, `audit.*`)
|
||||||
|
|
||||||
| Id | Trigger | Severity | Actor | Target | Fields |
|
| Id | Trigger | Severity | Actor | Target | Fields |
|
||||||
| ----------------------- | ------------------------------------------------------------------- | -------- | ---------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- |
|
| ----------------------- | -------------------------------------------------------------------- | -------- | ---------------------------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `file.integrity_failed` | Attachment download hash mismatch — fail-closed (issue #199) | critical | `null` (any downloader; detection) | `attachment` | `pondId`, `expected` (stored sha256), `actual` (computed sha256) |
|
| `file.integrity_failed` | Attachment download hash mismatch — fail-closed (issue #199) | critical | `null` (any downloader; detection) | `attachment` | `pondId`, `expected` (stored sha256), `actual` (computed sha256) |
|
||||||
| `pond.purged` | Pond irreversibly destroyed (manual or trash retention, issue #193) | notice | admin, `null` when retention-run | `pond` | `trigger` (`manual` \| `retention`) plus per-object-type deletion counts (e.g. `pages`, `attachments`, … — informational, keys may grow) |
|
| `pond.purged` | Pond irreversibly destroyed (manual or trash retention, issue #193) | notice | admin, `null` when retention-run | `pond` | `trigger` (`manual` \| `retention`) plus per-object-type deletion counts (e.g. `pages`, `attachments`, … — informational, keys may grow) |
|
||||||
| `audit.pruned` | Audit retention deleted rows past the period (issue #196) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` |
|
| `audit.pruned` | Audit retention deleted rows past the period (issue #196) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` |
|
||||||
|
| `read_trail.pruned` | Read-trail retention removed events past its own period (issue #224) | info | `null` (system) | — | `count`, `cutoff` (ISO), `retentionDays` |
|
||||||
|
|
||||||
### Public API (`api.*`)
|
### Public API (`api.*`)
|
||||||
|
|
||||||
|
|||||||
@ -186,6 +186,40 @@ Cached counters per pond: `storage_bytes_used`, `editor_count`,
|
|||||||
- `jobs`: conversion jobs for import/export (ADR 0009) and maintenance jobs
|
- `jobs`: conversion jobs for import/export (ADR 0009) and maintenance jobs
|
||||||
(compaction, trash purge, quota reconciliation) with status + timestamps.
|
(compaction, trash purge, quota reconciliation) with status + timestamps.
|
||||||
|
|
||||||
|
## Read-access trail (`read_events`, issues #222–#224, ADR 0023)
|
||||||
|
|
||||||
|
One row per read of a `classification = vs_nfd` page per channel and dedup
|
||||||
|
window — separate from `audit_log` because volume, purpose and legal basis
|
||||||
|
all differ.
|
||||||
|
|
||||||
|
| Column | Notes |
|
||||||
|
| ---------------------------- | ------------------------------------------------------------------------------------ |
|
||||||
|
| `id`, `occurred_at` | composite PK (the partition key must be part of it) |
|
||||||
|
| `actor_id` | reader, null = anonymous; **no FK** — evidence outlives accounts |
|
||||||
|
| `session_key` | `session:<id>` / `token:<id>` / `job:<id>` / `anon` |
|
||||||
|
| `page_id`, `pond_id` | plain ids, **no FK** — evidence outlives purges |
|
||||||
|
| `channel` | `page_view` / `no_js_shell` / `public_api` / `attachment` / `export` / `collab_join` |
|
||||||
|
| `classification` | at read time; reclassification never rewrites history |
|
||||||
|
| `dedup_key`, `window_bucket` | unique pair per partition — race-free dedup (#223) |
|
||||||
|
| `window_seconds` | the row represents up to this many seconds of access |
|
||||||
|
| `details` | small context (export format, attachment id) — never content |
|
||||||
|
|
||||||
|
Storage shape (issue #224): RANGE-partitioned by month on `occurred_at`,
|
||||||
|
with a DEFAULT partition as safety net (a lagging maintenance job must
|
||||||
|
never make classified reads fail — the trail write is hard-failing by
|
||||||
|
design). The daily `read-trail-maintenance` job creates months ahead
|
||||||
|
(each with its per-partition dedup index) and applies the trail's own
|
||||||
|
retention `readTrail.retentionDays` (default 365): whole expired months
|
||||||
|
are DROPped without scanning, remainders deleted by range; every pruning
|
||||||
|
run is audited as `read_trail.pruned`. Growth, measured on PG 17:
|
||||||
|
**~1 MB per 1000 events** including indexes (≈970 bytes/row) — a
|
||||||
|
100-person instance at hundreds of classified reads/day stays in the tens
|
||||||
|
of MB per year.
|
||||||
|
|
||||||
|
Query path: `GET /admin/system/read-events` (Site Admin, filters
|
||||||
|
`pageId`, `actor`, `channel`, `from`/`to`) — deliberately API-only, no
|
||||||
|
panel: the trail is an examiner's tool, not a daily screen.
|
||||||
|
|
||||||
## Comments & notifications (later milestone)
|
## Comments & notifications (later milestone)
|
||||||
|
|
||||||
- `comments`: `page_id`, `author_id`, `body` (Markdown), `anchor`
|
- `comments`: `page_id`, `author_id`, `body` (Markdown), `anchor`
|
||||||
|
|||||||
@ -109,6 +109,7 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack.
|
|||||||
| conversion payload prune | daily | null finished conversion jobs' raw bytes (#233) |
|
| conversion payload prune | daily | null finished conversion jobs' raw bytes (#233) |
|
||||||
| mail outbox retention | daily | delete sent/failed outbox rows past period (#234) |
|
| mail outbox retention | daily | delete sent/failed outbox rows past period (#234) |
|
||||||
| mail outbox retry | every minute | e-mail delivery with backoff |
|
| mail outbox retry | every minute | e-mail delivery with backoff |
|
||||||
|
| read-trail maintenance | daily | partition months ahead + prune `read_events` (#224) |
|
||||||
|
|
||||||
Job outcomes are visible in the Site Admin UI (last run, status) — that
|
Job outcomes are visible in the Site Admin UI (last run, status) — that
|
||||||
panel is the operator's single glance for instance health.
|
panel is the operator's single glance for instance health.
|
||||||
|
|||||||
@ -136,7 +136,7 @@ Protokolliert werden Lesezugriffe **ausschließlich** auf Seiten mit
|
|||||||
Attachment-Download, Export, No-JS-Shell, Collab-WS-Join · 4 AT · #222
|
Attachment-Download, Export, No-JS-Shell, Collab-WS-Join · 4 AT · #222
|
||||||
- [x] Dedup-Fenster (eine Sitzung + eine Seite innerhalb N Minuten = ein
|
- [x] Dedup-Fenster (eine Sitzung + eine Seite innerhalb N Minuten = ein
|
||||||
Ereignis), sonst erzeugt Yjs-Sync eine Ereignisflut · 2 AT · #223
|
Ereignis), sonst erzeugt Yjs-Sync eine Ereignisflut · 2 AT · #223
|
||||||
- [ ] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224
|
- [x] Getrennte Tabelle mit eigener Retention und Partitionierung · 2 AT · #224
|
||||||
- [ ] Abschaltbar, Zweckbindung dokumentiert · 1–2 AT · #225
|
- [ ] Abschaltbar, Zweckbindung dokumentiert · 1–2 AT · #225
|
||||||
|
|
||||||
Vorteil über den Aufwand hinaus: Die Zweckbindung ist sauber begründbar
|
Vorteil über den Aufwand hinaus: Die Zweckbindung ist sauber begründbar
|
||||||
|
|||||||
@ -40,6 +40,7 @@ Settings-Cache ist in-process (operations.md).
|
|||||||
| `backup.nextcloud.enabled` | `false` | `false` | „Backup nur lokal": kein Anwendungs-Upload von Restore-Sets zu Drittdiensten. Fernspiegel regelt ausschließlich die Deploy-Allowlist (1.2). |
|
| `backup.nextcloud.enabled` | `false` | `false` | „Backup nur lokal": kein Anwendungs-Upload von Restore-Sets zu Drittdiensten. Fernspiegel regelt ausschließlich die Deploy-Allowlist (1.2). |
|
||||||
| `trash.retentionDays`, `audit.retentionDays`, `conversion.payloadRetentionDays`, `mail.outboxRetentionDays` | Defaults (30/365/30/30) | ebd. | Aufbewahrung bewusst begrenzt; Verkürzung nach Betreiber-Löschkonzept zulässig (Betriebshandbuch §5). |
|
| `trash.retentionDays`, `audit.retentionDays`, `conversion.payloadRetentionDays`, `mail.outboxRetentionDays` | Defaults (30/365/30/30) | ebd. | Aufbewahrung bewusst begrenzt; Verkürzung nach Betreiber-Löschkonzept zulässig (Betriebshandbuch §5). |
|
||||||
| `readTrail.dedupWindowMinutes` | Default (5) | `5` | Dedup-Fenster des Lesetrails (#223): je (Sitzung, Seite, Kanal) ein Ereignis pro Fenster — begrenzt die Ereignisflut einer Live-Sitzung auf ~12/h. Kleiner = feineres Protokoll und mehr Zeilen; Änderung mit dem Zweckbindungs-Dokument (#225) abstimmen. |
|
| `readTrail.dedupWindowMinutes` | Default (5) | `5` | Dedup-Fenster des Lesetrails (#223): je (Sitzung, Seite, Kanal) ein Ereignis pro Fenster — begrenzt die Ereignisflut einer Live-Sitzung auf ~12/h. Kleiner = feineres Protokoll und mehr Zeilen; Änderung mit dem Zweckbindungs-Dokument (#225) abstimmen. |
|
||||||
|
| `readTrail.retentionDays` | Default (365) | `365` | eigene Aufbewahrung des Lesetrails (#224), bewusst getrennt von `audit.retentionDays`; Löschläufe sind selbst auditiert (`read_trail.pruned`). Dauer mit der Zweckbindung (#225) und dem Betreiber-Löschkonzept abstimmen. |
|
||||||
| `legal.imprint`, `legal.privacyPolicy` | befüllt | leer | Betreiberpflicht; leere Seiten zeigen einen Warnbanner. |
|
| `legal.imprint`, `legal.privacyPolicy` | befüllt | leer | Betreiberpflicht; leere Seiten zeigen einen Warnbanner. |
|
||||||
|
|
||||||
### 1.2 Deploy-Konfiguration (`.env` / Compose — nur Plattformzugriff, bewusst nicht per Admin-UI)
|
### 1.2 Deploy-Konfiguration (`.env` / Compose — nur Plattformzugriff, bewusst nicht per Admin-UI)
|
||||||
|
|||||||
@ -164,6 +164,49 @@ export interface AuditListView {
|
|||||||
total: number;
|
total: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Site-Admin query path over the read-access trail (issue #224, ADR 0023) —
|
||||||
|
* evidence nobody can read is not evidence. Answers the two expected
|
||||||
|
* questions: "who read page X" (pageId) and "what did user Y read" (actor),
|
||||||
|
* both within a period.
|
||||||
|
*/
|
||||||
|
export const READ_EVENT_PAGE_SIZE = 50;
|
||||||
|
|
||||||
|
export const readEventListQuerySchema = z.object({
|
||||||
|
/** Page id the events belong to. */
|
||||||
|
pageId: z.string().trim().min(1).optional(),
|
||||||
|
/** Exact username of the reading user. */
|
||||||
|
actor: z.string().trim().min(1).optional(),
|
||||||
|
channel: z.string().trim().min(1).optional(),
|
||||||
|
from: z.coerce.date().optional(),
|
||||||
|
to: z.coerce.date().optional(),
|
||||||
|
page: z.coerce.number().int().min(1).default(1),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type ReadEventListQuery = z.infer<typeof readEventListQuerySchema>;
|
||||||
|
|
||||||
|
export interface ReadEventView {
|
||||||
|
id: string;
|
||||||
|
occurredAt: string;
|
||||||
|
actor: AuditActorView | null;
|
||||||
|
/** Null for pond-level attachments (no page); the details name the file. */
|
||||||
|
pageId: string | null;
|
||||||
|
pondId: string;
|
||||||
|
channel: string;
|
||||||
|
classification: string;
|
||||||
|
/** Seconds the event's dedup window spans (#223) — the row represents up
|
||||||
|
* to this much access time, not a single request. */
|
||||||
|
windowSeconds: number;
|
||||||
|
details: Record<string, unknown> | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ReadEventListView {
|
||||||
|
entries: ReadEventView[];
|
||||||
|
page: number;
|
||||||
|
pageCount: number;
|
||||||
|
total: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface StoragePondView {
|
export interface StoragePondView {
|
||||||
pondId: string;
|
pondId: string;
|
||||||
name: string;
|
name: string;
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user