From 462eca96995a3d60a6073df59d3077a5f67a8a24 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.8" Date: Fri, 10 Jul 2026 13:01:51 +0200 Subject: [PATCH] Add self-service GDPR data export (#68) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A signed-in account can export all of its own data — profile, a list of its memberships/grants, and the Markdown of its personal pond plus the shared ponds it owns — as one ZIP. Foreign content never appears: only owned ponds are bundled and the per-page read filter (reused from #65) runs for each. - Reuse the conversion-job queue as the async carrier: a `data_export` job whose worker branch resolves DataExportService via a token (no DI cycle), builds the ZIP, and stores it with an `expiresAt`. The download link 404s past expiry and an hourly scheduled purge drops the bytes (data minimization, security.md §Privacy). - Extract ExportService.appendPondMarkdown so the pond ZIP (#65) and the data export share one read-filtered pond archiver. - Rate-limit requests per account (RateLimitService); POST /users/me/data-export enqueues, GET /jobs/:id(/result) poll/download. - Settings UI "Export my data" (de+en); web share pollJob/downloadJobResult between the document and data export hooks. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1 --- .../migration.sql | 2 + apps/api/prisma/schema.prisma | 4 + .../import-export/conversion-job.service.ts | 5 + .../conversion-worker.service.ts | 36 ++- .../import-export/data-export.constants.ts | 23 ++ .../import-export/data-export.controller.ts | 24 ++ .../data-export.service.db.test.ts | 299 ++++++++++++++++++ .../src/import-export/data-export.service.ts | 174 ++++++++++ apps/api/src/import-export/export.service.ts | 43 ++- .../src/import-export/import-export.module.ts | 38 ++- apps/api/src/import-export/import.service.ts | 1 + apps/web/src/export/job-download.ts | 46 +++ apps/web/src/export/use-data-export.ts | 54 ++++ apps/web/src/export/use-document-export.ts | 40 +-- apps/web/src/pages/SettingsPage.tsx | 48 +++ packages/shared/i18n/de/settings.json | 11 + packages/shared/i18n/en/settings.json | 11 + packages/shared/src/conversion.ts | 4 + 18 files changed, 804 insertions(+), 59 deletions(-) create mode 100644 apps/api/prisma/migrations/20260710120000_data_export_expiry/migration.sql create mode 100644 apps/api/src/import-export/data-export.constants.ts create mode 100644 apps/api/src/import-export/data-export.controller.ts create mode 100644 apps/api/src/import-export/data-export.service.db.test.ts create mode 100644 apps/api/src/import-export/data-export.service.ts create mode 100644 apps/web/src/export/job-download.ts create mode 100644 apps/web/src/export/use-data-export.ts diff --git a/apps/api/prisma/migrations/20260710120000_data_export_expiry/migration.sql b/apps/api/prisma/migrations/20260710120000_data_export_expiry/migration.sql new file mode 100644 index 0000000..d36b87f --- /dev/null +++ b/apps/api/prisma/migrations/20260710120000_data_export_expiry/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "conversion_jobs" ADD COLUMN "expires_at" TIMESTAMP(3); diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index e77ccf4..e1fec80 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -550,6 +550,10 @@ model ConversionJob { resultMimeType String? @map("result_mime_type") errorCode String? @map("error_code") lockedAt DateTime? @map("locked_at") + /// For a data-export job (#68): when its stored result stops being + /// downloadable and is purged (GDPR data minimization). Null for every + /// other job kind, whose result never expires. + expiresAt DateTime? @map("expires_at") createdAt DateTime @default(now()) @map("created_at") updatedAt DateTime @updatedAt @map("updated_at") diff --git a/apps/api/src/import-export/conversion-job.service.ts b/apps/api/src/import-export/conversion-job.service.ts index 5713f3b..514bfe4 100644 --- a/apps/api/src/import-export/conversion-job.service.ts +++ b/apps/api/src/import-export/conversion-job.service.ts @@ -89,6 +89,9 @@ export class ConversionJobService { async resultForOwner(id: string, userId: string): Promise { const job = await this.ownedJob(id, userId); if (job.status !== 'SUCCEEDED' || !job.result) throw new NotFoundException(); + // A data-export link expires (#68): past its window the result is treated as + // gone (a scheduled purge deletes the bytes), so the download 404s. + if (job.expiresAt && job.expiresAt.getTime() < Date.now()) throw new NotFoundException(); return { bytes: Buffer.from(job.result), mimeType: job.resultMimeType ?? 'application/octet-stream', @@ -111,6 +114,8 @@ export class ConversionJobService { errorCode: job.errorCode, // Set once an import job succeeds (#63) so the client can open the page. resultPageId: job.resultPageId, + // Set for a data-export job (#68): when its download link stops working. + expiresAt: job.expiresAt?.toISOString() ?? null, createdAt: job.createdAt.toISOString(), updatedAt: job.updatedAt.toISOString(), }; diff --git a/apps/api/src/import-export/conversion-worker.service.ts b/apps/api/src/import-export/conversion-worker.service.ts index b8805b2..28114bc 100644 --- a/apps/api/src/import-export/conversion-worker.service.ts +++ b/apps/api/src/import-export/conversion-worker.service.ts @@ -6,6 +6,12 @@ import { PinoLogger } from 'nestjs-pino'; import { AppConfig } from '../config/app-config.service'; import { PrismaService } from '../prisma/prisma.service'; +import { + DATA_EXPORT_KIND, + DATA_EXPORT_PROCESSOR, + DATA_EXPORT_TTL_MS, + DataExportProcessor, +} from './data-export.constants'; import { GotenbergRenderer, RenderError } from './gotenberg.renderer'; import { IMPORT_PROCESSOR, ImportProcessor, isImportKind } from './import.constants'; import { ConversionError, ConversionResult, PandocConverter } from './pandoc.converter'; @@ -127,15 +133,26 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy { await this.moduleRef.get(IMPORT_PROCESSOR, { strict: false }).run(job); return; } - // PDF export renders HTML through Gotenberg (#67); everything else is a - // pandoc byte→byte conversion (#62/#65). - const output = - job.targetFormat === 'pdf' - ? { - bytes: await this.renderer.renderHtmlToPdf(Buffer.from(job.input).toString('utf8')), - mimeType: 'application/pdf', - } - : await this.convert(job); + // A data export gathers the account's own data into a ZIP (#68) whose + // download link expires; a PDF export renders HTML through Gotenberg + // (#67); everything else is a pandoc byte→byte conversion (#62/#65). + let output: { bytes: Buffer; mimeType: string }; + let expiresAt: Date | null = null; + if (job.kind === DATA_EXPORT_KIND) { + // Resolved lazily via a token so this file never imports the export + // service's (avoids a construction cycle), mirroring the import path. + output = await this.moduleRef + .get(DATA_EXPORT_PROCESSOR, { strict: false }) + .build(job); + expiresAt = new Date(Date.now() + DATA_EXPORT_TTL_MS); + } else if (job.targetFormat === 'pdf') { + output = { + bytes: await this.renderer.renderHtmlToPdf(Buffer.from(job.input).toString('utf8')), + mimeType: 'application/pdf', + }; + } else { + output = await this.convert(job); + } await this.prisma.conversionJob.update({ where: { id: job.id }, data: { @@ -144,6 +161,7 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy { result: new Uint8Array(output.bytes), resultMimeType: output.mimeType, errorCode: null, + expiresAt, }, }); this.logger.info( diff --git a/apps/api/src/import-export/data-export.constants.ts b/apps/api/src/import-export/data-export.constants.ts new file mode 100644 index 0000000..0672e67 --- /dev/null +++ b/apps/api/src/import-export/data-export.constants.ts @@ -0,0 +1,23 @@ +import type { ConversionJob } from '@prisma/client'; + +/** + * Wiring for the GDPR data export (issue #68). The conversion worker builds a + * `data_export` job through this token — never by importing the service — so + * the worker's file stays free of a construction cycle (mirrors the import + * pipeline's {@link ./import.constants.ts}). + */ +export const DATA_EXPORT_PROCESSOR = Symbol('DATA_EXPORT_PROCESSOR'); + +/** The {@link ConversionJob.kind} of a self-service account data export. */ +export const DATA_EXPORT_KIND = 'data_export'; + +/** How long a generated export stays downloadable before its bytes are purged + * (security.md §Privacy: data minimization — the archive is a transient copy, + * not stored indefinitely). */ +export const DATA_EXPORT_TTL_MS = 24 * 60 * 60 * 1000; + +/** Builds the ZIP for a `data_export` job. The worker resolves this behind the + * token above and stores the returned bytes as the job result. */ +export interface DataExportProcessor { + build(job: ConversionJob): Promise<{ bytes: Buffer; mimeType: string }>; +} diff --git a/apps/api/src/import-export/data-export.controller.ts b/apps/api/src/import-export/data-export.controller.ts new file mode 100644 index 0000000..5c281bb --- /dev/null +++ b/apps/api/src/import-export/data-export.controller.ts @@ -0,0 +1,24 @@ +import { Controller, Post, Req } from '@nestjs/common'; +import { ConversionJobView } from '@dorfteich/shared'; + +import { AuthedRequest } from '../auth/auth.guard'; +import { AuthenticatedOnly } from '../permissions/permission.decorators'; + +import { DataExportService } from './data-export.service'; + +/** + * Self-service GDPR data export (issue #68, security.md §Privacy). A signed-in + * account requests its own export here; the job is owner-scoped, so polling and + * download stay on the generic `GET /jobs/:id(/result)` routes (JobsController). + */ +@AuthenticatedOnly() +@Controller('users/me') +export class DataExportController { + constructor(private readonly dataExport: DataExportService) {} + + /** Enqueue an export of the caller's own data; rate-limited per account. */ + @Post('data-export') + request(@Req() request: AuthedRequest): Promise { + return this.dataExport.request(request.user!); + } +} diff --git a/apps/api/src/import-export/data-export.service.db.test.ts b/apps/api/src/import-export/data-export.service.db.test.ts new file mode 100644 index 0000000..cade6c4 --- /dev/null +++ b/apps/api/src/import-export/data-export.service.db.test.ts @@ -0,0 +1,299 @@ +import { INestApplication } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; +import { unzipSync } from 'fflate'; +import request from 'supertest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; + +import { AuthTokensService } from '../auth/auth-tokens.service'; +import { FilesService } from '../files/files.service'; +import { createTestApp, sessionCookieOf } from '../testing/test-app'; +import { createTestPrisma, grantOwnerAdmin, hasTestDb, uniqueSuffix } from '../testing/test-db'; +import { UsersService } from '../users/users.service'; + +import { ConversionWorker } from './conversion-worker.service'; +import { DataExportService } from './data-export.service'; + +/** GDPR data export (issue #68): a signed-in account assembles its own profile, + * memberships, and the Markdown of every pond it owns into one ZIP — with a + * download link that expires — and never another user's content. */ + +const PNG_BASE64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + +function zipEntries(buffer: Buffer): Record { + return unzipSync(new Uint8Array(buffer)); +} + +function textOf(entries: Record, name: string): string { + return Buffer.from(entries[name]!).toString('utf8'); +} + +async function downloadZip(server: unknown, path: string, cookie: string): Promise { + const res = await request(server as never) + .get(path) + .set('Cookie', cookie) + .buffer(true) + .parse((r, cb) => { + const chunks: Buffer[] = []; + r.on('data', (c: Buffer) => chunks.push(c)); + r.on('end', () => cb(null, Buffer.concat(chunks))); + }) + .expect(200); + return res.body as Buffer; +} + +describe.skipIf(!hasTestDb)('data export (e2e, issue #68)', () => { + let app: INestApplication; + let prisma: PrismaClient; + let worker: ConversionWorker; + let files: FilesService; + const suffix = uniqueSuffix(); + const password = 'exportiere alle meine daten 1'; + + let ownerId: string; + let ownerCookie: string; + let personalSlug: string; + let ownedSharedSlug: string; + let foreignSlug: string; + + const api = () => request(app.getHttpServer()); + + async function login(username: string): Promise { + const res = await api() + .post('/api/v1/auth/login') + .send({ usernameOrEmail: username, password }) + .expect(200); + return sessionCookieOf(res); + } + + async function createUser(username: string): Promise { + const user = await app.get(UsersService).createUser({ + username, + email: `${username}@example.org`, + displayName: username, + password, + locale: 'en', + }); + // Verify via the endpoint (not a direct flag flip) so the personal pond and + // its owner-admin grant are created exactly as in production (#52). + const token = await app.get(AuthTokensService).issue(user.id, 'EMAIL_VERIFICATION', 600); + await api().post('/api/v1/auth/verify-email').send({ token }).expect(204); + return user.id; + } + + async function seedPage(pondId: string, title: string, markdown: string): Promise { + await prisma.page.create({ + data: { + pondId, + title, + slug: title.toLowerCase().replace(/[^a-z0-9]+/g, '-'), + ydocState: new Uint8Array(), + sortKey: title, + createdBy: ownerId, + contentCache: { create: { plainText: markdown, markdown, html: '', outline: [] } }, + }, + }); + } + + beforeAll(async () => { + prisma = createTestPrisma(); + await prisma.rateLimit.deleteMany({}); + app = await createTestApp(); + worker = app.get(ConversionWorker); + files = app.get(FilesService); + + ownerId = await createUser(`odette-owner-${suffix}`); + ownerCookie = await login(`odette-owner-${suffix}`); + + const personal = await prisma.pond.findFirstOrThrow({ + where: { ownerId, type: 'PERSONAL' }, + }); + personalSlug = personal.slug; + + // A shared pond the owner created — its content belongs in the export. + const shared = await prisma.pond.create({ + data: { + slug: `owned-shared-${suffix}`, + name: 'Owned Shared', + type: 'SHARED', + ownerId, + usage: { create: {} }, + }, + }); + ownedSharedSlug = shared.slug; + await grantOwnerAdmin(prisma, shared.id, ownerId); + + // A stored image referenced from a personal-pond page (media inclusion). + const image = await files.upload({ id: ownerId } as never, personal.id, { + buffer: Buffer.from(PNG_BASE64, 'base64'), + size: 70, + originalname: 'dot.png', + }); + await seedPage(personal.id, 'Diary', `# Diary\n\nMine. ![dot](${image.id})`); + await seedPage(shared.id, 'Shared Note', '# Shared Note\n\nAlso mine.'); + + // A foreign user's pond the owner is only a *reader* of: it must show up in + // memberships, but its content must never appear in the export. + const foreignId = await createUser(`fred-foreign-${suffix}`); + const foreign = await prisma.pond.create({ + data: { + slug: `foreign-secret-${suffix}`, + name: 'Foreign Secret', + type: 'SHARED', + ownerId: foreignId, + usage: { create: {} }, + }, + }); + foreignSlug = foreign.slug; + await prisma.page.create({ + data: { + pondId: foreign.id, + title: 'Foreign Page', + slug: 'foreign-page', + ydocState: new Uint8Array(), + sortKey: 'Foreign Page', + createdBy: foreignId, + contentCache: { + create: { plainText: 'TOP SECRET', markdown: 'TOP SECRET', html: '', outline: [] }, + }, + }, + }); + await prisma.roleGrant.create({ + data: { + pondId: foreign.id, + subjectType: 'USER', + subjectId: ownerId, + role: 'READER', + scopeType: 'POND', + effect: 'ALLOW', + createdBy: foreignId, + }, + }); + }); + + afterAll(async () => { + await prisma.conversionJob.deleteMany({ where: { owner: { username: { contains: suffix } } } }); + const where = { pond: { owner: { username: { contains: suffix } } } }; + await prisma.attachment.deleteMany({ where }); + await prisma.roleGrant.deleteMany({ + where: { pond: { owner: { username: { contains: suffix } } } }, + }); + await prisma.page.deleteMany({ where }); + await prisma.pond.deleteMany({ where: { owner: { username: { contains: suffix } } } }); + await prisma.rateLimit.deleteMany({}); + await prisma.user.deleteMany({ where: { username: { contains: suffix } } }); + await prisma.$disconnect(); + await app.close(); + }); + + // Each test starts from a clean rate-limit table so the per-account export + // limit (exercised deliberately in one test) never trips the others. + beforeEach(async () => { + await prisma.rateLimit.deleteMany({}); + }); + + it('exports the caller profile, memberships, and owned pond content only', async () => { + const enqueued = await api() + .post('/api/v1/users/me/data-export') + .set('Cookie', ownerCookie) + .expect(201); + expect(enqueued.body.kind).toBe('data_export'); + expect(enqueued.body.status).toBe('pending'); + + await worker.drain(); + + const done = await api() + .get(`/api/v1/jobs/${enqueued.body.id}`) + .set('Cookie', ownerCookie) + .expect(200); + expect(done.body.status).toBe('succeeded'); + // The download link carries an expiry (a future timestamp). + expect(done.body.expiresAt).toBeTruthy(); + expect(new Date(done.body.expiresAt).getTime()).toBeGreaterThan(Date.now()); + + const zip = await downloadZip( + app.getHttpServer(), + `/api/v1/jobs/${enqueued.body.id}/result`, + ownerCookie, + ); + const entries = zipEntries(zip); + const names = Object.keys(entries); + + // Profile: the caller's own account fields. + const profile = JSON.parse(textOf(entries, 'profile.json')); + expect(profile.username).toBe(`odette-owner-${suffix}`); + expect(profile.email).toBe(`odette-owner-${suffix}@example.org`); + + // Personal + owned-shared pond content is present, media included. + expect(names).toContain(`ponds/${personalSlug}/diary.md`); + expect(names.some((n) => n.startsWith(`ponds/${personalSlug}/media/`))).toBe(true); + expect(names).toContain(`ponds/${ownedSharedSlug}/shared-note.md`); + + // Memberships list the foreign pond the owner reads… + const memberships = JSON.parse(textOf(entries, 'memberships.json')) as { + pondSlug: string; + }[]; + expect(memberships.map((m) => m.pondSlug)).toContain(foreignSlug); + + // …but none of the foreign pond's content appears anywhere in the archive. + expect(names.some((n) => n.startsWith(`ponds/${foreignSlug}/`))).toBe(false); + for (const name of names) { + if (name.endsWith('.md')) expect(textOf(entries, name)).not.toContain('TOP SECRET'); + } + }); + + it('404s the download once the link has expired', async () => { + const enqueued = await api() + .post('/api/v1/users/me/data-export') + .set('Cookie', ownerCookie) + .expect(201); + await worker.drain(); + // Backdate the expiry to simulate an elapsed link. + await prisma.conversionJob.update({ + where: { id: enqueued.body.id }, + data: { expiresAt: new Date(Date.now() - 1000) }, + }); + + await api() + .get(`/api/v1/jobs/${enqueued.body.id}/result`) + .set('Cookie', ownerCookie) + .expect(404); + }); + + it('purges the stored bytes of an expired export', async () => { + const enqueued = await api() + .post('/api/v1/users/me/data-export') + .set('Cookie', ownerCookie) + .expect(201); + await worker.drain(); + await prisma.conversionJob.update({ + where: { id: enqueued.body.id }, + data: { expiresAt: new Date(Date.now() - 1000) }, + }); + + const purged = await app.get(DataExportService).purgeExpired(); + expect(purged).toBeGreaterThanOrEqual(1); + const row = await prisma.conversionJob.findUniqueOrThrow({ where: { id: enqueued.body.id } }); + expect(row.result).toBeNull(); + }); + + it('rate-limits repeated requests from the same account', async () => { + await createUser(`irene-idle-${suffix}`); + const cookie = await login(`irene-idle-${suffix}`); + // The limit is a few requests per window; the run past it is rejected. + for (let i = 0; i < 3; i += 1) { + await api().post('/api/v1/users/me/data-export').set('Cookie', cookie).expect(201); + } + await api().post('/api/v1/users/me/data-export').set('Cookie', cookie).expect(429); + }); + + it('keeps a data-export job owner-scoped (a foreign user cannot poll it)', async () => { + const enqueued = await api() + .post('/api/v1/users/me/data-export') + .set('Cookie', ownerCookie) + .expect(201); + await createUser(`sam-stranger-${suffix}`); + const strangerCookie = await login(`sam-stranger-${suffix}`); + await api().get(`/api/v1/jobs/${enqueued.body.id}`).set('Cookie', strangerCookie).expect(404); + }); +}); diff --git a/apps/api/src/import-export/data-export.service.ts b/apps/api/src/import-export/data-export.service.ts new file mode 100644 index 0000000..b863cf9 --- /dev/null +++ b/apps/api/src/import-export/data-export.service.ts @@ -0,0 +1,174 @@ +import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; +import { ConversionJobView } from '@dorfteich/shared'; +import { ConversionJob, User } from '@prisma/client'; +import archiver from 'archiver'; +import { PinoLogger } from 'nestjs-pino'; + +import { RateLimitService } from '../rate-limit/rate-limit.service'; +import { PrismaService } from '../prisma/prisma.service'; + +import { ConversionJobService } from './conversion-job.service'; +import { DATA_EXPORT_KIND, DataExportProcessor } from './data-export.constants'; +import { ExportService } from './export.service'; + +/** A data export may be requested at most this many times per window per + * account (security.md §Privacy: access requests are self-service but must not + * be a cheap way to hammer the server). */ +const MAX_REQUESTS_PER_WINDOW = 3; +const WINDOW_SECONDS = 60 * 60; + +/** + * GDPR data export (issue #68): a signed-in user assembles all of their own + * data — profile, a list of their memberships/grants, and the Markdown export + * of their personal pond and the shared ponds they own — into a single ZIP. + * It is produced as a conversion job (so a large account exports out of band) + * with a download link that expires; foreign content never appears, because + * only the requester's own ponds are included and the per-page read filter + * (reused from #65) runs for every one. + */ +@Injectable() +export class DataExportService implements DataExportProcessor { + constructor( + private readonly prisma: PrismaService, + private readonly exports: ExportService, + private readonly jobs: ConversionJobService, + private readonly rateLimits: RateLimitService, + private readonly logger: PinoLogger, + ) { + this.logger.setContext(DataExportService.name); + } + + /** Enqueue a data-export job for `user`, rate-limited per account. The client + * polls `GET /jobs/:id` and downloads `GET /jobs/:id/result` once it + * succeeds; the link stops working after the job's `expiresAt`. */ + async request(user: User): Promise { + const limit = await this.rateLimits.hit( + 'data_export', + `user:${user.id}`, + MAX_REQUESTS_PER_WINDOW, + WINDOW_SECONDS, + ); + if (!limit.allowed) { + throw new HttpException( + { code: 'rate_limited', details: { retryAfterSeconds: limit.retryAfterSeconds } }, + HttpStatus.TOO_MANY_REQUESTS, + ); + } + + const job = await this.jobs.enqueue({ + ownerId: user.id, + kind: DATA_EXPORT_KIND, + from: 'account', + to: 'zip', + // A data export gathers everything from the database at build time, so the + // job needs no input payload. + input: Buffer.alloc(0), + }); + this.logger.info({ jobId: job.id, userId: user.id }, 'audit: data export requested'); + return this.jobs.viewOf(job); + } + + /** Worker entry point (via {@link DATA_EXPORT_PROCESSOR}): build the ZIP for a + * requested export. */ + async build(job: ConversionJob): Promise<{ bytes: Buffer; mimeType: string }> { + const user = await this.prisma.user.findUniqueOrThrow({ where: { id: job.ownerId } }); + + const archive = archiver('zip', { zlib: { level: 9 } }); + const chunks: Buffer[] = []; + archive.on('data', (chunk: Buffer) => chunks.push(chunk)); + const finished = new Promise((resolve, reject) => { + archive.on('end', () => resolve()); + archive.on('error', reject); + }); + + archive.append(JSON.stringify(this.profileOf(user), null, 2), { name: 'profile.json' }); + archive.append(JSON.stringify(await this.membershipsOf(user), null, 2), { + name: 'memberships.json', + }); + + // Only ponds this user owns — their personal pond and any shared ponds they + // created. Foreign ponds they are merely a member of are represented by the + // memberships list, not their content. + const ponds = await this.prisma.pond.findMany({ + where: { ownerId: user.id, deletedAt: null }, + select: { id: true, slug: true }, + orderBy: { slug: 'asc' }, + }); + for (const pond of ponds) { + await this.exports.appendPondMarkdown(archive, user, pond, `ponds/${pond.slug}/`); + } + + await archive.finalize(); + await finished; + this.logger.info( + { jobId: job.id, userId: user.id, ponds: ponds.length }, + 'audit: data export built', + ); + return { bytes: Buffer.concat(chunks), mimeType: 'application/zip' }; + } + + /** Drop the stored bytes of every export whose link has expired (data + * minimization: the archive is transient). The row remains for its audit + * trail; only the payload is deleted, so a later download 404s. Registered as + * a scheduled maintenance job. */ + async purgeExpired(): Promise { + const result = await this.prisma.conversionJob.updateMany({ + where: { kind: DATA_EXPORT_KIND, expiresAt: { lt: new Date() }, result: { not: null } }, + data: { result: null, resultMimeType: null }, + }); + if (result.count > 0) { + this.logger.info({ purged: result.count }, 'audit: expired data exports purged'); + } + return result.count; + } + + /** The requester's own account fields — the data minimization set the app + * collects (security.md §Privacy): identity, locale, and lifecycle stamps. */ + private profileOf(user: User): Record { + return { + id: user.id, + username: user.username, + email: user.email, + displayName: user.displayName, + locale: user.locale, + isSiteAdmin: user.isSiteAdmin, + status: user.status, + emailVerifiedAt: user.emailVerifiedAt?.toISOString() ?? null, + createdAt: user.createdAt.toISOString(), + lastLoginAt: user.lastLoginAt?.toISOString() ?? null, + }; + } + + /** Every pond the user has a role grant in, with the grants that concern + * them. This is the user's own membership data — pond identity plus their + * roles — and deliberately carries none of those ponds' content. */ + private async membershipsOf(user: User): Promise { + const grants = await this.prisma.roleGrant.findMany({ + where: { subjectType: 'USER', subjectId: user.id }, + include: { pond: { select: { slug: true, name: true, type: true, ownerId: true } } }, + orderBy: { createdAt: 'asc' }, + }); + const byPond = new Map< + string, + { pondSlug: string; pondName: string; pondType: string; owner: boolean; grants: unknown[] } + >(); + for (const grant of grants) { + const entry = byPond.get(grant.pondId) ?? { + pondSlug: grant.pond.slug, + pondName: grant.pond.name, + pondType: grant.pond.type, + owner: grant.pond.ownerId === user.id, + grants: [], + }; + entry.grants.push({ + role: grant.role, + scopeType: grant.scopeType, + scopeId: grant.scopeId, + effect: grant.effect, + createdAt: grant.createdAt.toISOString(), + }); + byPond.set(grant.pondId, entry); + } + return [...byPond.values()]; + } +} diff --git a/apps/api/src/import-export/export.service.ts b/apps/api/src/import-export/export.service.ts index 380fcba..3efdc32 100644 --- a/apps/api/src/import-export/export.service.ts +++ b/apps/api/src/import-export/export.service.ts @@ -59,6 +59,34 @@ export class ExportService { const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } }); if (!pond) throw new NotFoundException(); + const archive = archiver('zip', { zlib: { level: 9 } }); + res.set('Content-Type', 'application/zip'); + res.set('Content-Disposition', `attachment; filename="${pond.slug}.zip"`); + res.set('X-Content-Type-Options', 'nosniff'); + archive.on('error', (error) => { + this.logger.error({ pondId, err: error.message }, 'pond export archive failed'); + res.destroy(error); + }); + archive.pipe(res); + await this.appendPondMarkdown(archive, user, pond); + await archive.finalize(); + } + + /** + * Append one pond's readable pages (as Markdown) and their media to an open + * archive, each entry under `prefix`. Shared by the pond ZIP above and the + * account data export (#68), which nests several ponds under `ponds//` + * — the read filter runs here, so neither caller can leak a page the + * requester may not read. Page Markdown is appended as small strings; media + * as read streams, keeping memory bounded regardless of pond size. + */ + async appendPondMarkdown( + archive: archiver.Archiver, + user: User, + pond: { id: string; slug: string }, + prefix = '', + ): Promise { + const pondId = pond.id; const pages = await this.prisma.page.findMany({ where: { pondId, deletedAt: null }, orderBy: { title: 'asc' }, @@ -99,16 +127,6 @@ export class ExportService { attachments.map((a) => [a.id, `${a.id}.${imageExtension(a.mimeType)}`]), ); - const archive = archiver('zip', { zlib: { level: 9 } }); - res.set('Content-Type', 'application/zip'); - res.set('Content-Disposition', `attachment; filename="${pond.slug}.zip"`); - res.set('X-Content-Type-Options', 'nosniff'); - archive.on('error', (error) => { - this.logger.error({ pondId, err: error.message }, 'pond export archive failed'); - res.destroy(error); - }); - archive.pipe(res); - for (const page of readablePages) { const markdown = markdownForZip( page.contentCache?.markdown ?? '', @@ -116,7 +134,7 @@ export class ExportService { mediaNameById, ); // Page slugs are unique within a pond, so `.md` never collides. - archive.append(markdown, { name: `${page.slug}.md` }); + archive.append(markdown, { name: `${prefix}${page.slug}.md` }); } for (const attachment of attachments) { const stream = this.storage.createReadStream(pond.id, attachment.id); @@ -128,13 +146,12 @@ export class ExportService { 'export: media read failed', ), ); - archive.append(stream, { name: `media/${mediaNameById.get(attachment.id)!}` }); + archive.append(stream, { name: `${prefix}media/${mediaNameById.get(attachment.id)!}` }); } this.logger.info( { pondId, pages: readablePages.length, media: attachments.length, userId: user.id }, 'audit: pond exported as markdown zip', ); - await archive.finalize(); } /** diff --git a/apps/api/src/import-export/import-export.module.ts b/apps/api/src/import-export/import-export.module.ts index 24768c4..0bf2928 100644 --- a/apps/api/src/import-export/import-export.module.ts +++ b/apps/api/src/import-export/import-export.module.ts @@ -1,10 +1,15 @@ -import { Module } from '@nestjs/common'; +import { Module, OnModuleInit } from '@nestjs/common'; import { FilesModule } from '../files/files.module'; import { PagesModule } from '../pages/pages.module'; +import { SchedulerModule } from '../scheduler/scheduler.module'; +import { SchedulerService } from '../scheduler/scheduler.service'; import { ConversionJobService } from './conversion-job.service'; import { ConversionWorker } from './conversion-worker.service'; +import { DATA_EXPORT_PROCESSOR } from './data-export.constants'; +import { DataExportController } from './data-export.controller'; +import { DataExportService } from './data-export.service'; import { ExportController } from './export.controller'; import { ExportService } from './export.service'; import { GotenbergHttpRenderer, GotenbergRenderer } from './gotenberg.renderer'; @@ -14,23 +19,29 @@ import { ImportService } from './import.service'; import { JobsController } from './jobs.controller'; import { PandocConverter, PandocServerConverter } from './pandoc.converter'; +/** How often expired data-export payloads are purged (#68). Hourly is ample: + * the link's own expiry check already stops downloads the moment it lapses. */ +const EXPORT_PURGE_CADENCE_SECONDS = 60 * 60; + /** * Import/export orchestration (ADR 0009): the conversion job queue, its worker, - * the pandoc-server client (#62), and the document import pipeline (#63, which - * turns an uploaded `.docx`/`.odt` into a new page). Export (#65) will add its - * feature endpoints here too. + * the pandoc-server client (#62), the document import pipeline (#63), the + * feature exports (#65/#67), and the GDPR account data export (#68). */ @Module({ - imports: [FilesModule, PagesModule], - controllers: [JobsController, ImportController, ExportController], + imports: [FilesModule, PagesModule, SchedulerModule], + controllers: [JobsController, ImportController, ExportController, DataExportController], providers: [ ConversionJobService, ConversionWorker, ImportService, ExportService, + DataExportService, // The worker resolves the import pipeline through this token (never the // class), so its file does not import the import service's (avoids a cycle). { provide: IMPORT_PROCESSOR, useExisting: ImportService }, + // Same token indirection for the data-export builder (#68). + { provide: DATA_EXPORT_PROCESSOR, useExisting: DataExportService }, // Bind the abstract converter to the HTTP implementation; tests override // this provider with a fake so the queue mechanics need no live sidecar. { provide: PandocConverter, useClass: PandocServerConverter }, @@ -39,4 +50,17 @@ import { PandocConverter, PandocServerConverter } from './pandoc.converter'; ], exports: [ConversionJobService, PandocConverter], }) -export class ImportExportModule {} +export class ImportExportModule implements OnModuleInit { + constructor( + private readonly scheduler: SchedulerService, + private readonly dataExport: DataExportService, + ) {} + + onModuleInit(): void { + this.scheduler.register({ + name: 'data-export-purge', + cadenceSeconds: EXPORT_PURGE_CADENCE_SECONDS, + run: () => this.dataExport.purgeExpired().then(() => undefined), + }); + } +} diff --git a/apps/api/src/import-export/import.service.ts b/apps/api/src/import-export/import.service.ts index dc383f9..c1d2555 100644 --- a/apps/api/src/import-export/import.service.ts +++ b/apps/api/src/import-export/import.service.ts @@ -171,6 +171,7 @@ export class ImportService implements ImportProcessor { targetFormat: 'page', errorCode: null, resultPageId: page.id, + expiresAt: null, createdAt: now, updatedAt: now, }; diff --git a/apps/web/src/export/job-download.ts b/apps/web/src/export/job-download.ts new file mode 100644 index 0000000..912d830 --- /dev/null +++ b/apps/web/src/export/job-download.ts @@ -0,0 +1,46 @@ +import type { ConversionJobView } from '@dorfteich/shared'; + +import { ApiError, apiGet } from '../lib/api'; + +const POLL_INTERVAL_MS = 1000; +/** Default poll budget — a whole-account export (#68) can take longer than a + * single-page conversion, so callers may raise or lower it. */ +const DEFAULT_MAX_POLLS = 300; + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Poll a conversion job until it succeeds or fails, or the poll budget runs + * out (returns whatever the last status was). Shared by every export hook. */ +export async function pollJob( + job: ConversionJobView, + maxPolls = DEFAULT_MAX_POLLS, +): Promise { + let current = job; + for ( + let poll = 0; + current.status !== 'succeeded' && current.status !== 'failed' && poll < maxPolls; + poll += 1 + ) { + await delay(POLL_INTERVAL_MS); + current = await apiGet(`/jobs/${current.id}`); + } + return current; +} + +/** Download a finished job's result and save it as `fileName`. The result + * endpoint is session-authenticated, so a plain `fetch` carries the cookie. */ +export async function downloadJobResult(jobId: string, fileName: string): Promise { + const response = await fetch(`/api/v1/jobs/${jobId}/result`); + if (!response.ok) { + throw new ApiError(response.status, { code: `http_${response.status}`, message: '' }); + } + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + const anchor = document.createElement('a'); + anchor.href = url; + anchor.download = fileName; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + URL.revokeObjectURL(url); +} diff --git a/apps/web/src/export/use-data-export.ts b/apps/web/src/export/use-data-export.ts new file mode 100644 index 0000000..7f1101e --- /dev/null +++ b/apps/web/src/export/use-data-export.ts @@ -0,0 +1,54 @@ +import type { ConversionJobView } from '@dorfteich/shared'; +import { useCallback, useState } from 'react'; + +import { ApiError, apiPost } from '../lib/api'; + +import { downloadJobResult, pollJob } from './job-download'; + +export type DataExportStatus = 'idle' | 'busy' | 'ready' | 'error' | 'rateLimited'; + +export interface UseDataExport { + status: DataExportStatus; + /** When the ready download link expires (ISO), or null. */ + expiresAt: string | null; + request: () => void; + download: () => void; +} + +/** + * Self-service GDPR data export (issue #68): request the export, poll the job, + * and — once it is ready — download the ZIP. A recent request is rejected with + * 429, surfaced as a distinct `rateLimited` status. + */ +export function useDataExport(): UseDataExport { + const [status, setStatus] = useState('idle'); + const [expiresAt, setExpiresAt] = useState(null); + const [jobId, setJobId] = useState(null); + + const request = useCallback((): void => { + void (async () => { + setStatus('busy'); + setExpiresAt(null); + setJobId(null); + try { + const job = await pollJob(await apiPost('/users/me/data-export')); + if (job.status === 'succeeded') { + setJobId(job.id); + setExpiresAt(job.expiresAt); + setStatus('ready'); + } else { + setStatus('error'); + } + } catch (error) { + setStatus(error instanceof ApiError && error.status === 429 ? 'rateLimited' : 'error'); + } + })(); + }, []); + + const download = useCallback((): void => { + if (!jobId) return; + void downloadJobResult(jobId, 'dorfteich-export.zip').catch(() => setStatus('error')); + }, [jobId]); + + return { status, expiresAt, request, download }; +} diff --git a/apps/web/src/export/use-document-export.ts b/apps/web/src/export/use-document-export.ts index 941eb36..cd6c19d 100644 --- a/apps/web/src/export/use-document-export.ts +++ b/apps/web/src/export/use-document-export.ts @@ -1,31 +1,16 @@ import type { ConversionJobView, ExportFormat } from '@dorfteich/shared'; import { useCallback, useState } from 'react'; -import { ApiError, apiGet, apiPost } from '../lib/api'; +import { apiPost } from '../lib/api'; + +import { downloadJobResult, pollJob } from './job-download'; export type ExportStatus = 'idle' | 'busy' | 'error'; -const POLL_INTERVAL_MS = 1000; +/** A single-page conversion is quick — cap the poll budget below the shared + * default. */ const MAX_POLLS = 180; -const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); - -/** Fetch a finished job's result and save it under `fileName`. The result - * endpoint is session-authenticated, so a plain `fetch` carries the cookie. */ -async function downloadResult(jobId: string, fileName: string): Promise { - const response = await fetch(`/api/v1/jobs/${jobId}/result`); - if (!response.ok) throw new ApiError(response.status, { code: 'conversion_failed', message: '' }); - const blob = await response.blob(); - const url = URL.createObjectURL(blob); - const anchor = document.createElement('a'); - anchor.href = url; - anchor.download = fileName; - document.body.appendChild(anchor); - anchor.click(); - anchor.remove(); - URL.revokeObjectURL(url); -} - export interface UseDocumentExport { /** Per-format status so each button shows its own progress/error. */ status: Partial>; @@ -48,17 +33,12 @@ export function useDocumentExport(): UseDocumentExport { void (async () => { set(format, 'busy'); try { - let job = await apiPost(`/pages/${pageId}/export`, { format }); - for ( - let poll = 0; - job.status !== 'succeeded' && job.status !== 'failed' && poll < MAX_POLLS; - poll += 1 - ) { - await delay(POLL_INTERVAL_MS); - job = await apiGet(`/jobs/${job.id}`); - } + const job = await pollJob( + await apiPost(`/pages/${pageId}/export`, { format }), + MAX_POLLS, + ); if (job.status === 'succeeded') { - await downloadResult(job.id, `${slug}.${format}`); + await downloadJobResult(job.id, `${slug}.${format}`); set(format, 'idle'); } else { set(format, 'error'); diff --git a/apps/web/src/pages/SettingsPage.tsx b/apps/web/src/pages/SettingsPage.tsx index be64990..9bd5590 100644 --- a/apps/web/src/pages/SettingsPage.tsx +++ b/apps/web/src/pages/SettingsPage.tsx @@ -7,6 +7,7 @@ import { useTranslation } from 'react-i18next'; import { useAuth } from '../auth/auth-context'; import { Field, FormError, FormSuccess, applyFieldErrors } from '../components/forms'; +import { useDataExport } from '../export/use-data-export'; import { apiDelete, apiGet, apiPatch, apiPost } from '../lib/api'; interface SessionView { @@ -25,10 +26,57 @@ export function SettingsPage(): React.JSX.Element { + ); } +function DataExportSection(): React.JSX.Element { + const { t, i18n } = useTranslation(); + const { status, expiresAt, request, download } = useDataExport(); + + const formatTime = (iso: string): string => + new Intl.DateTimeFormat(i18n.language, { dateStyle: 'medium', timeStyle: 'short' }).format( + new Date(iso), + ); + + return ( +
+

{t('settings:dataExport.title')}

+

{t('settings:dataExport.description')}

+ {status === 'error' && ( +

+ {t('settings:dataExport.failed')} +

+ )} + {status === 'rateLimited' && ( +

+ {t('settings:dataExport.rateLimited')} +

+ )} + {status === 'ready' ? ( + <> + + + {expiresAt && ( +

+ {t('settings:dataExport.expiresHint', { when: formatTime(expiresAt) })} +

+ )} + + ) : ( + + )} +
+ ); +} + function ProfileSection(): React.JSX.Element { const { t, i18n } = useTranslation(); const { user, refresh } = useAuth(); diff --git a/packages/shared/i18n/de/settings.json b/packages/shared/i18n/de/settings.json index ada199f..156ab32 100644 --- a/packages/shared/i18n/de/settings.json +++ b/packages/shared/i18n/de/settings.json @@ -25,6 +25,17 @@ "revokeAll": "Alle anderen Sitzungen abmelden", "empty": "Keine weiteren aktiven Sitzungen." }, + "dataExport": { + "title": "Meine Daten exportieren", + "description": "Lade ein ZIP mit deinem Profil, einer Liste deiner Mitgliedschaften und dem Markdown-Export deines persönlichen Teichs sowie der Teiche, die dir gehören.", + "request": "Export vorbereiten", + "preparing": "Dein Export wird vorbereitet…", + "download": "Export herunterladen", + "ready": "Dein Export ist bereit.", + "expiresHint": "Der Download-Link läuft am {{when}} ab.", + "failed": "Der Export konnte nicht erstellt werden. Bitte versuche es erneut.", + "rateLimited": "Du hast kürzlich einen Export angefordert. Bitte versuche es später erneut." + }, "admin": { "title": "Administration", "instanceName": "Name der Instanz", diff --git a/packages/shared/i18n/en/settings.json b/packages/shared/i18n/en/settings.json index 1395014..aca94db 100644 --- a/packages/shared/i18n/en/settings.json +++ b/packages/shared/i18n/en/settings.json @@ -25,6 +25,17 @@ "revokeAll": "Sign out all other sessions", "empty": "No other active sessions." }, + "dataExport": { + "title": "Export my data", + "description": "Download a ZIP with your profile, a list of your memberships, and the Markdown export of your personal pond and the ponds you own.", + "request": "Prepare export", + "preparing": "Preparing your export…", + "download": "Download export", + "ready": "Your export is ready.", + "expiresHint": "The download link expires on {{when}}.", + "failed": "The export could not be prepared. Please try again.", + "rateLimited": "You requested an export recently. Please try again later." + }, "admin": { "title": "Administration", "instanceName": "Instance name", diff --git a/packages/shared/src/conversion.ts b/packages/shared/src/conversion.ts index ea7cd9e..6bc1d9d 100644 --- a/packages/shared/src/conversion.ts +++ b/packages/shared/src/conversion.ts @@ -21,6 +21,10 @@ export interface ConversionJobView { * the client can navigate to it. `null` for a pending/failed import and for * plain byte→byte conversions (export). */ resultPageId: string | null; + /** For a data-export job (#68), when its download link stops working — the + * result is purged after this. `null` for every other job kind, which never + * expires. */ + expiresAt: string | null; createdAt: string; updatedAt: string; }