Add self-service GDPR data export (#68)
All checks were successful
CD / Build and push images (push) Successful in 10m39s
CI / Lint, typecheck, test (push) Successful in 3m12s
CI / Auth e2e pack (push) Successful in 4m9s
CI / Build container images (push) Has been skipped
CD / Deploy to Test (push) Successful in 9s
CD / Smoke tests against Test (push) Successful in 1m18s
CD / Promote to Int (push) Successful in 11s

A signed-in account can export all of its own data — profile, a list of
its memberships/grants, and the Markdown of its personal pond plus the
shared ponds it owns — as one ZIP. Foreign content never appears: only
owned ponds are bundled and the per-page read filter (reused from #65)
runs for each.

- Reuse the conversion-job queue as the async carrier: a `data_export`
  job whose worker branch resolves DataExportService via a token (no DI
  cycle), builds the ZIP, and stores it with an `expiresAt`. The download
  link 404s past expiry and an hourly scheduled purge drops the bytes
  (data minimization, security.md §Privacy).
- Extract ExportService.appendPondMarkdown so the pond ZIP (#65) and the
  data export share one read-filtered pond archiver.
- Rate-limit requests per account (RateLimitService); POST
  /users/me/data-export enqueues, GET /jobs/:id(/result) poll/download.
- Settings UI "Export my data" (de+en); web share pollJob/downloadJobResult
  between the document and data export hooks.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1
This commit is contained in:
Claude Opus 4.8 2026-07-10 13:01:51 +02:00
parent 8a68ef68e7
commit 462eca9699
18 changed files with 804 additions and 59 deletions

View File

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "conversion_jobs" ADD COLUMN "expires_at" TIMESTAMP(3);

View File

@ -550,6 +550,10 @@ model ConversionJob {
resultMimeType String? @map("result_mime_type")
errorCode String? @map("error_code")
lockedAt DateTime? @map("locked_at")
/// For a data-export job (#68): when its stored result stops being
/// downloadable and is purged (GDPR data minimization). Null for every
/// other job kind, whose result never expires.
expiresAt DateTime? @map("expires_at")
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")

View File

@ -89,6 +89,9 @@ export class ConversionJobService {
async resultForOwner(id: string, userId: string): Promise<ConversionResultPayload> {
const job = await this.ownedJob(id, userId);
if (job.status !== 'SUCCEEDED' || !job.result) throw new NotFoundException();
// A data-export link expires (#68): past its window the result is treated as
// gone (a scheduled purge deletes the bytes), so the download 404s.
if (job.expiresAt && job.expiresAt.getTime() < Date.now()) throw new NotFoundException();
return {
bytes: Buffer.from(job.result),
mimeType: job.resultMimeType ?? 'application/octet-stream',
@ -111,6 +114,8 @@ export class ConversionJobService {
errorCode: job.errorCode,
// Set once an import job succeeds (#63) so the client can open the page.
resultPageId: job.resultPageId,
// Set for a data-export job (#68): when its download link stops working.
expiresAt: job.expiresAt?.toISOString() ?? null,
createdAt: job.createdAt.toISOString(),
updatedAt: job.updatedAt.toISOString(),
};

View File

@ -6,6 +6,12 @@ import { PinoLogger } from 'nestjs-pino';
import { AppConfig } from '../config/app-config.service';
import { PrismaService } from '../prisma/prisma.service';
import {
DATA_EXPORT_KIND,
DATA_EXPORT_PROCESSOR,
DATA_EXPORT_TTL_MS,
DataExportProcessor,
} from './data-export.constants';
import { GotenbergRenderer, RenderError } from './gotenberg.renderer';
import { IMPORT_PROCESSOR, ImportProcessor, isImportKind } from './import.constants';
import { ConversionError, ConversionResult, PandocConverter } from './pandoc.converter';
@ -127,15 +133,26 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy {
await this.moduleRef.get<ImportProcessor>(IMPORT_PROCESSOR, { strict: false }).run(job);
return;
}
// PDF export renders HTML through Gotenberg (#67); everything else is a
// pandoc byte→byte conversion (#62/#65).
const output =
job.targetFormat === 'pdf'
? {
bytes: await this.renderer.renderHtmlToPdf(Buffer.from(job.input).toString('utf8')),
mimeType: 'application/pdf',
}
: await this.convert(job);
// A data export gathers the account's own data into a ZIP (#68) whose
// download link expires; a PDF export renders HTML through Gotenberg
// (#67); everything else is a pandoc byte→byte conversion (#62/#65).
let output: { bytes: Buffer; mimeType: string };
let expiresAt: Date | null = null;
if (job.kind === DATA_EXPORT_KIND) {
// Resolved lazily via a token so this file never imports the export
// service's (avoids a construction cycle), mirroring the import path.
output = await this.moduleRef
.get<DataExportProcessor>(DATA_EXPORT_PROCESSOR, { strict: false })
.build(job);
expiresAt = new Date(Date.now() + DATA_EXPORT_TTL_MS);
} else if (job.targetFormat === 'pdf') {
output = {
bytes: await this.renderer.renderHtmlToPdf(Buffer.from(job.input).toString('utf8')),
mimeType: 'application/pdf',
};
} else {
output = await this.convert(job);
}
await this.prisma.conversionJob.update({
where: { id: job.id },
data: {
@ -144,6 +161,7 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy {
result: new Uint8Array(output.bytes),
resultMimeType: output.mimeType,
errorCode: null,
expiresAt,
},
});
this.logger.info(

View File

@ -0,0 +1,23 @@
import type { ConversionJob } from '@prisma/client';
/**
* Wiring for the GDPR data export (issue #68). The conversion worker builds a
* `data_export` job through this token never by importing the service so
* the worker's file stays free of a construction cycle (mirrors the import
* pipeline's {@link ./import.constants.ts}).
*/
export const DATA_EXPORT_PROCESSOR = Symbol('DATA_EXPORT_PROCESSOR');
/** The {@link ConversionJob.kind} of a self-service account data export. */
export const DATA_EXPORT_KIND = 'data_export';
/** How long a generated export stays downloadable before its bytes are purged
* (security.md §Privacy: data minimization the archive is a transient copy,
* not stored indefinitely). */
export const DATA_EXPORT_TTL_MS = 24 * 60 * 60 * 1000;
/** Builds the ZIP for a `data_export` job. The worker resolves this behind the
* token above and stores the returned bytes as the job result. */
export interface DataExportProcessor {
build(job: ConversionJob): Promise<{ bytes: Buffer; mimeType: string }>;
}

View File

@ -0,0 +1,24 @@
import { Controller, Post, Req } from '@nestjs/common';
import { ConversionJobView } from '@dorfteich/shared';
import { AuthedRequest } from '../auth/auth.guard';
import { AuthenticatedOnly } from '../permissions/permission.decorators';
import { DataExportService } from './data-export.service';
/**
* Self-service GDPR data export (issue #68, security.md §Privacy). A signed-in
* account requests its own export here; the job is owner-scoped, so polling and
* download stay on the generic `GET /jobs/:id(/result)` routes (JobsController).
*/
@AuthenticatedOnly()
@Controller('users/me')
export class DataExportController {
constructor(private readonly dataExport: DataExportService) {}
/** Enqueue an export of the caller's own data; rate-limited per account. */
@Post('data-export')
request(@Req() request: AuthedRequest): Promise<ConversionJobView> {
return this.dataExport.request(request.user!);
}
}

View File

@ -0,0 +1,299 @@
import { INestApplication } from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import { unzipSync } from 'fflate';
import request from 'supertest';
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { AuthTokensService } from '../auth/auth-tokens.service';
import { FilesService } from '../files/files.service';
import { createTestApp, sessionCookieOf } from '../testing/test-app';
import { createTestPrisma, grantOwnerAdmin, hasTestDb, uniqueSuffix } from '../testing/test-db';
import { UsersService } from '../users/users.service';
import { ConversionWorker } from './conversion-worker.service';
import { DataExportService } from './data-export.service';
/** GDPR data export (issue #68): a signed-in account assembles its own profile,
* memberships, and the Markdown of every pond it owns into one ZIP with a
* download link that expires and never another user's content. */
const PNG_BASE64 =
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';
function zipEntries(buffer: Buffer): Record<string, Uint8Array> {
return unzipSync(new Uint8Array(buffer));
}
function textOf(entries: Record<string, Uint8Array>, name: string): string {
return Buffer.from(entries[name]!).toString('utf8');
}
async function downloadZip(server: unknown, path: string, cookie: string): Promise<Buffer> {
const res = await request(server as never)
.get(path)
.set('Cookie', cookie)
.buffer(true)
.parse((r, cb) => {
const chunks: Buffer[] = [];
r.on('data', (c: Buffer) => chunks.push(c));
r.on('end', () => cb(null, Buffer.concat(chunks)));
})
.expect(200);
return res.body as Buffer;
}
describe.skipIf(!hasTestDb)('data export (e2e, issue #68)', () => {
let app: INestApplication;
let prisma: PrismaClient;
let worker: ConversionWorker;
let files: FilesService;
const suffix = uniqueSuffix();
const password = 'exportiere alle meine daten 1';
let ownerId: string;
let ownerCookie: string;
let personalSlug: string;
let ownedSharedSlug: string;
let foreignSlug: string;
const api = () => request(app.getHttpServer());
async function login(username: string): Promise<string> {
const res = await api()
.post('/api/v1/auth/login')
.send({ usernameOrEmail: username, password })
.expect(200);
return sessionCookieOf(res);
}
async function createUser(username: string): Promise<string> {
const user = await app.get(UsersService).createUser({
username,
email: `${username}@example.org`,
displayName: username,
password,
locale: 'en',
});
// Verify via the endpoint (not a direct flag flip) so the personal pond and
// its owner-admin grant are created exactly as in production (#52).
const token = await app.get(AuthTokensService).issue(user.id, 'EMAIL_VERIFICATION', 600);
await api().post('/api/v1/auth/verify-email').send({ token }).expect(204);
return user.id;
}
async function seedPage(pondId: string, title: string, markdown: string): Promise<void> {
await prisma.page.create({
data: {
pondId,
title,
slug: title.toLowerCase().replace(/[^a-z0-9]+/g, '-'),
ydocState: new Uint8Array(),
sortKey: title,
createdBy: ownerId,
contentCache: { create: { plainText: markdown, markdown, html: '', outline: [] } },
},
});
}
beforeAll(async () => {
prisma = createTestPrisma();
await prisma.rateLimit.deleteMany({});
app = await createTestApp();
worker = app.get(ConversionWorker);
files = app.get(FilesService);
ownerId = await createUser(`odette-owner-${suffix}`);
ownerCookie = await login(`odette-owner-${suffix}`);
const personal = await prisma.pond.findFirstOrThrow({
where: { ownerId, type: 'PERSONAL' },
});
personalSlug = personal.slug;
// A shared pond the owner created — its content belongs in the export.
const shared = await prisma.pond.create({
data: {
slug: `owned-shared-${suffix}`,
name: 'Owned Shared',
type: 'SHARED',
ownerId,
usage: { create: {} },
},
});
ownedSharedSlug = shared.slug;
await grantOwnerAdmin(prisma, shared.id, ownerId);
// A stored image referenced from a personal-pond page (media inclusion).
const image = await files.upload({ id: ownerId } as never, personal.id, {
buffer: Buffer.from(PNG_BASE64, 'base64'),
size: 70,
originalname: 'dot.png',
});
await seedPage(personal.id, 'Diary', `# Diary\n\nMine. ![dot](${image.id})`);
await seedPage(shared.id, 'Shared Note', '# Shared Note\n\nAlso mine.');
// A foreign user's pond the owner is only a *reader* of: it must show up in
// memberships, but its content must never appear in the export.
const foreignId = await createUser(`fred-foreign-${suffix}`);
const foreign = await prisma.pond.create({
data: {
slug: `foreign-secret-${suffix}`,
name: 'Foreign Secret',
type: 'SHARED',
ownerId: foreignId,
usage: { create: {} },
},
});
foreignSlug = foreign.slug;
await prisma.page.create({
data: {
pondId: foreign.id,
title: 'Foreign Page',
slug: 'foreign-page',
ydocState: new Uint8Array(),
sortKey: 'Foreign Page',
createdBy: foreignId,
contentCache: {
create: { plainText: 'TOP SECRET', markdown: 'TOP SECRET', html: '', outline: [] },
},
},
});
await prisma.roleGrant.create({
data: {
pondId: foreign.id,
subjectType: 'USER',
subjectId: ownerId,
role: 'READER',
scopeType: 'POND',
effect: 'ALLOW',
createdBy: foreignId,
},
});
});
afterAll(async () => {
await prisma.conversionJob.deleteMany({ where: { owner: { username: { contains: suffix } } } });
const where = { pond: { owner: { username: { contains: suffix } } } };
await prisma.attachment.deleteMany({ where });
await prisma.roleGrant.deleteMany({
where: { pond: { owner: { username: { contains: suffix } } } },
});
await prisma.page.deleteMany({ where });
await prisma.pond.deleteMany({ where: { owner: { username: { contains: suffix } } } });
await prisma.rateLimit.deleteMany({});
await prisma.user.deleteMany({ where: { username: { contains: suffix } } });
await prisma.$disconnect();
await app.close();
});
// Each test starts from a clean rate-limit table so the per-account export
// limit (exercised deliberately in one test) never trips the others.
beforeEach(async () => {
await prisma.rateLimit.deleteMany({});
});
it('exports the caller profile, memberships, and owned pond content only', async () => {
const enqueued = await api()
.post('/api/v1/users/me/data-export')
.set('Cookie', ownerCookie)
.expect(201);
expect(enqueued.body.kind).toBe('data_export');
expect(enqueued.body.status).toBe('pending');
await worker.drain();
const done = await api()
.get(`/api/v1/jobs/${enqueued.body.id}`)
.set('Cookie', ownerCookie)
.expect(200);
expect(done.body.status).toBe('succeeded');
// The download link carries an expiry (a future timestamp).
expect(done.body.expiresAt).toBeTruthy();
expect(new Date(done.body.expiresAt).getTime()).toBeGreaterThan(Date.now());
const zip = await downloadZip(
app.getHttpServer(),
`/api/v1/jobs/${enqueued.body.id}/result`,
ownerCookie,
);
const entries = zipEntries(zip);
const names = Object.keys(entries);
// Profile: the caller's own account fields.
const profile = JSON.parse(textOf(entries, 'profile.json'));
expect(profile.username).toBe(`odette-owner-${suffix}`);
expect(profile.email).toBe(`odette-owner-${suffix}@example.org`);
// Personal + owned-shared pond content is present, media included.
expect(names).toContain(`ponds/${personalSlug}/diary.md`);
expect(names.some((n) => n.startsWith(`ponds/${personalSlug}/media/`))).toBe(true);
expect(names).toContain(`ponds/${ownedSharedSlug}/shared-note.md`);
// Memberships list the foreign pond the owner reads…
const memberships = JSON.parse(textOf(entries, 'memberships.json')) as {
pondSlug: string;
}[];
expect(memberships.map((m) => m.pondSlug)).toContain(foreignSlug);
// …but none of the foreign pond's content appears anywhere in the archive.
expect(names.some((n) => n.startsWith(`ponds/${foreignSlug}/`))).toBe(false);
for (const name of names) {
if (name.endsWith('.md')) expect(textOf(entries, name)).not.toContain('TOP SECRET');
}
});
it('404s the download once the link has expired', async () => {
const enqueued = await api()
.post('/api/v1/users/me/data-export')
.set('Cookie', ownerCookie)
.expect(201);
await worker.drain();
// Backdate the expiry to simulate an elapsed link.
await prisma.conversionJob.update({
where: { id: enqueued.body.id },
data: { expiresAt: new Date(Date.now() - 1000) },
});
await api()
.get(`/api/v1/jobs/${enqueued.body.id}/result`)
.set('Cookie', ownerCookie)
.expect(404);
});
it('purges the stored bytes of an expired export', async () => {
const enqueued = await api()
.post('/api/v1/users/me/data-export')
.set('Cookie', ownerCookie)
.expect(201);
await worker.drain();
await prisma.conversionJob.update({
where: { id: enqueued.body.id },
data: { expiresAt: new Date(Date.now() - 1000) },
});
const purged = await app.get(DataExportService).purgeExpired();
expect(purged).toBeGreaterThanOrEqual(1);
const row = await prisma.conversionJob.findUniqueOrThrow({ where: { id: enqueued.body.id } });
expect(row.result).toBeNull();
});
it('rate-limits repeated requests from the same account', async () => {
await createUser(`irene-idle-${suffix}`);
const cookie = await login(`irene-idle-${suffix}`);
// The limit is a few requests per window; the run past it is rejected.
for (let i = 0; i < 3; i += 1) {
await api().post('/api/v1/users/me/data-export').set('Cookie', cookie).expect(201);
}
await api().post('/api/v1/users/me/data-export').set('Cookie', cookie).expect(429);
});
it('keeps a data-export job owner-scoped (a foreign user cannot poll it)', async () => {
const enqueued = await api()
.post('/api/v1/users/me/data-export')
.set('Cookie', ownerCookie)
.expect(201);
await createUser(`sam-stranger-${suffix}`);
const strangerCookie = await login(`sam-stranger-${suffix}`);
await api().get(`/api/v1/jobs/${enqueued.body.id}`).set('Cookie', strangerCookie).expect(404);
});
});

View File

@ -0,0 +1,174 @@
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { ConversionJobView } from '@dorfteich/shared';
import { ConversionJob, User } from '@prisma/client';
import archiver from 'archiver';
import { PinoLogger } from 'nestjs-pino';
import { RateLimitService } from '../rate-limit/rate-limit.service';
import { PrismaService } from '../prisma/prisma.service';
import { ConversionJobService } from './conversion-job.service';
import { DATA_EXPORT_KIND, DataExportProcessor } from './data-export.constants';
import { ExportService } from './export.service';
/** A data export may be requested at most this many times per window per
* account (security.md §Privacy: access requests are self-service but must not
* be a cheap way to hammer the server). */
const MAX_REQUESTS_PER_WINDOW = 3;
const WINDOW_SECONDS = 60 * 60;
/**
* GDPR data export (issue #68): a signed-in user assembles all of their own
* data profile, a list of their memberships/grants, and the Markdown export
* of their personal pond and the shared ponds they own into a single ZIP.
* It is produced as a conversion job (so a large account exports out of band)
* with a download link that expires; foreign content never appears, because
* only the requester's own ponds are included and the per-page read filter
* (reused from #65) runs for every one.
*/
@Injectable()
export class DataExportService implements DataExportProcessor {
constructor(
private readonly prisma: PrismaService,
private readonly exports: ExportService,
private readonly jobs: ConversionJobService,
private readonly rateLimits: RateLimitService,
private readonly logger: PinoLogger,
) {
this.logger.setContext(DataExportService.name);
}
/** Enqueue a data-export job for `user`, rate-limited per account. The client
* polls `GET /jobs/:id` and downloads `GET /jobs/:id/result` once it
* succeeds; the link stops working after the job's `expiresAt`. */
async request(user: User): Promise<ConversionJobView> {
const limit = await this.rateLimits.hit(
'data_export',
`user:${user.id}`,
MAX_REQUESTS_PER_WINDOW,
WINDOW_SECONDS,
);
if (!limit.allowed) {
throw new HttpException(
{ code: 'rate_limited', details: { retryAfterSeconds: limit.retryAfterSeconds } },
HttpStatus.TOO_MANY_REQUESTS,
);
}
const job = await this.jobs.enqueue({
ownerId: user.id,
kind: DATA_EXPORT_KIND,
from: 'account',
to: 'zip',
// A data export gathers everything from the database at build time, so the
// job needs no input payload.
input: Buffer.alloc(0),
});
this.logger.info({ jobId: job.id, userId: user.id }, 'audit: data export requested');
return this.jobs.viewOf(job);
}
/** Worker entry point (via {@link DATA_EXPORT_PROCESSOR}): build the ZIP for a
* requested export. */
async build(job: ConversionJob): Promise<{ bytes: Buffer; mimeType: string }> {
const user = await this.prisma.user.findUniqueOrThrow({ where: { id: job.ownerId } });
const archive = archiver('zip', { zlib: { level: 9 } });
const chunks: Buffer[] = [];
archive.on('data', (chunk: Buffer) => chunks.push(chunk));
const finished = new Promise<void>((resolve, reject) => {
archive.on('end', () => resolve());
archive.on('error', reject);
});
archive.append(JSON.stringify(this.profileOf(user), null, 2), { name: 'profile.json' });
archive.append(JSON.stringify(await this.membershipsOf(user), null, 2), {
name: 'memberships.json',
});
// Only ponds this user owns — their personal pond and any shared ponds they
// created. Foreign ponds they are merely a member of are represented by the
// memberships list, not their content.
const ponds = await this.prisma.pond.findMany({
where: { ownerId: user.id, deletedAt: null },
select: { id: true, slug: true },
orderBy: { slug: 'asc' },
});
for (const pond of ponds) {
await this.exports.appendPondMarkdown(archive, user, pond, `ponds/${pond.slug}/`);
}
await archive.finalize();
await finished;
this.logger.info(
{ jobId: job.id, userId: user.id, ponds: ponds.length },
'audit: data export built',
);
return { bytes: Buffer.concat(chunks), mimeType: 'application/zip' };
}
/** Drop the stored bytes of every export whose link has expired (data
* minimization: the archive is transient). The row remains for its audit
* trail; only the payload is deleted, so a later download 404s. Registered as
* a scheduled maintenance job. */
async purgeExpired(): Promise<number> {
const result = await this.prisma.conversionJob.updateMany({
where: { kind: DATA_EXPORT_KIND, expiresAt: { lt: new Date() }, result: { not: null } },
data: { result: null, resultMimeType: null },
});
if (result.count > 0) {
this.logger.info({ purged: result.count }, 'audit: expired data exports purged');
}
return result.count;
}
/** The requester's own account fields the data minimization set the app
* collects (security.md §Privacy): identity, locale, and lifecycle stamps. */
private profileOf(user: User): Record<string, unknown> {
return {
id: user.id,
username: user.username,
email: user.email,
displayName: user.displayName,
locale: user.locale,
isSiteAdmin: user.isSiteAdmin,
status: user.status,
emailVerifiedAt: user.emailVerifiedAt?.toISOString() ?? null,
createdAt: user.createdAt.toISOString(),
lastLoginAt: user.lastLoginAt?.toISOString() ?? null,
};
}
/** Every pond the user has a role grant in, with the grants that concern
* them. This is the user's own membership data pond identity plus their
* roles and deliberately carries none of those ponds' content. */
private async membershipsOf(user: User): Promise<unknown[]> {
const grants = await this.prisma.roleGrant.findMany({
where: { subjectType: 'USER', subjectId: user.id },
include: { pond: { select: { slug: true, name: true, type: true, ownerId: true } } },
orderBy: { createdAt: 'asc' },
});
const byPond = new Map<
string,
{ pondSlug: string; pondName: string; pondType: string; owner: boolean; grants: unknown[] }
>();
for (const grant of grants) {
const entry = byPond.get(grant.pondId) ?? {
pondSlug: grant.pond.slug,
pondName: grant.pond.name,
pondType: grant.pond.type,
owner: grant.pond.ownerId === user.id,
grants: [],
};
entry.grants.push({
role: grant.role,
scopeType: grant.scopeType,
scopeId: grant.scopeId,
effect: grant.effect,
createdAt: grant.createdAt.toISOString(),
});
byPond.set(grant.pondId, entry);
}
return [...byPond.values()];
}
}

View File

@ -59,6 +59,34 @@ export class ExportService {
const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } });
if (!pond) throw new NotFoundException();
const archive = archiver('zip', { zlib: { level: 9 } });
res.set('Content-Type', 'application/zip');
res.set('Content-Disposition', `attachment; filename="${pond.slug}.zip"`);
res.set('X-Content-Type-Options', 'nosniff');
archive.on('error', (error) => {
this.logger.error({ pondId, err: error.message }, 'pond export archive failed');
res.destroy(error);
});
archive.pipe(res);
await this.appendPondMarkdown(archive, user, pond);
await archive.finalize();
}
/**
* Append one pond's readable pages (as Markdown) and their media to an open
* archive, each entry under `prefix`. Shared by the pond ZIP above and the
* account data export (#68), which nests several ponds under `ponds/<slug>/`
* the read filter runs here, so neither caller can leak a page the
* requester may not read. Page Markdown is appended as small strings; media
* as read streams, keeping memory bounded regardless of pond size.
*/
async appendPondMarkdown(
archive: archiver.Archiver,
user: User,
pond: { id: string; slug: string },
prefix = '',
): Promise<void> {
const pondId = pond.id;
const pages = await this.prisma.page.findMany({
where: { pondId, deletedAt: null },
orderBy: { title: 'asc' },
@ -99,16 +127,6 @@ export class ExportService {
attachments.map((a) => [a.id, `${a.id}.${imageExtension(a.mimeType)}`]),
);
const archive = archiver('zip', { zlib: { level: 9 } });
res.set('Content-Type', 'application/zip');
res.set('Content-Disposition', `attachment; filename="${pond.slug}.zip"`);
res.set('X-Content-Type-Options', 'nosniff');
archive.on('error', (error) => {
this.logger.error({ pondId, err: error.message }, 'pond export archive failed');
res.destroy(error);
});
archive.pipe(res);
for (const page of readablePages) {
const markdown = markdownForZip(
page.contentCache?.markdown ?? '',
@ -116,7 +134,7 @@ export class ExportService {
mediaNameById,
);
// Page slugs are unique within a pond, so `<slug>.md` never collides.
archive.append(markdown, { name: `${page.slug}.md` });
archive.append(markdown, { name: `${prefix}${page.slug}.md` });
}
for (const attachment of attachments) {
const stream = this.storage.createReadStream(pond.id, attachment.id);
@ -128,13 +146,12 @@ export class ExportService {
'export: media read failed',
),
);
archive.append(stream, { name: `media/${mediaNameById.get(attachment.id)!}` });
archive.append(stream, { name: `${prefix}media/${mediaNameById.get(attachment.id)!}` });
}
this.logger.info(
{ pondId, pages: readablePages.length, media: attachments.length, userId: user.id },
'audit: pond exported as markdown zip',
);
await archive.finalize();
}
/**

View File

@ -1,10 +1,15 @@
import { Module } from '@nestjs/common';
import { Module, OnModuleInit } from '@nestjs/common';
import { FilesModule } from '../files/files.module';
import { PagesModule } from '../pages/pages.module';
import { SchedulerModule } from '../scheduler/scheduler.module';
import { SchedulerService } from '../scheduler/scheduler.service';
import { ConversionJobService } from './conversion-job.service';
import { ConversionWorker } from './conversion-worker.service';
import { DATA_EXPORT_PROCESSOR } from './data-export.constants';
import { DataExportController } from './data-export.controller';
import { DataExportService } from './data-export.service';
import { ExportController } from './export.controller';
import { ExportService } from './export.service';
import { GotenbergHttpRenderer, GotenbergRenderer } from './gotenberg.renderer';
@ -14,23 +19,29 @@ import { ImportService } from './import.service';
import { JobsController } from './jobs.controller';
import { PandocConverter, PandocServerConverter } from './pandoc.converter';
/** How often expired data-export payloads are purged (#68). Hourly is ample:
* the link's own expiry check already stops downloads the moment it lapses. */
const EXPORT_PURGE_CADENCE_SECONDS = 60 * 60;
/**
* Import/export orchestration (ADR 0009): the conversion job queue, its worker,
* the pandoc-server client (#62), and the document import pipeline (#63, which
* turns an uploaded `.docx`/`.odt` into a new page). Export (#65) will add its
* feature endpoints here too.
* the pandoc-server client (#62), the document import pipeline (#63), the
* feature exports (#65/#67), and the GDPR account data export (#68).
*/
@Module({
imports: [FilesModule, PagesModule],
controllers: [JobsController, ImportController, ExportController],
imports: [FilesModule, PagesModule, SchedulerModule],
controllers: [JobsController, ImportController, ExportController, DataExportController],
providers: [
ConversionJobService,
ConversionWorker,
ImportService,
ExportService,
DataExportService,
// The worker resolves the import pipeline through this token (never the
// class), so its file does not import the import service's (avoids a cycle).
{ provide: IMPORT_PROCESSOR, useExisting: ImportService },
// Same token indirection for the data-export builder (#68).
{ provide: DATA_EXPORT_PROCESSOR, useExisting: DataExportService },
// Bind the abstract converter to the HTTP implementation; tests override
// this provider with a fake so the queue mechanics need no live sidecar.
{ provide: PandocConverter, useClass: PandocServerConverter },
@ -39,4 +50,17 @@ import { PandocConverter, PandocServerConverter } from './pandoc.converter';
],
exports: [ConversionJobService, PandocConverter],
})
export class ImportExportModule {}
export class ImportExportModule implements OnModuleInit {
constructor(
private readonly scheduler: SchedulerService,
private readonly dataExport: DataExportService,
) {}
onModuleInit(): void {
this.scheduler.register({
name: 'data-export-purge',
cadenceSeconds: EXPORT_PURGE_CADENCE_SECONDS,
run: () => this.dataExport.purgeExpired().then(() => undefined),
});
}
}

View File

@ -171,6 +171,7 @@ export class ImportService implements ImportProcessor {
targetFormat: 'page',
errorCode: null,
resultPageId: page.id,
expiresAt: null,
createdAt: now,
updatedAt: now,
};

View File

@ -0,0 +1,46 @@
import type { ConversionJobView } from '@dorfteich/shared';
import { ApiError, apiGet } from '../lib/api';
const POLL_INTERVAL_MS = 1000;
/** Default poll budget a whole-account export (#68) can take longer than a
* single-page conversion, so callers may raise or lower it. */
const DEFAULT_MAX_POLLS = 300;
const delay = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
/** Poll a conversion job until it succeeds or fails, or the poll budget runs
* out (returns whatever the last status was). Shared by every export hook. */
export async function pollJob(
job: ConversionJobView,
maxPolls = DEFAULT_MAX_POLLS,
): Promise<ConversionJobView> {
let current = job;
for (
let poll = 0;
current.status !== 'succeeded' && current.status !== 'failed' && poll < maxPolls;
poll += 1
) {
await delay(POLL_INTERVAL_MS);
current = await apiGet<ConversionJobView>(`/jobs/${current.id}`);
}
return current;
}
/** Download a finished job's result and save it as `fileName`. The result
* endpoint is session-authenticated, so a plain `fetch` carries the cookie. */
export async function downloadJobResult(jobId: string, fileName: string): Promise<void> {
const response = await fetch(`/api/v1/jobs/${jobId}/result`);
if (!response.ok) {
throw new ApiError(response.status, { code: `http_${response.status}`, message: '' });
}
const blob = await response.blob();
const url = URL.createObjectURL(blob);
const anchor = document.createElement('a');
anchor.href = url;
anchor.download = fileName;
document.body.appendChild(anchor);
anchor.click();
anchor.remove();
URL.revokeObjectURL(url);
}

View File

@ -0,0 +1,54 @@
import type { ConversionJobView } from '@dorfteich/shared';
import { useCallback, useState } from 'react';
import { ApiError, apiPost } from '../lib/api';
import { downloadJobResult, pollJob } from './job-download';
export type DataExportStatus = 'idle' | 'busy' | 'ready' | 'error' | 'rateLimited';
export interface UseDataExport {
status: DataExportStatus;
/** When the ready download link expires (ISO), or null. */
expiresAt: string | null;
request: () => void;
download: () => void;
}
/**
* Self-service GDPR data export (issue #68): request the export, poll the job,
* and once it is ready download the ZIP. A recent request is rejected with
* 429, surfaced as a distinct `rateLimited` status.
*/
export function useDataExport(): UseDataExport {
const [status, setStatus] = useState<DataExportStatus>('idle');
const [expiresAt, setExpiresAt] = useState<string | null>(null);
const [jobId, setJobId] = useState<string | null>(null);
const request = useCallback((): void => {
void (async () => {
setStatus('busy');
setExpiresAt(null);
setJobId(null);
try {
const job = await pollJob(await apiPost<ConversionJobView>('/users/me/data-export'));
if (job.status === 'succeeded') {
setJobId(job.id);
setExpiresAt(job.expiresAt);
setStatus('ready');
} else {
setStatus('error');
}
} catch (error) {
setStatus(error instanceof ApiError && error.status === 429 ? 'rateLimited' : 'error');
}
})();
}, []);
const download = useCallback((): void => {
if (!jobId) return;
void downloadJobResult(jobId, 'dorfteich-export.zip').catch(() => setStatus('error'));
}, [jobId]);
return { status, expiresAt, request, download };
}

View File

@ -1,31 +1,16 @@
import type { ConversionJobView, ExportFormat } from '@dorfteich/shared';
import { useCallback, useState } from 'react';
import { ApiError, apiGet, apiPost } from '../lib/api';
import { apiPost } from '../lib/api';
import { downloadJobResult, pollJob } from './job-download';
export type ExportStatus = 'idle' | 'busy' | 'error';
const POLL_INTERVAL_MS = 1000;
/** A single-page conversion is quick cap the poll budget below the shared
* default. */
const MAX_POLLS = 180;
const delay = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
/** Fetch a finished job's result and save it under `fileName`. The result
* endpoint is session-authenticated, so a plain `fetch` carries the cookie. */
async function downloadResult(jobId: string, fileName: string): Promise<void> {
const response = await fetch(`/api/v1/jobs/${jobId}/result`);
if (!response.ok) throw new ApiError(response.status, { code: 'conversion_failed', message: '' });
const blob = await response.blob();
const url = URL.createObjectURL(blob);
const anchor = document.createElement('a');
anchor.href = url;
anchor.download = fileName;
document.body.appendChild(anchor);
anchor.click();
anchor.remove();
URL.revokeObjectURL(url);
}
export interface UseDocumentExport {
/** Per-format status so each button shows its own progress/error. */
status: Partial<Record<ExportFormat, ExportStatus>>;
@ -48,17 +33,12 @@ export function useDocumentExport(): UseDocumentExport {
void (async () => {
set(format, 'busy');
try {
let job = await apiPost<ConversionJobView>(`/pages/${pageId}/export`, { format });
for (
let poll = 0;
job.status !== 'succeeded' && job.status !== 'failed' && poll < MAX_POLLS;
poll += 1
) {
await delay(POLL_INTERVAL_MS);
job = await apiGet<ConversionJobView>(`/jobs/${job.id}`);
}
const job = await pollJob(
await apiPost<ConversionJobView>(`/pages/${pageId}/export`, { format }),
MAX_POLLS,
);
if (job.status === 'succeeded') {
await downloadResult(job.id, `${slug}.${format}`);
await downloadJobResult(job.id, `${slug}.${format}`);
set(format, 'idle');
} else {
set(format, 'error');

View File

@ -7,6 +7,7 @@ import { useTranslation } from 'react-i18next';
import { useAuth } from '../auth/auth-context';
import { Field, FormError, FormSuccess, applyFieldErrors } from '../components/forms';
import { useDataExport } from '../export/use-data-export';
import { apiDelete, apiGet, apiPatch, apiPost } from '../lib/api';
interface SessionView {
@ -25,10 +26,57 @@ export function SettingsPage(): React.JSX.Element {
<ProfileSection />
<PasswordSection />
<SessionsSection />
<DataExportSection />
</>
);
}
function DataExportSection(): React.JSX.Element {
const { t, i18n } = useTranslation();
const { status, expiresAt, request, download } = useDataExport();
const formatTime = (iso: string): string =>
new Intl.DateTimeFormat(i18n.language, { dateStyle: 'medium', timeStyle: 'short' }).format(
new Date(iso),
);
return (
<section className="settings-section">
<h2>{t('settings:dataExport.title')}</h2>
<p className="sidebar__hint">{t('settings:dataExport.description')}</p>
{status === 'error' && (
<p className="form-banner form-banner--error" role="alert">
{t('settings:dataExport.failed')}
</p>
)}
{status === 'rateLimited' && (
<p className="form-banner form-banner--error" role="alert">
{t('settings:dataExport.rateLimited')}
</p>
)}
{status === 'ready' ? (
<>
<FormSuccess message={t('settings:dataExport.ready')} />
<button type="button" className="button" onClick={download}>
{t('settings:dataExport.download')}
</button>
{expiresAt && (
<p className="sidebar__hint">
{t('settings:dataExport.expiresHint', { when: formatTime(expiresAt) })}
</p>
)}
</>
) : (
<button type="button" className="button" onClick={request} disabled={status === 'busy'}>
{status === 'busy'
? t('settings:dataExport.preparing')
: t('settings:dataExport.request')}
</button>
)}
</section>
);
}
function ProfileSection(): React.JSX.Element {
const { t, i18n } = useTranslation();
const { user, refresh } = useAuth();

View File

@ -25,6 +25,17 @@
"revokeAll": "Alle anderen Sitzungen abmelden",
"empty": "Keine weiteren aktiven Sitzungen."
},
"dataExport": {
"title": "Meine Daten exportieren",
"description": "Lade ein ZIP mit deinem Profil, einer Liste deiner Mitgliedschaften und dem Markdown-Export deines persönlichen Teichs sowie der Teiche, die dir gehören.",
"request": "Export vorbereiten",
"preparing": "Dein Export wird vorbereitet…",
"download": "Export herunterladen",
"ready": "Dein Export ist bereit.",
"expiresHint": "Der Download-Link läuft am {{when}} ab.",
"failed": "Der Export konnte nicht erstellt werden. Bitte versuche es erneut.",
"rateLimited": "Du hast kürzlich einen Export angefordert. Bitte versuche es später erneut."
},
"admin": {
"title": "Administration",
"instanceName": "Name der Instanz",

View File

@ -25,6 +25,17 @@
"revokeAll": "Sign out all other sessions",
"empty": "No other active sessions."
},
"dataExport": {
"title": "Export my data",
"description": "Download a ZIP with your profile, a list of your memberships, and the Markdown export of your personal pond and the ponds you own.",
"request": "Prepare export",
"preparing": "Preparing your export…",
"download": "Download export",
"ready": "Your export is ready.",
"expiresHint": "The download link expires on {{when}}.",
"failed": "The export could not be prepared. Please try again.",
"rateLimited": "You requested an export recently. Please try again later."
},
"admin": {
"title": "Administration",
"instanceName": "Instance name",

View File

@ -21,6 +21,10 @@ export interface ConversionJobView {
* the client can navigate to it. `null` for a pending/failed import and for
* plain bytebyte conversions (export). */
resultPageId: string | null;
/** For a data-export job (#68), when its download link stops working the
* result is purged after this. `null` for every other job kind, which never
* expires. */
expiresAt: string | null;
createdAt: string;
updatedAt: string;
}