Add self-service GDPR data export (#68)
All checks were successful
CD / Build and push images (push) Successful in 10m39s
CI / Lint, typecheck, test (push) Successful in 3m12s
CI / Auth e2e pack (push) Successful in 4m9s
CI / Build container images (push) Has been skipped
CD / Deploy to Test (push) Successful in 9s
CD / Smoke tests against Test (push) Successful in 1m18s
CD / Promote to Int (push) Successful in 11s
All checks were successful
CD / Build and push images (push) Successful in 10m39s
CI / Lint, typecheck, test (push) Successful in 3m12s
CI / Auth e2e pack (push) Successful in 4m9s
CI / Build container images (push) Has been skipped
CD / Deploy to Test (push) Successful in 9s
CD / Smoke tests against Test (push) Successful in 1m18s
CD / Promote to Int (push) Successful in 11s
A signed-in account can export all of its own data — profile, a list of its memberships/grants, and the Markdown of its personal pond plus the shared ponds it owns — as one ZIP. Foreign content never appears: only owned ponds are bundled and the per-page read filter (reused from #65) runs for each. - Reuse the conversion-job queue as the async carrier: a `data_export` job whose worker branch resolves DataExportService via a token (no DI cycle), builds the ZIP, and stores it with an `expiresAt`. The download link 404s past expiry and an hourly scheduled purge drops the bytes (data minimization, security.md §Privacy). - Extract ExportService.appendPondMarkdown so the pond ZIP (#65) and the data export share one read-filtered pond archiver. - Rate-limit requests per account (RateLimitService); POST /users/me/data-export enqueues, GET /jobs/:id(/result) poll/download. - Settings UI "Export my data" (de+en); web share pollJob/downloadJobResult between the document and data export hooks. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1
This commit is contained in:
parent
8a68ef68e7
commit
462eca9699
@ -0,0 +1,2 @@
|
|||||||
|
-- AlterTable
|
||||||
|
ALTER TABLE "conversion_jobs" ADD COLUMN "expires_at" TIMESTAMP(3);
|
||||||
@ -550,6 +550,10 @@ model ConversionJob {
|
|||||||
resultMimeType String? @map("result_mime_type")
|
resultMimeType String? @map("result_mime_type")
|
||||||
errorCode String? @map("error_code")
|
errorCode String? @map("error_code")
|
||||||
lockedAt DateTime? @map("locked_at")
|
lockedAt DateTime? @map("locked_at")
|
||||||
|
/// For a data-export job (#68): when its stored result stops being
|
||||||
|
/// downloadable and is purged (GDPR data minimization). Null for every
|
||||||
|
/// other job kind, whose result never expires.
|
||||||
|
expiresAt DateTime? @map("expires_at")
|
||||||
createdAt DateTime @default(now()) @map("created_at")
|
createdAt DateTime @default(now()) @map("created_at")
|
||||||
updatedAt DateTime @updatedAt @map("updated_at")
|
updatedAt DateTime @updatedAt @map("updated_at")
|
||||||
|
|
||||||
|
|||||||
@ -89,6 +89,9 @@ export class ConversionJobService {
|
|||||||
async resultForOwner(id: string, userId: string): Promise<ConversionResultPayload> {
|
async resultForOwner(id: string, userId: string): Promise<ConversionResultPayload> {
|
||||||
const job = await this.ownedJob(id, userId);
|
const job = await this.ownedJob(id, userId);
|
||||||
if (job.status !== 'SUCCEEDED' || !job.result) throw new NotFoundException();
|
if (job.status !== 'SUCCEEDED' || !job.result) throw new NotFoundException();
|
||||||
|
// A data-export link expires (#68): past its window the result is treated as
|
||||||
|
// gone (a scheduled purge deletes the bytes), so the download 404s.
|
||||||
|
if (job.expiresAt && job.expiresAt.getTime() < Date.now()) throw new NotFoundException();
|
||||||
return {
|
return {
|
||||||
bytes: Buffer.from(job.result),
|
bytes: Buffer.from(job.result),
|
||||||
mimeType: job.resultMimeType ?? 'application/octet-stream',
|
mimeType: job.resultMimeType ?? 'application/octet-stream',
|
||||||
@ -111,6 +114,8 @@ export class ConversionJobService {
|
|||||||
errorCode: job.errorCode,
|
errorCode: job.errorCode,
|
||||||
// Set once an import job succeeds (#63) so the client can open the page.
|
// Set once an import job succeeds (#63) so the client can open the page.
|
||||||
resultPageId: job.resultPageId,
|
resultPageId: job.resultPageId,
|
||||||
|
// Set for a data-export job (#68): when its download link stops working.
|
||||||
|
expiresAt: job.expiresAt?.toISOString() ?? null,
|
||||||
createdAt: job.createdAt.toISOString(),
|
createdAt: job.createdAt.toISOString(),
|
||||||
updatedAt: job.updatedAt.toISOString(),
|
updatedAt: job.updatedAt.toISOString(),
|
||||||
};
|
};
|
||||||
|
|||||||
@ -6,6 +6,12 @@ import { PinoLogger } from 'nestjs-pino';
|
|||||||
import { AppConfig } from '../config/app-config.service';
|
import { AppConfig } from '../config/app-config.service';
|
||||||
import { PrismaService } from '../prisma/prisma.service';
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
|
|
||||||
|
import {
|
||||||
|
DATA_EXPORT_KIND,
|
||||||
|
DATA_EXPORT_PROCESSOR,
|
||||||
|
DATA_EXPORT_TTL_MS,
|
||||||
|
DataExportProcessor,
|
||||||
|
} from './data-export.constants';
|
||||||
import { GotenbergRenderer, RenderError } from './gotenberg.renderer';
|
import { GotenbergRenderer, RenderError } from './gotenberg.renderer';
|
||||||
import { IMPORT_PROCESSOR, ImportProcessor, isImportKind } from './import.constants';
|
import { IMPORT_PROCESSOR, ImportProcessor, isImportKind } from './import.constants';
|
||||||
import { ConversionError, ConversionResult, PandocConverter } from './pandoc.converter';
|
import { ConversionError, ConversionResult, PandocConverter } from './pandoc.converter';
|
||||||
@ -127,15 +133,26 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy {
|
|||||||
await this.moduleRef.get<ImportProcessor>(IMPORT_PROCESSOR, { strict: false }).run(job);
|
await this.moduleRef.get<ImportProcessor>(IMPORT_PROCESSOR, { strict: false }).run(job);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// PDF export renders HTML through Gotenberg (#67); everything else is a
|
// A data export gathers the account's own data into a ZIP (#68) whose
|
||||||
// pandoc byte→byte conversion (#62/#65).
|
// download link expires; a PDF export renders HTML through Gotenberg
|
||||||
const output =
|
// (#67); everything else is a pandoc byte→byte conversion (#62/#65).
|
||||||
job.targetFormat === 'pdf'
|
let output: { bytes: Buffer; mimeType: string };
|
||||||
? {
|
let expiresAt: Date | null = null;
|
||||||
|
if (job.kind === DATA_EXPORT_KIND) {
|
||||||
|
// Resolved lazily via a token so this file never imports the export
|
||||||
|
// service's (avoids a construction cycle), mirroring the import path.
|
||||||
|
output = await this.moduleRef
|
||||||
|
.get<DataExportProcessor>(DATA_EXPORT_PROCESSOR, { strict: false })
|
||||||
|
.build(job);
|
||||||
|
expiresAt = new Date(Date.now() + DATA_EXPORT_TTL_MS);
|
||||||
|
} else if (job.targetFormat === 'pdf') {
|
||||||
|
output = {
|
||||||
bytes: await this.renderer.renderHtmlToPdf(Buffer.from(job.input).toString('utf8')),
|
bytes: await this.renderer.renderHtmlToPdf(Buffer.from(job.input).toString('utf8')),
|
||||||
mimeType: 'application/pdf',
|
mimeType: 'application/pdf',
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
output = await this.convert(job);
|
||||||
}
|
}
|
||||||
: await this.convert(job);
|
|
||||||
await this.prisma.conversionJob.update({
|
await this.prisma.conversionJob.update({
|
||||||
where: { id: job.id },
|
where: { id: job.id },
|
||||||
data: {
|
data: {
|
||||||
@ -144,6 +161,7 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy {
|
|||||||
result: new Uint8Array(output.bytes),
|
result: new Uint8Array(output.bytes),
|
||||||
resultMimeType: output.mimeType,
|
resultMimeType: output.mimeType,
|
||||||
errorCode: null,
|
errorCode: null,
|
||||||
|
expiresAt,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
|
|||||||
23
apps/api/src/import-export/data-export.constants.ts
Normal file
23
apps/api/src/import-export/data-export.constants.ts
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
import type { ConversionJob } from '@prisma/client';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wiring for the GDPR data export (issue #68). The conversion worker builds a
|
||||||
|
* `data_export` job through this token — never by importing the service — so
|
||||||
|
* the worker's file stays free of a construction cycle (mirrors the import
|
||||||
|
* pipeline's {@link ./import.constants.ts}).
|
||||||
|
*/
|
||||||
|
export const DATA_EXPORT_PROCESSOR = Symbol('DATA_EXPORT_PROCESSOR');
|
||||||
|
|
||||||
|
/** The {@link ConversionJob.kind} of a self-service account data export. */
|
||||||
|
export const DATA_EXPORT_KIND = 'data_export';
|
||||||
|
|
||||||
|
/** How long a generated export stays downloadable before its bytes are purged
|
||||||
|
* (security.md §Privacy: data minimization — the archive is a transient copy,
|
||||||
|
* not stored indefinitely). */
|
||||||
|
export const DATA_EXPORT_TTL_MS = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
/** Builds the ZIP for a `data_export` job. The worker resolves this behind the
|
||||||
|
* token above and stores the returned bytes as the job result. */
|
||||||
|
export interface DataExportProcessor {
|
||||||
|
build(job: ConversionJob): Promise<{ bytes: Buffer; mimeType: string }>;
|
||||||
|
}
|
||||||
24
apps/api/src/import-export/data-export.controller.ts
Normal file
24
apps/api/src/import-export/data-export.controller.ts
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
import { Controller, Post, Req } from '@nestjs/common';
|
||||||
|
import { ConversionJobView } from '@dorfteich/shared';
|
||||||
|
|
||||||
|
import { AuthedRequest } from '../auth/auth.guard';
|
||||||
|
import { AuthenticatedOnly } from '../permissions/permission.decorators';
|
||||||
|
|
||||||
|
import { DataExportService } from './data-export.service';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Self-service GDPR data export (issue #68, security.md §Privacy). A signed-in
|
||||||
|
* account requests its own export here; the job is owner-scoped, so polling and
|
||||||
|
* download stay on the generic `GET /jobs/:id(/result)` routes (JobsController).
|
||||||
|
*/
|
||||||
|
@AuthenticatedOnly()
|
||||||
|
@Controller('users/me')
|
||||||
|
export class DataExportController {
|
||||||
|
constructor(private readonly dataExport: DataExportService) {}
|
||||||
|
|
||||||
|
/** Enqueue an export of the caller's own data; rate-limited per account. */
|
||||||
|
@Post('data-export')
|
||||||
|
request(@Req() request: AuthedRequest): Promise<ConversionJobView> {
|
||||||
|
return this.dataExport.request(request.user!);
|
||||||
|
}
|
||||||
|
}
|
||||||
299
apps/api/src/import-export/data-export.service.db.test.ts
Normal file
299
apps/api/src/import-export/data-export.service.db.test.ts
Normal file
@ -0,0 +1,299 @@
|
|||||||
|
import { INestApplication } from '@nestjs/common';
|
||||||
|
import { PrismaClient } from '@prisma/client';
|
||||||
|
import { unzipSync } from 'fflate';
|
||||||
|
import request from 'supertest';
|
||||||
|
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { AuthTokensService } from '../auth/auth-tokens.service';
|
||||||
|
import { FilesService } from '../files/files.service';
|
||||||
|
import { createTestApp, sessionCookieOf } from '../testing/test-app';
|
||||||
|
import { createTestPrisma, grantOwnerAdmin, hasTestDb, uniqueSuffix } from '../testing/test-db';
|
||||||
|
import { UsersService } from '../users/users.service';
|
||||||
|
|
||||||
|
import { ConversionWorker } from './conversion-worker.service';
|
||||||
|
import { DataExportService } from './data-export.service';
|
||||||
|
|
||||||
|
/** GDPR data export (issue #68): a signed-in account assembles its own profile,
|
||||||
|
* memberships, and the Markdown of every pond it owns into one ZIP — with a
|
||||||
|
* download link that expires — and never another user's content. */
|
||||||
|
|
||||||
|
const PNG_BASE64 =
|
||||||
|
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';
|
||||||
|
|
||||||
|
function zipEntries(buffer: Buffer): Record<string, Uint8Array> {
|
||||||
|
return unzipSync(new Uint8Array(buffer));
|
||||||
|
}
|
||||||
|
|
||||||
|
function textOf(entries: Record<string, Uint8Array>, name: string): string {
|
||||||
|
return Buffer.from(entries[name]!).toString('utf8');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function downloadZip(server: unknown, path: string, cookie: string): Promise<Buffer> {
|
||||||
|
const res = await request(server as never)
|
||||||
|
.get(path)
|
||||||
|
.set('Cookie', cookie)
|
||||||
|
.buffer(true)
|
||||||
|
.parse((r, cb) => {
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
r.on('data', (c: Buffer) => chunks.push(c));
|
||||||
|
r.on('end', () => cb(null, Buffer.concat(chunks)));
|
||||||
|
})
|
||||||
|
.expect(200);
|
||||||
|
return res.body as Buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe.skipIf(!hasTestDb)('data export (e2e, issue #68)', () => {
|
||||||
|
let app: INestApplication;
|
||||||
|
let prisma: PrismaClient;
|
||||||
|
let worker: ConversionWorker;
|
||||||
|
let files: FilesService;
|
||||||
|
const suffix = uniqueSuffix();
|
||||||
|
const password = 'exportiere alle meine daten 1';
|
||||||
|
|
||||||
|
let ownerId: string;
|
||||||
|
let ownerCookie: string;
|
||||||
|
let personalSlug: string;
|
||||||
|
let ownedSharedSlug: string;
|
||||||
|
let foreignSlug: string;
|
||||||
|
|
||||||
|
const api = () => request(app.getHttpServer());
|
||||||
|
|
||||||
|
async function login(username: string): Promise<string> {
|
||||||
|
const res = await api()
|
||||||
|
.post('/api/v1/auth/login')
|
||||||
|
.send({ usernameOrEmail: username, password })
|
||||||
|
.expect(200);
|
||||||
|
return sessionCookieOf(res);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createUser(username: string): Promise<string> {
|
||||||
|
const user = await app.get(UsersService).createUser({
|
||||||
|
username,
|
||||||
|
email: `${username}@example.org`,
|
||||||
|
displayName: username,
|
||||||
|
password,
|
||||||
|
locale: 'en',
|
||||||
|
});
|
||||||
|
// Verify via the endpoint (not a direct flag flip) so the personal pond and
|
||||||
|
// its owner-admin grant are created exactly as in production (#52).
|
||||||
|
const token = await app.get(AuthTokensService).issue(user.id, 'EMAIL_VERIFICATION', 600);
|
||||||
|
await api().post('/api/v1/auth/verify-email').send({ token }).expect(204);
|
||||||
|
return user.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function seedPage(pondId: string, title: string, markdown: string): Promise<void> {
|
||||||
|
await prisma.page.create({
|
||||||
|
data: {
|
||||||
|
pondId,
|
||||||
|
title,
|
||||||
|
slug: title.toLowerCase().replace(/[^a-z0-9]+/g, '-'),
|
||||||
|
ydocState: new Uint8Array(),
|
||||||
|
sortKey: title,
|
||||||
|
createdBy: ownerId,
|
||||||
|
contentCache: { create: { plainText: markdown, markdown, html: '', outline: [] } },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
prisma = createTestPrisma();
|
||||||
|
await prisma.rateLimit.deleteMany({});
|
||||||
|
app = await createTestApp();
|
||||||
|
worker = app.get(ConversionWorker);
|
||||||
|
files = app.get(FilesService);
|
||||||
|
|
||||||
|
ownerId = await createUser(`odette-owner-${suffix}`);
|
||||||
|
ownerCookie = await login(`odette-owner-${suffix}`);
|
||||||
|
|
||||||
|
const personal = await prisma.pond.findFirstOrThrow({
|
||||||
|
where: { ownerId, type: 'PERSONAL' },
|
||||||
|
});
|
||||||
|
personalSlug = personal.slug;
|
||||||
|
|
||||||
|
// A shared pond the owner created — its content belongs in the export.
|
||||||
|
const shared = await prisma.pond.create({
|
||||||
|
data: {
|
||||||
|
slug: `owned-shared-${suffix}`,
|
||||||
|
name: 'Owned Shared',
|
||||||
|
type: 'SHARED',
|
||||||
|
ownerId,
|
||||||
|
usage: { create: {} },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
ownedSharedSlug = shared.slug;
|
||||||
|
await grantOwnerAdmin(prisma, shared.id, ownerId);
|
||||||
|
|
||||||
|
// A stored image referenced from a personal-pond page (media inclusion).
|
||||||
|
const image = await files.upload({ id: ownerId } as never, personal.id, {
|
||||||
|
buffer: Buffer.from(PNG_BASE64, 'base64'),
|
||||||
|
size: 70,
|
||||||
|
originalname: 'dot.png',
|
||||||
|
});
|
||||||
|
await seedPage(personal.id, 'Diary', `# Diary\n\nMine. `);
|
||||||
|
await seedPage(shared.id, 'Shared Note', '# Shared Note\n\nAlso mine.');
|
||||||
|
|
||||||
|
// A foreign user's pond the owner is only a *reader* of: it must show up in
|
||||||
|
// memberships, but its content must never appear in the export.
|
||||||
|
const foreignId = await createUser(`fred-foreign-${suffix}`);
|
||||||
|
const foreign = await prisma.pond.create({
|
||||||
|
data: {
|
||||||
|
slug: `foreign-secret-${suffix}`,
|
||||||
|
name: 'Foreign Secret',
|
||||||
|
type: 'SHARED',
|
||||||
|
ownerId: foreignId,
|
||||||
|
usage: { create: {} },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
foreignSlug = foreign.slug;
|
||||||
|
await prisma.page.create({
|
||||||
|
data: {
|
||||||
|
pondId: foreign.id,
|
||||||
|
title: 'Foreign Page',
|
||||||
|
slug: 'foreign-page',
|
||||||
|
ydocState: new Uint8Array(),
|
||||||
|
sortKey: 'Foreign Page',
|
||||||
|
createdBy: foreignId,
|
||||||
|
contentCache: {
|
||||||
|
create: { plainText: 'TOP SECRET', markdown: 'TOP SECRET', html: '', outline: [] },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await prisma.roleGrant.create({
|
||||||
|
data: {
|
||||||
|
pondId: foreign.id,
|
||||||
|
subjectType: 'USER',
|
||||||
|
subjectId: ownerId,
|
||||||
|
role: 'READER',
|
||||||
|
scopeType: 'POND',
|
||||||
|
effect: 'ALLOW',
|
||||||
|
createdBy: foreignId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await prisma.conversionJob.deleteMany({ where: { owner: { username: { contains: suffix } } } });
|
||||||
|
const where = { pond: { owner: { username: { contains: suffix } } } };
|
||||||
|
await prisma.attachment.deleteMany({ where });
|
||||||
|
await prisma.roleGrant.deleteMany({
|
||||||
|
where: { pond: { owner: { username: { contains: suffix } } } },
|
||||||
|
});
|
||||||
|
await prisma.page.deleteMany({ where });
|
||||||
|
await prisma.pond.deleteMany({ where: { owner: { username: { contains: suffix } } } });
|
||||||
|
await prisma.rateLimit.deleteMany({});
|
||||||
|
await prisma.user.deleteMany({ where: { username: { contains: suffix } } });
|
||||||
|
await prisma.$disconnect();
|
||||||
|
await app.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Each test starts from a clean rate-limit table so the per-account export
|
||||||
|
// limit (exercised deliberately in one test) never trips the others.
|
||||||
|
beforeEach(async () => {
|
||||||
|
await prisma.rateLimit.deleteMany({});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('exports the caller profile, memberships, and owned pond content only', async () => {
|
||||||
|
const enqueued = await api()
|
||||||
|
.post('/api/v1/users/me/data-export')
|
||||||
|
.set('Cookie', ownerCookie)
|
||||||
|
.expect(201);
|
||||||
|
expect(enqueued.body.kind).toBe('data_export');
|
||||||
|
expect(enqueued.body.status).toBe('pending');
|
||||||
|
|
||||||
|
await worker.drain();
|
||||||
|
|
||||||
|
const done = await api()
|
||||||
|
.get(`/api/v1/jobs/${enqueued.body.id}`)
|
||||||
|
.set('Cookie', ownerCookie)
|
||||||
|
.expect(200);
|
||||||
|
expect(done.body.status).toBe('succeeded');
|
||||||
|
// The download link carries an expiry (a future timestamp).
|
||||||
|
expect(done.body.expiresAt).toBeTruthy();
|
||||||
|
expect(new Date(done.body.expiresAt).getTime()).toBeGreaterThan(Date.now());
|
||||||
|
|
||||||
|
const zip = await downloadZip(
|
||||||
|
app.getHttpServer(),
|
||||||
|
`/api/v1/jobs/${enqueued.body.id}/result`,
|
||||||
|
ownerCookie,
|
||||||
|
);
|
||||||
|
const entries = zipEntries(zip);
|
||||||
|
const names = Object.keys(entries);
|
||||||
|
|
||||||
|
// Profile: the caller's own account fields.
|
||||||
|
const profile = JSON.parse(textOf(entries, 'profile.json'));
|
||||||
|
expect(profile.username).toBe(`odette-owner-${suffix}`);
|
||||||
|
expect(profile.email).toBe(`odette-owner-${suffix}@example.org`);
|
||||||
|
|
||||||
|
// Personal + owned-shared pond content is present, media included.
|
||||||
|
expect(names).toContain(`ponds/${personalSlug}/diary.md`);
|
||||||
|
expect(names.some((n) => n.startsWith(`ponds/${personalSlug}/media/`))).toBe(true);
|
||||||
|
expect(names).toContain(`ponds/${ownedSharedSlug}/shared-note.md`);
|
||||||
|
|
||||||
|
// Memberships list the foreign pond the owner reads…
|
||||||
|
const memberships = JSON.parse(textOf(entries, 'memberships.json')) as {
|
||||||
|
pondSlug: string;
|
||||||
|
}[];
|
||||||
|
expect(memberships.map((m) => m.pondSlug)).toContain(foreignSlug);
|
||||||
|
|
||||||
|
// …but none of the foreign pond's content appears anywhere in the archive.
|
||||||
|
expect(names.some((n) => n.startsWith(`ponds/${foreignSlug}/`))).toBe(false);
|
||||||
|
for (const name of names) {
|
||||||
|
if (name.endsWith('.md')) expect(textOf(entries, name)).not.toContain('TOP SECRET');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('404s the download once the link has expired', async () => {
|
||||||
|
const enqueued = await api()
|
||||||
|
.post('/api/v1/users/me/data-export')
|
||||||
|
.set('Cookie', ownerCookie)
|
||||||
|
.expect(201);
|
||||||
|
await worker.drain();
|
||||||
|
// Backdate the expiry to simulate an elapsed link.
|
||||||
|
await prisma.conversionJob.update({
|
||||||
|
where: { id: enqueued.body.id },
|
||||||
|
data: { expiresAt: new Date(Date.now() - 1000) },
|
||||||
|
});
|
||||||
|
|
||||||
|
await api()
|
||||||
|
.get(`/api/v1/jobs/${enqueued.body.id}/result`)
|
||||||
|
.set('Cookie', ownerCookie)
|
||||||
|
.expect(404);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('purges the stored bytes of an expired export', async () => {
|
||||||
|
const enqueued = await api()
|
||||||
|
.post('/api/v1/users/me/data-export')
|
||||||
|
.set('Cookie', ownerCookie)
|
||||||
|
.expect(201);
|
||||||
|
await worker.drain();
|
||||||
|
await prisma.conversionJob.update({
|
||||||
|
where: { id: enqueued.body.id },
|
||||||
|
data: { expiresAt: new Date(Date.now() - 1000) },
|
||||||
|
});
|
||||||
|
|
||||||
|
const purged = await app.get(DataExportService).purgeExpired();
|
||||||
|
expect(purged).toBeGreaterThanOrEqual(1);
|
||||||
|
const row = await prisma.conversionJob.findUniqueOrThrow({ where: { id: enqueued.body.id } });
|
||||||
|
expect(row.result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rate-limits repeated requests from the same account', async () => {
|
||||||
|
await createUser(`irene-idle-${suffix}`);
|
||||||
|
const cookie = await login(`irene-idle-${suffix}`);
|
||||||
|
// The limit is a few requests per window; the run past it is rejected.
|
||||||
|
for (let i = 0; i < 3; i += 1) {
|
||||||
|
await api().post('/api/v1/users/me/data-export').set('Cookie', cookie).expect(201);
|
||||||
|
}
|
||||||
|
await api().post('/api/v1/users/me/data-export').set('Cookie', cookie).expect(429);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps a data-export job owner-scoped (a foreign user cannot poll it)', async () => {
|
||||||
|
const enqueued = await api()
|
||||||
|
.post('/api/v1/users/me/data-export')
|
||||||
|
.set('Cookie', ownerCookie)
|
||||||
|
.expect(201);
|
||||||
|
await createUser(`sam-stranger-${suffix}`);
|
||||||
|
const strangerCookie = await login(`sam-stranger-${suffix}`);
|
||||||
|
await api().get(`/api/v1/jobs/${enqueued.body.id}`).set('Cookie', strangerCookie).expect(404);
|
||||||
|
});
|
||||||
|
});
|
||||||
174
apps/api/src/import-export/data-export.service.ts
Normal file
174
apps/api/src/import-export/data-export.service.ts
Normal file
@ -0,0 +1,174 @@
|
|||||||
|
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
|
||||||
|
import { ConversionJobView } from '@dorfteich/shared';
|
||||||
|
import { ConversionJob, User } from '@prisma/client';
|
||||||
|
import archiver from 'archiver';
|
||||||
|
import { PinoLogger } from 'nestjs-pino';
|
||||||
|
|
||||||
|
import { RateLimitService } from '../rate-limit/rate-limit.service';
|
||||||
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
|
|
||||||
|
import { ConversionJobService } from './conversion-job.service';
|
||||||
|
import { DATA_EXPORT_KIND, DataExportProcessor } from './data-export.constants';
|
||||||
|
import { ExportService } from './export.service';
|
||||||
|
|
||||||
|
/** A data export may be requested at most this many times per window per
|
||||||
|
* account (security.md §Privacy: access requests are self-service but must not
|
||||||
|
* be a cheap way to hammer the server). */
|
||||||
|
const MAX_REQUESTS_PER_WINDOW = 3;
|
||||||
|
const WINDOW_SECONDS = 60 * 60;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GDPR data export (issue #68): a signed-in user assembles all of their own
|
||||||
|
* data — profile, a list of their memberships/grants, and the Markdown export
|
||||||
|
* of their personal pond and the shared ponds they own — into a single ZIP.
|
||||||
|
* It is produced as a conversion job (so a large account exports out of band)
|
||||||
|
* with a download link that expires; foreign content never appears, because
|
||||||
|
* only the requester's own ponds are included and the per-page read filter
|
||||||
|
* (reused from #65) runs for every one.
|
||||||
|
*/
|
||||||
|
@Injectable()
|
||||||
|
export class DataExportService implements DataExportProcessor {
|
||||||
|
constructor(
|
||||||
|
private readonly prisma: PrismaService,
|
||||||
|
private readonly exports: ExportService,
|
||||||
|
private readonly jobs: ConversionJobService,
|
||||||
|
private readonly rateLimits: RateLimitService,
|
||||||
|
private readonly logger: PinoLogger,
|
||||||
|
) {
|
||||||
|
this.logger.setContext(DataExportService.name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Enqueue a data-export job for `user`, rate-limited per account. The client
|
||||||
|
* polls `GET /jobs/:id` and downloads `GET /jobs/:id/result` once it
|
||||||
|
* succeeds; the link stops working after the job's `expiresAt`. */
|
||||||
|
async request(user: User): Promise<ConversionJobView> {
|
||||||
|
const limit = await this.rateLimits.hit(
|
||||||
|
'data_export',
|
||||||
|
`user:${user.id}`,
|
||||||
|
MAX_REQUESTS_PER_WINDOW,
|
||||||
|
WINDOW_SECONDS,
|
||||||
|
);
|
||||||
|
if (!limit.allowed) {
|
||||||
|
throw new HttpException(
|
||||||
|
{ code: 'rate_limited', details: { retryAfterSeconds: limit.retryAfterSeconds } },
|
||||||
|
HttpStatus.TOO_MANY_REQUESTS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const job = await this.jobs.enqueue({
|
||||||
|
ownerId: user.id,
|
||||||
|
kind: DATA_EXPORT_KIND,
|
||||||
|
from: 'account',
|
||||||
|
to: 'zip',
|
||||||
|
// A data export gathers everything from the database at build time, so the
|
||||||
|
// job needs no input payload.
|
||||||
|
input: Buffer.alloc(0),
|
||||||
|
});
|
||||||
|
this.logger.info({ jobId: job.id, userId: user.id }, 'audit: data export requested');
|
||||||
|
return this.jobs.viewOf(job);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Worker entry point (via {@link DATA_EXPORT_PROCESSOR}): build the ZIP for a
|
||||||
|
* requested export. */
|
||||||
|
async build(job: ConversionJob): Promise<{ bytes: Buffer; mimeType: string }> {
|
||||||
|
const user = await this.prisma.user.findUniqueOrThrow({ where: { id: job.ownerId } });
|
||||||
|
|
||||||
|
const archive = archiver('zip', { zlib: { level: 9 } });
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
archive.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||||
|
const finished = new Promise<void>((resolve, reject) => {
|
||||||
|
archive.on('end', () => resolve());
|
||||||
|
archive.on('error', reject);
|
||||||
|
});
|
||||||
|
|
||||||
|
archive.append(JSON.stringify(this.profileOf(user), null, 2), { name: 'profile.json' });
|
||||||
|
archive.append(JSON.stringify(await this.membershipsOf(user), null, 2), {
|
||||||
|
name: 'memberships.json',
|
||||||
|
});
|
||||||
|
|
||||||
|
// Only ponds this user owns — their personal pond and any shared ponds they
|
||||||
|
// created. Foreign ponds they are merely a member of are represented by the
|
||||||
|
// memberships list, not their content.
|
||||||
|
const ponds = await this.prisma.pond.findMany({
|
||||||
|
where: { ownerId: user.id, deletedAt: null },
|
||||||
|
select: { id: true, slug: true },
|
||||||
|
orderBy: { slug: 'asc' },
|
||||||
|
});
|
||||||
|
for (const pond of ponds) {
|
||||||
|
await this.exports.appendPondMarkdown(archive, user, pond, `ponds/${pond.slug}/`);
|
||||||
|
}
|
||||||
|
|
||||||
|
await archive.finalize();
|
||||||
|
await finished;
|
||||||
|
this.logger.info(
|
||||||
|
{ jobId: job.id, userId: user.id, ponds: ponds.length },
|
||||||
|
'audit: data export built',
|
||||||
|
);
|
||||||
|
return { bytes: Buffer.concat(chunks), mimeType: 'application/zip' };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Drop the stored bytes of every export whose link has expired (data
|
||||||
|
* minimization: the archive is transient). The row remains for its audit
|
||||||
|
* trail; only the payload is deleted, so a later download 404s. Registered as
|
||||||
|
* a scheduled maintenance job. */
|
||||||
|
async purgeExpired(): Promise<number> {
|
||||||
|
const result = await this.prisma.conversionJob.updateMany({
|
||||||
|
where: { kind: DATA_EXPORT_KIND, expiresAt: { lt: new Date() }, result: { not: null } },
|
||||||
|
data: { result: null, resultMimeType: null },
|
||||||
|
});
|
||||||
|
if (result.count > 0) {
|
||||||
|
this.logger.info({ purged: result.count }, 'audit: expired data exports purged');
|
||||||
|
}
|
||||||
|
return result.count;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The requester's own account fields — the data minimization set the app
|
||||||
|
* collects (security.md §Privacy): identity, locale, and lifecycle stamps. */
|
||||||
|
private profileOf(user: User): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
id: user.id,
|
||||||
|
username: user.username,
|
||||||
|
email: user.email,
|
||||||
|
displayName: user.displayName,
|
||||||
|
locale: user.locale,
|
||||||
|
isSiteAdmin: user.isSiteAdmin,
|
||||||
|
status: user.status,
|
||||||
|
emailVerifiedAt: user.emailVerifiedAt?.toISOString() ?? null,
|
||||||
|
createdAt: user.createdAt.toISOString(),
|
||||||
|
lastLoginAt: user.lastLoginAt?.toISOString() ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Every pond the user has a role grant in, with the grants that concern
|
||||||
|
* them. This is the user's own membership data — pond identity plus their
|
||||||
|
* roles — and deliberately carries none of those ponds' content. */
|
||||||
|
private async membershipsOf(user: User): Promise<unknown[]> {
|
||||||
|
const grants = await this.prisma.roleGrant.findMany({
|
||||||
|
where: { subjectType: 'USER', subjectId: user.id },
|
||||||
|
include: { pond: { select: { slug: true, name: true, type: true, ownerId: true } } },
|
||||||
|
orderBy: { createdAt: 'asc' },
|
||||||
|
});
|
||||||
|
const byPond = new Map<
|
||||||
|
string,
|
||||||
|
{ pondSlug: string; pondName: string; pondType: string; owner: boolean; grants: unknown[] }
|
||||||
|
>();
|
||||||
|
for (const grant of grants) {
|
||||||
|
const entry = byPond.get(grant.pondId) ?? {
|
||||||
|
pondSlug: grant.pond.slug,
|
||||||
|
pondName: grant.pond.name,
|
||||||
|
pondType: grant.pond.type,
|
||||||
|
owner: grant.pond.ownerId === user.id,
|
||||||
|
grants: [],
|
||||||
|
};
|
||||||
|
entry.grants.push({
|
||||||
|
role: grant.role,
|
||||||
|
scopeType: grant.scopeType,
|
||||||
|
scopeId: grant.scopeId,
|
||||||
|
effect: grant.effect,
|
||||||
|
createdAt: grant.createdAt.toISOString(),
|
||||||
|
});
|
||||||
|
byPond.set(grant.pondId, entry);
|
||||||
|
}
|
||||||
|
return [...byPond.values()];
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -59,6 +59,34 @@ export class ExportService {
|
|||||||
const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } });
|
const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } });
|
||||||
if (!pond) throw new NotFoundException();
|
if (!pond) throw new NotFoundException();
|
||||||
|
|
||||||
|
const archive = archiver('zip', { zlib: { level: 9 } });
|
||||||
|
res.set('Content-Type', 'application/zip');
|
||||||
|
res.set('Content-Disposition', `attachment; filename="${pond.slug}.zip"`);
|
||||||
|
res.set('X-Content-Type-Options', 'nosniff');
|
||||||
|
archive.on('error', (error) => {
|
||||||
|
this.logger.error({ pondId, err: error.message }, 'pond export archive failed');
|
||||||
|
res.destroy(error);
|
||||||
|
});
|
||||||
|
archive.pipe(res);
|
||||||
|
await this.appendPondMarkdown(archive, user, pond);
|
||||||
|
await archive.finalize();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append one pond's readable pages (as Markdown) and their media to an open
|
||||||
|
* archive, each entry under `prefix`. Shared by the pond ZIP above and the
|
||||||
|
* account data export (#68), which nests several ponds under `ponds/<slug>/`
|
||||||
|
* — the read filter runs here, so neither caller can leak a page the
|
||||||
|
* requester may not read. Page Markdown is appended as small strings; media
|
||||||
|
* as read streams, keeping memory bounded regardless of pond size.
|
||||||
|
*/
|
||||||
|
async appendPondMarkdown(
|
||||||
|
archive: archiver.Archiver,
|
||||||
|
user: User,
|
||||||
|
pond: { id: string; slug: string },
|
||||||
|
prefix = '',
|
||||||
|
): Promise<void> {
|
||||||
|
const pondId = pond.id;
|
||||||
const pages = await this.prisma.page.findMany({
|
const pages = await this.prisma.page.findMany({
|
||||||
where: { pondId, deletedAt: null },
|
where: { pondId, deletedAt: null },
|
||||||
orderBy: { title: 'asc' },
|
orderBy: { title: 'asc' },
|
||||||
@ -99,16 +127,6 @@ export class ExportService {
|
|||||||
attachments.map((a) => [a.id, `${a.id}.${imageExtension(a.mimeType)}`]),
|
attachments.map((a) => [a.id, `${a.id}.${imageExtension(a.mimeType)}`]),
|
||||||
);
|
);
|
||||||
|
|
||||||
const archive = archiver('zip', { zlib: { level: 9 } });
|
|
||||||
res.set('Content-Type', 'application/zip');
|
|
||||||
res.set('Content-Disposition', `attachment; filename="${pond.slug}.zip"`);
|
|
||||||
res.set('X-Content-Type-Options', 'nosniff');
|
|
||||||
archive.on('error', (error) => {
|
|
||||||
this.logger.error({ pondId, err: error.message }, 'pond export archive failed');
|
|
||||||
res.destroy(error);
|
|
||||||
});
|
|
||||||
archive.pipe(res);
|
|
||||||
|
|
||||||
for (const page of readablePages) {
|
for (const page of readablePages) {
|
||||||
const markdown = markdownForZip(
|
const markdown = markdownForZip(
|
||||||
page.contentCache?.markdown ?? '',
|
page.contentCache?.markdown ?? '',
|
||||||
@ -116,7 +134,7 @@ export class ExportService {
|
|||||||
mediaNameById,
|
mediaNameById,
|
||||||
);
|
);
|
||||||
// Page slugs are unique within a pond, so `<slug>.md` never collides.
|
// Page slugs are unique within a pond, so `<slug>.md` never collides.
|
||||||
archive.append(markdown, { name: `${page.slug}.md` });
|
archive.append(markdown, { name: `${prefix}${page.slug}.md` });
|
||||||
}
|
}
|
||||||
for (const attachment of attachments) {
|
for (const attachment of attachments) {
|
||||||
const stream = this.storage.createReadStream(pond.id, attachment.id);
|
const stream = this.storage.createReadStream(pond.id, attachment.id);
|
||||||
@ -128,13 +146,12 @@ export class ExportService {
|
|||||||
'export: media read failed',
|
'export: media read failed',
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
archive.append(stream, { name: `media/${mediaNameById.get(attachment.id)!}` });
|
archive.append(stream, { name: `${prefix}media/${mediaNameById.get(attachment.id)!}` });
|
||||||
}
|
}
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
{ pondId, pages: readablePages.length, media: attachments.length, userId: user.id },
|
{ pondId, pages: readablePages.length, media: attachments.length, userId: user.id },
|
||||||
'audit: pond exported as markdown zip',
|
'audit: pond exported as markdown zip',
|
||||||
);
|
);
|
||||||
await archive.finalize();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@ -1,10 +1,15 @@
|
|||||||
import { Module } from '@nestjs/common';
|
import { Module, OnModuleInit } from '@nestjs/common';
|
||||||
|
|
||||||
import { FilesModule } from '../files/files.module';
|
import { FilesModule } from '../files/files.module';
|
||||||
import { PagesModule } from '../pages/pages.module';
|
import { PagesModule } from '../pages/pages.module';
|
||||||
|
import { SchedulerModule } from '../scheduler/scheduler.module';
|
||||||
|
import { SchedulerService } from '../scheduler/scheduler.service';
|
||||||
|
|
||||||
import { ConversionJobService } from './conversion-job.service';
|
import { ConversionJobService } from './conversion-job.service';
|
||||||
import { ConversionWorker } from './conversion-worker.service';
|
import { ConversionWorker } from './conversion-worker.service';
|
||||||
|
import { DATA_EXPORT_PROCESSOR } from './data-export.constants';
|
||||||
|
import { DataExportController } from './data-export.controller';
|
||||||
|
import { DataExportService } from './data-export.service';
|
||||||
import { ExportController } from './export.controller';
|
import { ExportController } from './export.controller';
|
||||||
import { ExportService } from './export.service';
|
import { ExportService } from './export.service';
|
||||||
import { GotenbergHttpRenderer, GotenbergRenderer } from './gotenberg.renderer';
|
import { GotenbergHttpRenderer, GotenbergRenderer } from './gotenberg.renderer';
|
||||||
@ -14,23 +19,29 @@ import { ImportService } from './import.service';
|
|||||||
import { JobsController } from './jobs.controller';
|
import { JobsController } from './jobs.controller';
|
||||||
import { PandocConverter, PandocServerConverter } from './pandoc.converter';
|
import { PandocConverter, PandocServerConverter } from './pandoc.converter';
|
||||||
|
|
||||||
|
/** How often expired data-export payloads are purged (#68). Hourly is ample:
|
||||||
|
* the link's own expiry check already stops downloads the moment it lapses. */
|
||||||
|
const EXPORT_PURGE_CADENCE_SECONDS = 60 * 60;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Import/export orchestration (ADR 0009): the conversion job queue, its worker,
|
* Import/export orchestration (ADR 0009): the conversion job queue, its worker,
|
||||||
* the pandoc-server client (#62), and the document import pipeline (#63, which
|
* the pandoc-server client (#62), the document import pipeline (#63), the
|
||||||
* turns an uploaded `.docx`/`.odt` into a new page). Export (#65) will add its
|
* feature exports (#65/#67), and the GDPR account data export (#68).
|
||||||
* feature endpoints here too.
|
|
||||||
*/
|
*/
|
||||||
@Module({
|
@Module({
|
||||||
imports: [FilesModule, PagesModule],
|
imports: [FilesModule, PagesModule, SchedulerModule],
|
||||||
controllers: [JobsController, ImportController, ExportController],
|
controllers: [JobsController, ImportController, ExportController, DataExportController],
|
||||||
providers: [
|
providers: [
|
||||||
ConversionJobService,
|
ConversionJobService,
|
||||||
ConversionWorker,
|
ConversionWorker,
|
||||||
ImportService,
|
ImportService,
|
||||||
ExportService,
|
ExportService,
|
||||||
|
DataExportService,
|
||||||
// The worker resolves the import pipeline through this token (never the
|
// The worker resolves the import pipeline through this token (never the
|
||||||
// class), so its file does not import the import service's (avoids a cycle).
|
// class), so its file does not import the import service's (avoids a cycle).
|
||||||
{ provide: IMPORT_PROCESSOR, useExisting: ImportService },
|
{ provide: IMPORT_PROCESSOR, useExisting: ImportService },
|
||||||
|
// Same token indirection for the data-export builder (#68).
|
||||||
|
{ provide: DATA_EXPORT_PROCESSOR, useExisting: DataExportService },
|
||||||
// Bind the abstract converter to the HTTP implementation; tests override
|
// Bind the abstract converter to the HTTP implementation; tests override
|
||||||
// this provider with a fake so the queue mechanics need no live sidecar.
|
// this provider with a fake so the queue mechanics need no live sidecar.
|
||||||
{ provide: PandocConverter, useClass: PandocServerConverter },
|
{ provide: PandocConverter, useClass: PandocServerConverter },
|
||||||
@ -39,4 +50,17 @@ import { PandocConverter, PandocServerConverter } from './pandoc.converter';
|
|||||||
],
|
],
|
||||||
exports: [ConversionJobService, PandocConverter],
|
exports: [ConversionJobService, PandocConverter],
|
||||||
})
|
})
|
||||||
export class ImportExportModule {}
|
export class ImportExportModule implements OnModuleInit {
|
||||||
|
constructor(
|
||||||
|
private readonly scheduler: SchedulerService,
|
||||||
|
private readonly dataExport: DataExportService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
onModuleInit(): void {
|
||||||
|
this.scheduler.register({
|
||||||
|
name: 'data-export-purge',
|
||||||
|
cadenceSeconds: EXPORT_PURGE_CADENCE_SECONDS,
|
||||||
|
run: () => this.dataExport.purgeExpired().then(() => undefined),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -171,6 +171,7 @@ export class ImportService implements ImportProcessor {
|
|||||||
targetFormat: 'page',
|
targetFormat: 'page',
|
||||||
errorCode: null,
|
errorCode: null,
|
||||||
resultPageId: page.id,
|
resultPageId: page.id,
|
||||||
|
expiresAt: null,
|
||||||
createdAt: now,
|
createdAt: now,
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
};
|
};
|
||||||
|
|||||||
46
apps/web/src/export/job-download.ts
Normal file
46
apps/web/src/export/job-download.ts
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
import type { ConversionJobView } from '@dorfteich/shared';
|
||||||
|
|
||||||
|
import { ApiError, apiGet } from '../lib/api';
|
||||||
|
|
||||||
|
const POLL_INTERVAL_MS = 1000;
|
||||||
|
/** Default poll budget — a whole-account export (#68) can take longer than a
|
||||||
|
* single-page conversion, so callers may raise or lower it. */
|
||||||
|
const DEFAULT_MAX_POLLS = 300;
|
||||||
|
|
||||||
|
const delay = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
|
||||||
|
/** Poll a conversion job until it succeeds or fails, or the poll budget runs
|
||||||
|
* out (returns whatever the last status was). Shared by every export hook. */
|
||||||
|
export async function pollJob(
|
||||||
|
job: ConversionJobView,
|
||||||
|
maxPolls = DEFAULT_MAX_POLLS,
|
||||||
|
): Promise<ConversionJobView> {
|
||||||
|
let current = job;
|
||||||
|
for (
|
||||||
|
let poll = 0;
|
||||||
|
current.status !== 'succeeded' && current.status !== 'failed' && poll < maxPolls;
|
||||||
|
poll += 1
|
||||||
|
) {
|
||||||
|
await delay(POLL_INTERVAL_MS);
|
||||||
|
current = await apiGet<ConversionJobView>(`/jobs/${current.id}`);
|
||||||
|
}
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Download a finished job's result and save it as `fileName`. The result
|
||||||
|
* endpoint is session-authenticated, so a plain `fetch` carries the cookie. */
|
||||||
|
export async function downloadJobResult(jobId: string, fileName: string): Promise<void> {
|
||||||
|
const response = await fetch(`/api/v1/jobs/${jobId}/result`);
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new ApiError(response.status, { code: `http_${response.status}`, message: '' });
|
||||||
|
}
|
||||||
|
const blob = await response.blob();
|
||||||
|
const url = URL.createObjectURL(blob);
|
||||||
|
const anchor = document.createElement('a');
|
||||||
|
anchor.href = url;
|
||||||
|
anchor.download = fileName;
|
||||||
|
document.body.appendChild(anchor);
|
||||||
|
anchor.click();
|
||||||
|
anchor.remove();
|
||||||
|
URL.revokeObjectURL(url);
|
||||||
|
}
|
||||||
54
apps/web/src/export/use-data-export.ts
Normal file
54
apps/web/src/export/use-data-export.ts
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
import type { ConversionJobView } from '@dorfteich/shared';
|
||||||
|
import { useCallback, useState } from 'react';
|
||||||
|
|
||||||
|
import { ApiError, apiPost } from '../lib/api';
|
||||||
|
|
||||||
|
import { downloadJobResult, pollJob } from './job-download';
|
||||||
|
|
||||||
|
export type DataExportStatus = 'idle' | 'busy' | 'ready' | 'error' | 'rateLimited';
|
||||||
|
|
||||||
|
export interface UseDataExport {
|
||||||
|
status: DataExportStatus;
|
||||||
|
/** When the ready download link expires (ISO), or null. */
|
||||||
|
expiresAt: string | null;
|
||||||
|
request: () => void;
|
||||||
|
download: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Self-service GDPR data export (issue #68): request the export, poll the job,
|
||||||
|
* and — once it is ready — download the ZIP. A recent request is rejected with
|
||||||
|
* 429, surfaced as a distinct `rateLimited` status.
|
||||||
|
*/
|
||||||
|
export function useDataExport(): UseDataExport {
|
||||||
|
const [status, setStatus] = useState<DataExportStatus>('idle');
|
||||||
|
const [expiresAt, setExpiresAt] = useState<string | null>(null);
|
||||||
|
const [jobId, setJobId] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const request = useCallback((): void => {
|
||||||
|
void (async () => {
|
||||||
|
setStatus('busy');
|
||||||
|
setExpiresAt(null);
|
||||||
|
setJobId(null);
|
||||||
|
try {
|
||||||
|
const job = await pollJob(await apiPost<ConversionJobView>('/users/me/data-export'));
|
||||||
|
if (job.status === 'succeeded') {
|
||||||
|
setJobId(job.id);
|
||||||
|
setExpiresAt(job.expiresAt);
|
||||||
|
setStatus('ready');
|
||||||
|
} else {
|
||||||
|
setStatus('error');
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
setStatus(error instanceof ApiError && error.status === 429 ? 'rateLimited' : 'error');
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const download = useCallback((): void => {
|
||||||
|
if (!jobId) return;
|
||||||
|
void downloadJobResult(jobId, 'dorfteich-export.zip').catch(() => setStatus('error'));
|
||||||
|
}, [jobId]);
|
||||||
|
|
||||||
|
return { status, expiresAt, request, download };
|
||||||
|
}
|
||||||
@ -1,31 +1,16 @@
|
|||||||
import type { ConversionJobView, ExportFormat } from '@dorfteich/shared';
|
import type { ConversionJobView, ExportFormat } from '@dorfteich/shared';
|
||||||
import { useCallback, useState } from 'react';
|
import { useCallback, useState } from 'react';
|
||||||
|
|
||||||
import { ApiError, apiGet, apiPost } from '../lib/api';
|
import { apiPost } from '../lib/api';
|
||||||
|
|
||||||
|
import { downloadJobResult, pollJob } from './job-download';
|
||||||
|
|
||||||
export type ExportStatus = 'idle' | 'busy' | 'error';
|
export type ExportStatus = 'idle' | 'busy' | 'error';
|
||||||
|
|
||||||
const POLL_INTERVAL_MS = 1000;
|
/** A single-page conversion is quick — cap the poll budget below the shared
|
||||||
|
* default. */
|
||||||
const MAX_POLLS = 180;
|
const MAX_POLLS = 180;
|
||||||
|
|
||||||
const delay = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
|
|
||||||
|
|
||||||
/** Fetch a finished job's result and save it under `fileName`. The result
|
|
||||||
* endpoint is session-authenticated, so a plain `fetch` carries the cookie. */
|
|
||||||
async function downloadResult(jobId: string, fileName: string): Promise<void> {
|
|
||||||
const response = await fetch(`/api/v1/jobs/${jobId}/result`);
|
|
||||||
if (!response.ok) throw new ApiError(response.status, { code: 'conversion_failed', message: '' });
|
|
||||||
const blob = await response.blob();
|
|
||||||
const url = URL.createObjectURL(blob);
|
|
||||||
const anchor = document.createElement('a');
|
|
||||||
anchor.href = url;
|
|
||||||
anchor.download = fileName;
|
|
||||||
document.body.appendChild(anchor);
|
|
||||||
anchor.click();
|
|
||||||
anchor.remove();
|
|
||||||
URL.revokeObjectURL(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface UseDocumentExport {
|
export interface UseDocumentExport {
|
||||||
/** Per-format status so each button shows its own progress/error. */
|
/** Per-format status so each button shows its own progress/error. */
|
||||||
status: Partial<Record<ExportFormat, ExportStatus>>;
|
status: Partial<Record<ExportFormat, ExportStatus>>;
|
||||||
@ -48,17 +33,12 @@ export function useDocumentExport(): UseDocumentExport {
|
|||||||
void (async () => {
|
void (async () => {
|
||||||
set(format, 'busy');
|
set(format, 'busy');
|
||||||
try {
|
try {
|
||||||
let job = await apiPost<ConversionJobView>(`/pages/${pageId}/export`, { format });
|
const job = await pollJob(
|
||||||
for (
|
await apiPost<ConversionJobView>(`/pages/${pageId}/export`, { format }),
|
||||||
let poll = 0;
|
MAX_POLLS,
|
||||||
job.status !== 'succeeded' && job.status !== 'failed' && poll < MAX_POLLS;
|
);
|
||||||
poll += 1
|
|
||||||
) {
|
|
||||||
await delay(POLL_INTERVAL_MS);
|
|
||||||
job = await apiGet<ConversionJobView>(`/jobs/${job.id}`);
|
|
||||||
}
|
|
||||||
if (job.status === 'succeeded') {
|
if (job.status === 'succeeded') {
|
||||||
await downloadResult(job.id, `${slug}.${format}`);
|
await downloadJobResult(job.id, `${slug}.${format}`);
|
||||||
set(format, 'idle');
|
set(format, 'idle');
|
||||||
} else {
|
} else {
|
||||||
set(format, 'error');
|
set(format, 'error');
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import { useTranslation } from 'react-i18next';
|
|||||||
|
|
||||||
import { useAuth } from '../auth/auth-context';
|
import { useAuth } from '../auth/auth-context';
|
||||||
import { Field, FormError, FormSuccess, applyFieldErrors } from '../components/forms';
|
import { Field, FormError, FormSuccess, applyFieldErrors } from '../components/forms';
|
||||||
|
import { useDataExport } from '../export/use-data-export';
|
||||||
import { apiDelete, apiGet, apiPatch, apiPost } from '../lib/api';
|
import { apiDelete, apiGet, apiPatch, apiPost } from '../lib/api';
|
||||||
|
|
||||||
interface SessionView {
|
interface SessionView {
|
||||||
@ -25,10 +26,57 @@ export function SettingsPage(): React.JSX.Element {
|
|||||||
<ProfileSection />
|
<ProfileSection />
|
||||||
<PasswordSection />
|
<PasswordSection />
|
||||||
<SessionsSection />
|
<SessionsSection />
|
||||||
|
<DataExportSection />
|
||||||
</>
|
</>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function DataExportSection(): React.JSX.Element {
|
||||||
|
const { t, i18n } = useTranslation();
|
||||||
|
const { status, expiresAt, request, download } = useDataExport();
|
||||||
|
|
||||||
|
const formatTime = (iso: string): string =>
|
||||||
|
new Intl.DateTimeFormat(i18n.language, { dateStyle: 'medium', timeStyle: 'short' }).format(
|
||||||
|
new Date(iso),
|
||||||
|
);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<section className="settings-section">
|
||||||
|
<h2>{t('settings:dataExport.title')}</h2>
|
||||||
|
<p className="sidebar__hint">{t('settings:dataExport.description')}</p>
|
||||||
|
{status === 'error' && (
|
||||||
|
<p className="form-banner form-banner--error" role="alert">
|
||||||
|
{t('settings:dataExport.failed')}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
{status === 'rateLimited' && (
|
||||||
|
<p className="form-banner form-banner--error" role="alert">
|
||||||
|
{t('settings:dataExport.rateLimited')}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
{status === 'ready' ? (
|
||||||
|
<>
|
||||||
|
<FormSuccess message={t('settings:dataExport.ready')} />
|
||||||
|
<button type="button" className="button" onClick={download}>
|
||||||
|
{t('settings:dataExport.download')}
|
||||||
|
</button>
|
||||||
|
{expiresAt && (
|
||||||
|
<p className="sidebar__hint">
|
||||||
|
{t('settings:dataExport.expiresHint', { when: formatTime(expiresAt) })}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<button type="button" className="button" onClick={request} disabled={status === 'busy'}>
|
||||||
|
{status === 'busy'
|
||||||
|
? t('settings:dataExport.preparing')
|
||||||
|
: t('settings:dataExport.request')}
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function ProfileSection(): React.JSX.Element {
|
function ProfileSection(): React.JSX.Element {
|
||||||
const { t, i18n } = useTranslation();
|
const { t, i18n } = useTranslation();
|
||||||
const { user, refresh } = useAuth();
|
const { user, refresh } = useAuth();
|
||||||
|
|||||||
@ -25,6 +25,17 @@
|
|||||||
"revokeAll": "Alle anderen Sitzungen abmelden",
|
"revokeAll": "Alle anderen Sitzungen abmelden",
|
||||||
"empty": "Keine weiteren aktiven Sitzungen."
|
"empty": "Keine weiteren aktiven Sitzungen."
|
||||||
},
|
},
|
||||||
|
"dataExport": {
|
||||||
|
"title": "Meine Daten exportieren",
|
||||||
|
"description": "Lade ein ZIP mit deinem Profil, einer Liste deiner Mitgliedschaften und dem Markdown-Export deines persönlichen Teichs sowie der Teiche, die dir gehören.",
|
||||||
|
"request": "Export vorbereiten",
|
||||||
|
"preparing": "Dein Export wird vorbereitet…",
|
||||||
|
"download": "Export herunterladen",
|
||||||
|
"ready": "Dein Export ist bereit.",
|
||||||
|
"expiresHint": "Der Download-Link läuft am {{when}} ab.",
|
||||||
|
"failed": "Der Export konnte nicht erstellt werden. Bitte versuche es erneut.",
|
||||||
|
"rateLimited": "Du hast kürzlich einen Export angefordert. Bitte versuche es später erneut."
|
||||||
|
},
|
||||||
"admin": {
|
"admin": {
|
||||||
"title": "Administration",
|
"title": "Administration",
|
||||||
"instanceName": "Name der Instanz",
|
"instanceName": "Name der Instanz",
|
||||||
|
|||||||
@ -25,6 +25,17 @@
|
|||||||
"revokeAll": "Sign out all other sessions",
|
"revokeAll": "Sign out all other sessions",
|
||||||
"empty": "No other active sessions."
|
"empty": "No other active sessions."
|
||||||
},
|
},
|
||||||
|
"dataExport": {
|
||||||
|
"title": "Export my data",
|
||||||
|
"description": "Download a ZIP with your profile, a list of your memberships, and the Markdown export of your personal pond and the ponds you own.",
|
||||||
|
"request": "Prepare export",
|
||||||
|
"preparing": "Preparing your export…",
|
||||||
|
"download": "Download export",
|
||||||
|
"ready": "Your export is ready.",
|
||||||
|
"expiresHint": "The download link expires on {{when}}.",
|
||||||
|
"failed": "The export could not be prepared. Please try again.",
|
||||||
|
"rateLimited": "You requested an export recently. Please try again later."
|
||||||
|
},
|
||||||
"admin": {
|
"admin": {
|
||||||
"title": "Administration",
|
"title": "Administration",
|
||||||
"instanceName": "Instance name",
|
"instanceName": "Instance name",
|
||||||
|
|||||||
@ -21,6 +21,10 @@ export interface ConversionJobView {
|
|||||||
* the client can navigate to it. `null` for a pending/failed import and for
|
* the client can navigate to it. `null` for a pending/failed import and for
|
||||||
* plain byte→byte conversions (export). */
|
* plain byte→byte conversions (export). */
|
||||||
resultPageId: string | null;
|
resultPageId: string | null;
|
||||||
|
/** For a data-export job (#68), when its download link stops working — the
|
||||||
|
* result is purged after this. `null` for every other job kind, which never
|
||||||
|
* expires. */
|
||||||
|
expiresAt: string | null;
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
updatedAt: string;
|
updatedAt: string;
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user