import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { INestApplication } from '@nestjs/common'; import { PrismaClient } from '@prisma/client'; import { unzipSync, zipSync } from 'fflate'; import request from 'supertest'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { AuthTokensService } from '../auth/auth-tokens.service'; import { FileStorageService } from '../files/file-storage.service'; import { FilesService } from '../files/files.service'; import { PluginsService } from '../plugins/plugins.service'; import { createTestApp, sessionCookieOf } from '../testing/test-app'; import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db'; import { UsersService } from '../users/users.service'; import { ConversionWorker } from './conversion-worker.service'; import { GotenbergRenderer, RenderError } from './gotenberg.renderer'; import { ConversionRequest, ConversionResult, PandocConverter } from './pandoc.converter'; /** Export (issues #65/#67): pond ZIP of Markdown, per-page docx/odt (pandoc), * and per-page PDF (Gotenberg). The office/PDF paths use injected fakes that * record the input they are handed, so image-inlining / font-inlining is checked * without a live sidecar. */ const enc = (text: string) => new TextEncoder().encode(text); const PNG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; class RecordingConverter extends PandocConverter { lastInput = ''; convert(request: ConversionRequest): Promise { this.lastInput = request.input.toString('utf8'); return Promise.resolve({ output: Buffer.from('OFFICE-BYTES'), mimeType: 'application/x-test' }); } reachable(): Promise { return Promise.resolve(true); } } class RecordingRenderer extends GotenbergRenderer { lastHtml = ''; failWith: RenderError | null = null; renderHtmlToPdf(html: string): Promise { this.lastHtml = html; if (this.failWith) return Promise.reject(this.failWith); return Promise.resolve(Buffer.from('%PDF-1.7 fake')); } reachable(): Promise { return Promise.resolve(true); } } /** Filenames of every entry in a ZIP buffer. */ function zipEntries(buffer: Buffer): Record { return unzipSync(new Uint8Array(buffer)); } describe.skipIf(!hasTestDb)('export (e2e, issue #65)', () => { let app: INestApplication; let prisma: PrismaClient; let worker: ConversionWorker; let files: FilesService; let fake: RecordingConverter; let renderer: RecordingRenderer; const suffix = uniqueSuffix(); const password = 'exportiere meine sachen 1'; const owner = { username: `ella-export-${suffix}`, displayName: `Ella Export ${suffix}` }; let ownerId: string; let ownerCookie: string; let personalPondId: string; const api = () => request(app.getHttpServer()); async function login(username: string): Promise { const res = await api() .post('/api/v1/auth/login') .send({ usernameOrEmail: username, password }) .expect(200); return sessionCookieOf(res); } /** Create a page with a crafted content-cache Markdown (the exporter reads the * cache, not the Yjs state), returning its slug. */ async function seedPage( pondId: string, title: string, markdown: string, html = '', ): Promise { const slug = title.toLowerCase().replace(/[^a-z0-9]+/g, '-'); await prisma.page.create({ data: { pondId, title, slug, ydocState: new Uint8Array(), sortKey: title, createdBy: ownerId, contentCache: { create: { plainText: markdown, markdown, html, outline: [] } }, }, }); return slug; } beforeAll(async () => { prisma = createTestPrisma(); await prisma.rateLimit.deleteMany({}); fake = new RecordingConverter(); renderer = new RecordingRenderer(); app = await createTestApp((builder) => builder .overrideProvider(PandocConverter) .useValue(fake) .overrideProvider(GotenbergRenderer) .useValue(renderer), ); worker = app.get(ConversionWorker); files = app.get(FilesService); const users = app.get(UsersService); const tokens = app.get(AuthTokensService); const ownerUser = await users.createUser({ username: owner.username, email: `${owner.username}@example.org`, displayName: owner.displayName, password, locale: 'en', }); ownerId = ownerUser.id; const verify = await tokens.issue(ownerUser.id, 'EMAIL_VERIFICATION', 600); await api().post('/api/v1/auth/verify-email').send({ token: verify }).expect(204); ownerCookie = await login(owner.username); personalPondId = (await prisma.pond.findFirstOrThrow({ where: { ownerId, type: 'PERSONAL' } })) .id; }); afterAll(async () => { await prisma.conversionJob.deleteMany({ where: { owner: { username: { contains: suffix } } } }); const where = { pond: { owner: { username: { contains: suffix } } } }; await prisma.attachment.deleteMany({ where }); await prisma.pageLabel.deleteMany({ where: { page: where } }); await prisma.label.deleteMany({ where }); await prisma.page.deleteMany({ where }); await prisma.roleGrant.deleteMany({ where }); await prisma.pond.deleteMany({ where: { owner: { username: { contains: suffix } } } }); await prisma.user.deleteMany({ where: { username: { contains: suffix } } }); await prisma.$disconnect(); await app.close(); }); it('exports a pond ZIP with one .md per page, a media dir, and relative links', async () => { // A real stored image so the media stream has bytes. const image = await files.upload({ id: ownerId } as never, personalPondId, { buffer: Buffer.from(PNG_BASE64, 'base64'), size: 70, originalname: 'dot.png', }); await seedPage(personalPondId, 'Zip Intro', 'Welcome. See [[zip-target]].'); await seedPage(personalPondId, 'Zip Target', `# Zip Target\n\nAn image: ![dot](${image.id})`); const res = await api() .get(`/api/v1/ponds/${personalPondId}/export/markdown`) .set('Cookie', ownerCookie) .buffer(true) .parse((r, cb) => { const chunks: Buffer[] = []; r.on('data', (c: Buffer) => chunks.push(c)); r.on('end', () => cb(null, Buffer.concat(chunks))); }) .expect(200); expect(res.headers['content-type']).toContain('application/zip'); expect(res.headers['content-disposition']).toContain('attachment'); const entries = zipEntries(res.body as Buffer); const names = Object.keys(entries); expect(names).toContain('zip-intro.md'); expect(names).toContain('zip-target.md'); expect(names).toContain(`media/${image.id}.png`); const intro = Buffer.from(entries['zip-intro.md']!).toString('utf8'); // The wikilink became a relative link to the readable target's .md. expect(intro).toContain('[zip-target](zip-target.md)'); const target = Buffer.from(entries['zip-target.md']!).toString('utf8'); // The image src became a working relative path into media/. expect(target).toContain(`![dot](media/${image.id}.png)`); }); it('skips an attachment whose bytes are missing on disk instead of crashing', async () => { // An attachment row with no file (data drift): upload then remove the bytes. const image = await files.upload({ id: ownerId } as never, personalPondId, { buffer: Buffer.from(PNG_BASE64, 'base64'), size: 70, originalname: 'ghost.png', }); await app.get(FileStorageService).delete(personalPondId, image.id); await seedPage(personalPondId, 'Ghost Image', `# Ghost\n\n![gone](${image.id})`); const res = await api() .get(`/api/v1/ponds/${personalPondId}/export/markdown`) .set('Cookie', ownerCookie) .buffer(true) .parse((r, cb) => { const chunks: Buffer[] = []; r.on('data', (c: Buffer) => chunks.push(c)); r.on('end', () => cb(null, Buffer.concat(chunks))); }) .expect(200); const names = Object.keys(zipEntries(res.body as Buffer)); // The page is still exported; the missing media file is simply not included. expect(names).toContain('ghost-image.md'); expect(names).not.toContain(`media/${image.id}.png`); }); it('omits pages a label-restricted reader cannot read from their pond ZIP', async () => { // A shared pond with a public page and a `secret`-labelled page; a reader is // allowed pond-wide but denied on the secret label. const shared = await prisma.pond.create({ data: { slug: `shared-export-${suffix}`, name: 'Shared Export', type: 'SHARED', ownerId, usage: { create: {} }, }, }); const reader = await app.get(UsersService).createUser({ username: `rudy-reader-${suffix}`, email: `rudy-reader-${suffix}@example.org`, displayName: `Rudy Reader ${suffix}`, password, locale: 'en', }); await app.get(UsersService).markEmailVerified(reader.id); const secret = await prisma.label.create({ data: { pondId: shared.id, name: 'secret', color: '#334455' }, }); const publicSlug = await seedPageIn(shared.id, 'Public Page', 'Everyone may read this.'); const secretSlug = await seedPageIn(shared.id, 'Secret Page', 'Classified.'); const secretPage = await prisma.page.findFirstOrThrow({ where: { pondId: shared.id, slug: secretSlug }, }); await prisma.pageLabel.create({ data: { pageId: secretPage.id, labelId: secret.id } }); // Reader: pond-wide reader ALLOW + secret-label DENY. await prisma.roleGrant.createMany({ data: [ { pondId: shared.id, subjectType: 'USER', subjectId: reader.id, role: 'READER', scopeType: 'POND', effect: 'ALLOW', createdBy: ownerId, }, { pondId: shared.id, subjectType: 'USER', subjectId: reader.id, role: 'READER', scopeType: 'LABEL', scopeId: secret.id, effect: 'DENY', createdBy: ownerId, }, ], }); const readerCookie = await login(`rudy-reader-${suffix}`); const res = await api() .get(`/api/v1/ponds/${shared.id}/export/markdown`) .set('Cookie', readerCookie) .buffer(true) .parse((r, cb) => { const chunks: Buffer[] = []; r.on('data', (c: Buffer) => chunks.push(c)); r.on('end', () => cb(null, Buffer.concat(chunks))); }) .expect(200); const names = Object.keys(zipEntries(res.body as Buffer)); expect(names).toContain(`${publicSlug}.md`); expect(names).not.toContain(`${secretSlug}.md`); }); it('exports a page to .docx: images inlined as data URIs, wikilinks flattened', async () => { const image = await files.upload({ id: ownerId } as never, personalPondId, { buffer: Buffer.from(PNG_BASE64, 'base64'), size: 70, originalname: 'inl.png', }); const slug = await seedPage( personalPondId, 'Docx Source', `# Docx Source\n\n![pic](${image.id}) and [[other|Other Page]].`, ); const page = await prisma.page.findFirstOrThrow({ where: { pondId: personalPondId, slug }, }); const enqueued = await api() .post(`/api/v1/pages/${page.id}/export`) .set('Cookie', ownerCookie) .send({ format: 'docx' }) .expect(201); expect(enqueued.body.status).toBe('pending'); expect(enqueued.body.kind).toBe('export_docx'); await worker.drain(); // The Markdown handed to pandoc inlined the image and flattened the wikilink. expect(fake.lastInput).toContain('data:image/png;base64,'); expect(fake.lastInput).toContain('Other Page'); expect(fake.lastInput).not.toContain('[['); const done = await api() .get(`/api/v1/jobs/${enqueued.body.id}`) .set('Cookie', ownerCookie) .expect(200); expect(done.body.status).toBe('succeeded'); const result = await api() .get(`/api/v1/jobs/${enqueued.body.id}/result`) .set('Cookie', ownerCookie) .expect(200); expect(result.text).toBe('OFFICE-BYTES'); }); it('exports a page to PDF: content + image inlined, font CSS, via Gotenberg', async () => { const image = await files.upload({ id: ownerId } as never, personalPondId, { buffer: Buffer.from(PNG_BASE64, 'base64'), size: 70, originalname: 'pdf.png', }); const slug = await seedPage( personalPondId, 'Pdf Source', '# Pdf Source\n\nbody', `

A PDF body paragraph.

pic`, ); const page = await prisma.page.findFirstOrThrow({ where: { pondId: personalPondId, slug }, }); const enqueued = await api() .post(`/api/v1/pages/${page.id}/export`) .set('Cookie', ownerCookie) .send({ format: 'pdf' }) .expect(201); expect(enqueued.body.kind).toBe('export_pdf'); await worker.drain(); // The HTML handed to Gotenberg carries the title, the pond's font stack as a // CSS variable, and the image inlined as a data URI (no network needed). expect(renderer.lastHtml).toContain('Pdf Source'); expect(renderer.lastHtml).toContain("--font-body: 'Roboto'"); expect(renderer.lastHtml).toContain('src="data:image/png;base64,'); const done = await api() .get(`/api/v1/jobs/${enqueued.body.id}`) .set('Cookie', ownerCookie) .expect(200); expect(done.body.status).toBe('succeeded'); const result = await api() .get(`/api/v1/jobs/${enqueued.body.id}/result`) .set('Cookie', ownerCookie) .buffer(true) .parse((r, cb) => { const chunks: Buffer[] = []; r.on('data', (c: Buffer) => chunks.push(c)); r.on('end', () => cb(null, Buffer.concat(chunks))); }) .expect(200); expect(result.headers['content-type']).toContain('application/pdf'); expect((result.body as Buffer).toString('utf8')).toContain('%PDF'); }); it('inlines active section-style plugin CSS into the PDF html (#75)', async () => { // Install the real reference plugin and make it active everywhere, so the // export path exercises the same package users get. const pluginDir = join(__dirname, '../../../../packages/plugins/section-styles-basic'); const plugins = app.get(PluginsService); // A `required` plugin refuses uninstall, and the local dev DB may carry // state from an aborted earlier run — always demote + remove, both ways. async function removeIfInstalled(): Promise { await plugins.setMode('section-styles-basic', 'disabled').catch(() => undefined); await plugins.uninstall('section-styles-basic').catch(() => undefined); } await removeIfInstalled(); await plugins.install( Buffer.from( zipSync({ 'manifest.json': new Uint8Array(readFileSync(join(pluginDir, 'manifest.json'))), 'styles.css': new Uint8Array(readFileSync(join(pluginDir, 'styles.css'))), }), ), ); try { await plugins.setMode('section-styles-basic', 'required'); const slug = await seedPage( personalPondId, 'Sectioned Pdf', 'body', '

boxed

', ); const page = await prisma.page.findFirstOrThrow({ where: { pondId: personalPondId, slug }, }); await api() .post(`/api/v1/pages/${page.id}/export`) .set('Cookie', ownerCookie) .send({ format: 'pdf' }) .expect(201); await worker.drain(); // The Gotenberg HTML carries both the section markup and the plugin's // scoped CSS, so the box renders in the (network-isolated) PDF. expect(renderer.lastHtml).toContain('dt-style-section-styles-basic-callout"'); expect(renderer.lastHtml).toContain('.dt-style-section-styles-basic-callout {'); expect(renderer.lastHtml).toContain('/* section-styles-basic@'); } finally { await removeIfInstalled(); } }); it('degrades plugin blocks in exports: snapshot SVG, manifest text, tombstone (#79)', async () => { const plugins = app.get(PluginsService); async function removeIfInstalled(id: string): Promise { await plugins.setMode(id, 'disabled').catch(() => undefined); await plugins.uninstall(id).catch(() => undefined); } // A block plugin with a text fallback; a second one that gets uninstalled. const blockManifest = (id: string, fallback: string) => ({ id, name: `Fixture ${id}`, version: '1.0.0', apiVersion: '1', kind: 'code', extensionPoints: [{ type: 'block', id: 'main', title: { de: id, en: id } }], permissions: [], fallback: { type: 'text', value: fallback }, license: 'MIT', }); await removeIfInstalled('fx-toc'); await removeIfInstalled('fx-gone'); await plugins.install( Buffer.from( zipSync({ 'manifest.json': enc(JSON.stringify(blockManifest('fx-toc', '[Table of contents]'))), 'plugin.js': enc('export default {}'), }), ), ); await plugins.install( Buffer.from( zipSync({ 'manifest.json': enc(JSON.stringify(blockManifest('fx-gone', '[Gone but text]'))), 'plugin.js': enc('export default {}'), }), ), ); await plugins.uninstall('fx-gone'); // tombstone: manifest snapshot remains try { // Content cache as the shared renderer emits it: a diagram block with a // stored SVG snapshot (hostile bits included), a toc block without one, // and a block of the uninstalled plugin. const svg = ''; // Exactly the attribute encoding the shared HTML renderer applies. const escapeAttr = (value: string) => value .replace(/&/g, '&') .replace(//g, '>') .replace(/"/g, '"'); const svgData = escapeAttr(JSON.stringify({ source: 'graph', svg })); const html = `
[mermaid/diagram]
` + '
[fx-toc/main]
' + '
[fx-gone/main]
'; const markdown = [ '```dorfteich-plugin fx-toc/main', '{}', '```', '', '::: {data-section-style="p/callout"}', 'Sectioned text.', ':::', ].join('\n'); const slug = await seedPage(personalPondId, 'Fallback Exports', markdown, html); const page = await prisma.page.findFirstOrThrow({ where: { pondId: personalPondId, slug }, }); // PDF: snapshot SVG (sanitized!), manifest text, tombstone text. await api() .post(`/api/v1/pages/${page.id}/export`) .set('Cookie', ownerCookie) .send({ format: 'pdf' }) .expect(201); await worker.drain(); expect(renderer.lastHtml).toContain(''); expect(renderer.lastHtml).toContain('[Table of contents]'); expect(renderer.lastHtml).toContain('[Gone but text]'); expect(renderer.lastHtml).not.toContain('dt-plugin-block"'); // docx: fallback text, quoted section, no fence artifacts. await api() .post(`/api/v1/pages/${page.id}/export`) .set('Cookie', ownerCookie) .send({ format: 'docx' }) .expect(201); await worker.drain(); expect(fake.lastInput).toContain('\\[Table of contents\\]'); expect(fake.lastInput).toContain('> Sectioned text.'); expect(fake.lastInput).not.toContain('dorfteich-plugin'); expect(fake.lastInput).not.toContain(':::'); } finally { await removeIfInstalled('fx-toc'); await removeIfInstalled('fx-gone'); } }); it('fails a PDF export when the renderer is down', async () => { renderer.failWith = new RenderError('render_failed', false, 'gotenberg exploded'); const slug = await seedPage(personalPondId, 'Pdf Fails', 'x', '

x

'); const page = await prisma.page.findFirstOrThrow({ where: { pondId: personalPondId, slug }, }); const enqueued = await api() .post(`/api/v1/pages/${page.id}/export`) .set('Cookie', ownerCookie) .send({ format: 'pdf' }) .expect(201); await worker.drain(); renderer.failWith = null; const done = await api() .get(`/api/v1/jobs/${enqueued.body.id}`) .set('Cookie', ownerCookie) .expect(200); expect(done.body.status).toBe('failed'); expect(done.body.errorCode).toBe('render_failed'); }); it('streams a large pond export (500 pages) without buffering it all', async () => { const big = await prisma.pond.create({ data: { slug: `big-export-${suffix}`, name: 'Big Export', type: 'SHARED', ownerId, usage: { create: {} }, }, }); await prisma.roleGrant.create({ data: { pondId: big.id, subjectType: 'USER', subjectId: ownerId, role: 'POND_ADMIN', scopeType: 'POND', effect: 'ALLOW', createdBy: ownerId, }, }); const rows = Array.from({ length: 500 }, (_, i) => ({ pondId: big.id, title: `Page ${i}`, slug: `page-${i}`, ydocState: new Uint8Array(), sortKey: String(i).padStart(4, '0'), createdBy: ownerId, })); await prisma.page.createMany({ data: rows }); const created = await prisma.page.findMany({ where: { pondId: big.id }, select: { id: true } }); await prisma.pageContentCache.createMany({ data: created.map((p, i) => ({ pageId: p.id, plainText: `Body ${i}`, markdown: `# Page ${i}\n\nBody ${i}.`, html: '', outline: [], })), }); const res = await api() .get(`/api/v1/ponds/${big.id}/export/markdown`) .set('Cookie', ownerCookie) .buffer(true) .parse((r, cb) => { const chunks: Buffer[] = []; r.on('data', (c: Buffer) => chunks.push(c)); r.on('end', () => cb(null, Buffer.concat(chunks))); }) .expect(200); const mdEntries = Object.keys(zipEntries(res.body as Buffer)).filter((n) => n.endsWith('.md')); expect(mdEntries).toHaveLength(500); }); /** Same as {@link seedPage} but for an arbitrary pond. */ async function seedPageIn(pondId: string, title: string, markdown: string): Promise { return seedPage(pondId, title, markdown); } });