import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { Injectable, NotFoundException } from '@nestjs/common'; import { ConversionJobView, ExportFormat, PondFonts, fontSlug, PageClassification, classificationMarking, classificationRank, highestClassification, pondSettingsSchema, } from '@dorfteich/shared'; import { User } from '@prisma/client'; import archiver from 'archiver'; import type { Response } from 'express'; import { PinoLogger } from 'nestjs-pino'; import { AppConfig } from '../config/app-config.service'; import { FileStorageService } from '../files/file-storage.service'; import { PermissionService } from '../permissions/permission.service'; import { PluginFallbackRenderer } from '../plugins/plugin-fallback-renderer'; import { PluginsService } from '../plugins/plugins.service'; import { PrismaService } from '../prisma/prisma.service'; import { ReadTrailService, type ReadActor } from '../read-trail/read-trail.service'; import { markClassifiedMarkdown } from './classified-markdown'; import { ConversionJobService } from './conversion-job.service'; import { imageExtension, imageFileIds, markdownForDocument, markdownForZip, } from './export-markdown'; import { buildPdfHtml } from './pdf-html'; /** * Export (ADR 0009, issue #65): a whole pond as a ZIP of Markdown (one `.md` * per readable page, a `media/` directory, wikilinks as relative links), and a * single page to `.docx`/`.odt` via a conversion job. Both respect the * requester's read permissions — a pond export contains only the pages the * requester may read (permissions.md). */ @Injectable() export class ExportService { constructor( private readonly prisma: PrismaService, private readonly permissions: PermissionService, private readonly storage: FileStorageService, private readonly jobs: ConversionJobService, private readonly plugins: PluginsService, private readonly fallbacks: PluginFallbackRenderer, private readonly config: AppConfig, private readonly readTrail: ReadTrailService, private readonly logger: PinoLogger, ) { this.logger.setContext(ExportService.name); } /** * Stream a ZIP of the pond's readable pages as Markdown to `res`. Page * Markdown is appended as small strings; media is appended as read streams, so * memory stays bounded regardless of pond size (the 500-page AC). The guard * already checked the requester may see the pond; here we filter to the pages * they may actually read. */ async streamPondMarkdownZip( user: User, pondId: string, res: Response, read: ReadActor, ): Promise { const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } }); if (!pond) throw new NotFoundException(); const archive = archiver('zip', { zlib: { level: 9 } }); res.set('Content-Type', 'application/zip'); res.set('Content-Disposition', `attachment; filename="${pond.slug}.zip"`); res.set('X-Content-Type-Options', 'nosniff'); archive.on('error', (error) => { this.logger.error({ pondId, err: error.message }, 'pond export archive failed'); res.destroy(error); }); archive.pipe(res); await this.appendPondMarkdown(archive, user, pond, '', read); await archive.finalize(); } /** * Append one pond's readable pages (as Markdown) and their media to an open * archive, each entry under `prefix`. Shared by the pond ZIP above and the * account data export (#68), which nests several ponds under `ponds//` * — the read filter runs here, so neither caller can leak a page the * requester may not read. Page Markdown is appended as small strings; media * as read streams, keeping memory bounded regardless of pond size. */ async appendPondMarkdown( archive: archiver.Archiver, user: User, pond: { id: string; slug: string }, prefix: string, read: ReadActor, ): Promise { const pondId = pond.id; const pages = await this.prisma.page.findMany({ where: { pondId, deletedAt: null }, orderBy: { title: 'asc' }, include: { labels: { select: { labelId: true } }, contentCache: { select: { markdown: true } }, }, }); const readableIds = await this.permissions.filterPages( user, pondId, pages.map((page) => ({ id: page.id, labelIds: page.labels.map((l) => l.labelId) })), 'read', ); const readablePages = pages.filter((page) => readableIds.has(page.id)); const readableSlugs = new Set(readablePages.map((page) => page.slug)); // Read trail (issue #222): the ZIP is a bulk-egress channel — one event // per classified page it will contain, recorded BEFORE any classified // bytes enter the stream, so a failed write aborts the download while the // evidence is still complete (ADR 0023). for (const page of readablePages) { if (page.classification !== 'VS_NFD') continue; await this.readTrail.record({ ...read, pageId: page.id, pondId, channel: 'export', details: { format: 'markdown_zip' }, }); } // Every image referenced by a readable page — resolved to attachments that // still exist in this pond, so the media directory matches the rewrites. const referenced = new Set(); for (const page of readablePages) { for (const id of imageFileIds(page.contentCache?.markdown ?? '')) referenced.add(id); } const attachmentRows = referenced.size > 0 ? await this.prisma.attachment.findMany({ where: { id: { in: [...referenced] }, pondId }, select: { id: true, mimeType: true }, }) : []; // Only include media whose bytes are actually on disk — a row whose file is // missing (data drift) must not error the archive stream and crash the api. const present = await Promise.all( attachmentRows.map((a) => this.storage.exists(pond.id, a.id)), ); const attachments = attachmentRows.filter((_, i) => present[i]); const mediaNameById = new Map( attachments.map((a) => [a.id, `${a.id}.${imageExtension(a.mimeType)}`]), ); // Media inherits the highest classification among the readable pages that // reference it (fail-closed, ADR 0022 — a shared image is as classified // as its most classified use). const mediaClassification = new Map(); for (const page of readablePages) { const level = page.classification.toLowerCase() as PageClassification; for (const id of imageFileIds(page.contentCache?.markdown ?? '')) { const current = mediaClassification.get(id) ?? 'unclassified'; if (classificationRank(level) > classificationRank(current)) { mediaClassification.set(id, level); } } } const manifestFiles: { path: string; classification: PageClassification }[] = []; for (const page of readablePages) { const level = page.classification.toLowerCase() as PageClassification; // A classified page's file carries the level in YAML frontmatter and // the marking line at top and bottom (#210); unclassified files are // byte-identical to the pre-#210 export. const markdown = markClassifiedMarkdown( markdownForZip(page.contentCache?.markdown ?? '', readableSlugs, mediaNameById), level, ); // Page slugs are unique within a pond, so `.md` never collides. archive.append(markdown, { name: `${prefix}${page.slug}.md` }); manifestFiles.push({ path: `${prefix}${page.slug}.md`, classification: level }); } for (const attachment of attachments) { const mediaLevel = mediaClassification.get(attachment.id) ?? 'unclassified'; manifestFiles.push({ path: `${prefix}media/${mediaNameById.get(attachment.id)!}`, classification: mediaLevel, }); // Companion file for classified media (issue #212): the binary itself // cannot carry the marking, so a sibling text file states it — it // survives unpacking and copying, where the manifest may be dropped. const mediaMarking = classificationMarking(mediaLevel); if (mediaMarking) { archive.append(`${mediaMarking}\n`, { name: `${prefix}media/${mediaNameById.get(attachment.id)!}.classification.txt`, }); } } // The archive-level manifest (#210): every file with its level, and the // highest level contained stated once — the bulk-egress channel stays // machine-checkable even after the ZIP is unpacked and copied onward. archive.append( JSON.stringify( { classification: highestClassification(manifestFiles.map((f) => f.classification)), files: manifestFiles, }, null, 2, ), { name: `${prefix}manifest.json` }, ); for (const attachment of attachments) { const stream = this.storage.createReadStream(pond.id, attachment.id); // Defence in depth: a file removed between the existence check and the // read must not throw an unhandled stream error — let the archive skip it. stream.on('error', (error) => this.logger.warn( { pondId, fileId: attachment.id, err: error.message }, 'export: media read failed', ), ); archive.append(stream, { name: `${prefix}media/${mediaNameById.get(attachment.id)!}` }); } this.logger.info( { pondId, pages: readablePages.length, media: attachments.length, userId: user.id }, 'audit: pond exported as markdown zip', ); } /** * Enqueue a `markdown → pandoc → .docx/.odt` conversion for one page (issue * #65). Embedded images are inlined as `data:` URIs so the sidecar embeds * them; wikilinks become plain text (a standalone document has no targets). * The client polls `GET /jobs/:id` and downloads `GET /jobs/:id/result`. */ async enqueuePageExport( user: User, pageId: string, format: ExportFormat, read: ReadActor, ): Promise { if (format === 'pdf') return this.enqueuePdfExport(user, pageId, read); const page = await this.prisma.page.findFirst({ where: { id: pageId, deletedAt: null }, include: { contentCache: { select: { markdown: true } } }, }); if (!page) throw new NotFoundException(); // Read trail (issue #222): recorded at enqueue — the user's action; the // worker's later conversion is machinery, not a second read. await this.recordClassifiedExport(page, read, format); // Plugin blocks degrade to their fallback text and sections to quoted // blocks first (#79) — GFM knows neither construct, and pandoc would // otherwise emit the literal fences into the .docx/.odt. const markdown = await this.fallbacks.applyToMarkdown(page.contentCache?.markdown ?? ''); const dataUriById = await this.inlineImages(page.pondId, imageFileIds(markdown)); const document = markdownForDocument(markdown, dataUriById); // A classified page's export records its marking as a job option (#209): // the worker then hands pandoc the classified reference document whose // header/footer carry the marking on every page in Word/LibreOffice. const marking = classificationMarking(page.classification.toLowerCase() as PageClassification); const job = await this.jobs.enqueue({ ownerId: user.id, kind: `export_${format}`, from: 'gfm', to: format, input: Buffer.from(document, 'utf8'), standalone: true, ...(marking ? { options: { marking } } : {}), }); this.logger.info( { jobId: job.id, pageId, format, userId: user.id }, 'audit: page export enqueued', ); return this.jobs.viewOf(job); } /** * Enqueue a PDF export (issue #67). The self-contained export HTML — content * with images inlined, the pond's fonts inlined as base64 `@font-face`, print * CSS — is built here (permission already checked by the guard) and stored as * the job input; the worker sends it to Gotenberg (`html → pdf`). Building it * up front keeps the job a plain byte→byte render the worker can retry. */ /** One `export` event for a classified page leaving as a document (#222). */ private async recordClassifiedExport( page: { id: string; pondId: string; classification: string }, read: ReadActor, format: ExportFormat, ): Promise { if (page.classification !== 'VS_NFD') return; await this.readTrail.record({ ...read, pageId: page.id, pondId: page.pondId, channel: 'export', details: { format }, }); } private async enqueuePdfExport( user: User, pageId: string, read: ReadActor, ): Promise { const page = await this.prisma.page.findFirst({ where: { id: pageId, deletedAt: null }, include: { pond: { select: { name: true, settings: true } }, contentCache: { select: { html: true } }, }, }); if (!page) throw new NotFoundException(); await this.recordClassifiedExport(page, read, 'pdf'); const fonts = pondSettingsSchema.parse(page.pond.settings ?? {}).fonts; // Plugin blocks first become their best static form (#79: stored SVG // snapshot → manifest fallback → neutral marker), then images inline. const withFallbacks = await this.fallbacks.applyToHtml(page.contentCache?.html ?? ''); const bodyHtml = await this.inlineHtmlImages(page.pondId, withFallbacks); const html = buildPdfHtml({ title: page.title, pondName: page.pond.name, bodyHtml, fonts, fontFaceCss: await this.fontFaceCss(fonts), // Styled sections keep their look in the PDF (#75); a pond without // active style plugins contributes an empty string. sectionStyleCss: await this.plugins.sectionStyleCssForPond(page.pondId), }); // The VS-NfD marking (issue #208, ADR 0022) travels as a job option so // the worker can hand it to Gotenberg's per-page header/footer templates // — an unclassified page carries none and renders exactly as before. const marking = classificationMarking(page.classification.toLowerCase() as PageClassification); const job = await this.jobs.enqueue({ ownerId: user.id, kind: 'export_pdf', from: 'html', to: 'pdf', input: Buffer.from(html, 'utf8'), standalone: true, ...(marking ? { options: { marking } } : {}), }); this.logger.info( { jobId: job.id, pageId, format: 'pdf', userId: user.id }, 'audit: page export enqueued', ); return this.jobs.viewOf(job); } /** Replace each `` with an inlined `data:` URI so the * PDF render (isolated from the api) needs no network to fetch media. */ private async inlineHtmlImages(pondId: string, html: string): Promise { const ids = [...html.matchAll(/data-file-id="([A-Za-z0-9-]+)"/g)].map((m) => m[1]!); if (ids.length === 0) return html; const dataUriById = await this.inlineImages(pondId, ids); return html.replace(/data-file-id="([A-Za-z0-9-]+)"/g, (whole, id: string) => { const uri = dataUriById.get(id); return uri ? `src="${uri}" ${whole}` : whole; }); } /** Base64 `@font-face` rules for the pond's three fonts, read from the * catalog baked into the image (ADR 0016). A font file that is absent (a * native dev run without `FONTS_DIR` populated) is skipped — the render falls * back to the system stack rather than failing. */ private async fontFaceCss(fonts: PondFonts): Promise { const slots = [fonts.heading, fonts.body, fonts.mono]; // Dedup identical family+weight so a doc that repeats a font embeds it once. const seen = new Set(); const faces: string[] = []; for (const slot of slots) { const key = `${slot.family}:${slot.weight}`; if (seen.has(key)) continue; seen.add(key); const slug = fontSlug(slot.family); const file = join(this.config.env.FONTS_DIR, slug, `${slug}-${slot.weight}.woff2`); try { const bytes = await readFile(file); faces.push( `@font-face { font-family: '${slot.family}'; font-style: normal; font-weight: ${slot.weight};` + ` src: url('data:font/woff2;base64,${bytes.toString('base64')}') format('woff2'); }`, ); } catch { this.logger.warn({ font: key }, 'pdf export: catalog font file missing, using fallback'); } } return faces.join('\n'); } /** Read each referenced attachment's bytes into a `data:` URI (a page has few * images, so holding them briefly is fine — unlike the streamed pond ZIP). */ private async inlineImages(pondId: string, ids: string[]): Promise> { const dataUriById = new Map(); if (ids.length === 0) return dataUriById; const attachments = await this.prisma.attachment.findMany({ where: { id: { in: ids }, pondId }, select: { id: true, mimeType: true }, }); for (const attachment of attachments) { try { const bytes = await readStream(this.storage.createReadStream(pondId, attachment.id)); dataUriById.set( attachment.id, `data:${attachment.mimeType};base64,${bytes.toString('base64')}`, ); } catch (error) { // A missing file (data drift) drops the image rather than failing the // whole export — the rest of the page still converts. this.logger.warn( { pondId, fileId: attachment.id, err: (error as Error).message }, 'export: inline image read failed', ); } } return dataUriById; } } function readStream(stream: NodeJS.ReadableStream): Promise { return new Promise((resolve, reject) => { const chunks: Buffer[] = []; stream.on('data', (chunk: Buffer) => chunks.push(chunk)); stream.on('end', () => resolve(Buffer.concat(chunks))); stream.on('error', reject); }); }