dorfteich/apps/api/src/import-export/export.service.ts

import { readFile } from 'node:fs/promises';
import { join } from 'node:path';

import { Injectable, NotFoundException } from '@nestjs/common';
import {
  ConversionJobView,
  ExportFormat,
  PondFonts,
  fontSlug,
  PageClassification,
  classificationMarking,
  classificationRank,
  highestClassification,
  pondSettingsSchema,
} from '@dorfteich/shared';
import { User } from '@prisma/client';
import archiver from 'archiver';
import type { Response } from 'express';
import { PinoLogger } from 'nestjs-pino';

import { AppConfig } from '../config/app-config.service';
import { FileStorageService } from '../files/file-storage.service';
import { PermissionService } from '../permissions/permission.service';
import { PluginFallbackRenderer } from '../plugins/plugin-fallback-renderer';
import { PluginsService } from '../plugins/plugins.service';
import { PrismaService } from '../prisma/prisma.service';
import { ReadTrailService, type ReadActor } from '../read-trail/read-trail.service';

import { markClassifiedMarkdown } from './classified-markdown';
import { ConversionJobService } from './conversion-job.service';
import {
  imageExtension,
  imageFileIds,
  markdownForDocument,
  markdownForZip,
} from './export-markdown';
import { buildPdfHtml } from './pdf-html';

/**
 * Export (ADR 0009, issue #65): a whole pond as a ZIP of Markdown (one `.md`
 * per readable page, a `media/` directory, wikilinks as relative links), and a
 * single page to `.docx`/`.odt` via a conversion job. Both respect the
 * requester's read permissions — a pond export contains only the pages the
 * requester may read (permissions.md).
 */
@Injectable()
export class ExportService {
  constructor(
    private readonly prisma: PrismaService,
    private readonly permissions: PermissionService,
    private readonly storage: FileStorageService,
    private readonly jobs: ConversionJobService,
    private readonly plugins: PluginsService,
    private readonly fallbacks: PluginFallbackRenderer,
    private readonly config: AppConfig,
    private readonly readTrail: ReadTrailService,
    private readonly logger: PinoLogger,
  ) {
    this.logger.setContext(ExportService.name);
  }

  /**
   * Stream a ZIP of the pond's readable pages as Markdown to `res`. Page
   * Markdown is appended as small strings; media is appended as read streams, so
   * memory stays bounded regardless of pond size (the 500-page AC). The guard
   * already checked the requester may see the pond; here we filter to the pages
   * they may actually read.
   */
  async streamPondMarkdownZip(
    user: User,
    pondId: string,
    res: Response,
    read: ReadActor,
  ): Promise<void> {
    const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } });
    if (!pond) throw new NotFoundException();

    const archive = archiver('zip', { zlib: { level: 9 } });
    res.set('Content-Type', 'application/zip');
    res.set('Content-Disposition', `attachment; filename="${pond.slug}.zip"`);
    res.set('X-Content-Type-Options', 'nosniff');
    archive.on('error', (error) => {
      this.logger.error({ pondId, err: error.message }, 'pond export archive failed');
      res.destroy(error);
    });
    archive.pipe(res);
    await this.appendPondMarkdown(archive, user, pond, '', read);
    await archive.finalize();
  }

  /**
   * Append one pond's readable pages (as Markdown) and their media to an open
   * archive, each entry under `prefix`. Shared by the pond ZIP above and the
   * account data export (#68), which nests several ponds under `ponds/<slug>/`
   * — the read filter runs here, so neither caller can leak a page the
   * requester may not read. Page Markdown is appended as small strings; media
   * as read streams, keeping memory bounded regardless of pond size.
   */
  async appendPondMarkdown(
    archive: archiver.Archiver,
    user: User,
    pond: { id: string; slug: string },
    prefix: string,
    read: ReadActor,
  ): Promise<void> {
    const pondId = pond.id;
    const pages = await this.prisma.page.findMany({
      where: { pondId, deletedAt: null },
      orderBy: { title: 'asc' },
      include: {
        labels: { select: { labelId: true } },
        contentCache: { select: { markdown: true } },
      },
    });
    const readableIds = await this.permissions.filterPages(
      user,
      pondId,
      pages.map((page) => ({ id: page.id, labelIds: page.labels.map((l) => l.labelId) })),
      'read',
    );
    const readablePages = pages.filter((page) => readableIds.has(page.id));
    const readableSlugs = new Set(readablePages.map((page) => page.slug));

    // Read trail (issue #222): the ZIP is a bulk-egress channel — one event
    // per classified page it will contain, recorded BEFORE any classified
    // bytes enter the stream, so a failed write aborts the download while the
    // evidence is still complete (ADR 0023).
    for (const page of readablePages) {
      if (page.classification !== 'VS_NFD') continue;
      await this.readTrail.record({
        ...read,
        pageId: page.id,
        pondId,
        channel: 'export',
        details: { format: 'markdown_zip' },
      });
    }

    // Every image referenced by a readable page — resolved to attachments that
    // still exist in this pond, so the media directory matches the rewrites.
    const referenced = new Set<string>();
    for (const page of readablePages) {
      for (const id of imageFileIds(page.contentCache?.markdown ?? '')) referenced.add(id);
    }
    const attachmentRows =
      referenced.size > 0
        ? await this.prisma.attachment.findMany({
            where: { id: { in: [...referenced] }, pondId },
            select: { id: true, mimeType: true },
          })
        : [];
    // Only include media whose bytes are actually on disk — a row whose file is
    // missing (data drift) must not error the archive stream and crash the api.
    const present = await Promise.all(
      attachmentRows.map((a) => this.storage.exists(pond.id, a.id)),
    );
    const attachments = attachmentRows.filter((_, i) => present[i]);
    const mediaNameById = new Map(
      attachments.map((a) => [a.id, `${a.id}.${imageExtension(a.mimeType)}`]),
    );

    // Media inherits the highest classification among the readable pages that
    // reference it (fail-closed, ADR 0022 — a shared image is as classified
    // as its most classified use).
    const mediaClassification = new Map<string, PageClassification>();
    for (const page of readablePages) {
      const level = page.classification.toLowerCase() as PageClassification;
      for (const id of imageFileIds(page.contentCache?.markdown ?? '')) {
        const current = mediaClassification.get(id) ?? 'unclassified';
        if (classificationRank(level) > classificationRank(current)) {
          mediaClassification.set(id, level);
        }
      }
    }

    const manifestFiles: { path: string; classification: PageClassification }[] = [];
    for (const page of readablePages) {
      const level = page.classification.toLowerCase() as PageClassification;
      // A classified page's file carries the level in YAML frontmatter and
      // the marking line at top and bottom (#210); unclassified files are
      // byte-identical to the pre-#210 export.
      const markdown = markClassifiedMarkdown(
        markdownForZip(page.contentCache?.markdown ?? '', readableSlugs, mediaNameById),
        level,
      );
      // Page slugs are unique within a pond, so `<slug>.md` never collides.
      archive.append(markdown, { name: `${prefix}${page.slug}.md` });
      manifestFiles.push({ path: `${prefix}${page.slug}.md`, classification: level });
    }
    for (const attachment of attachments) {
      const mediaLevel = mediaClassification.get(attachment.id) ?? 'unclassified';
      manifestFiles.push({
        path: `${prefix}media/${mediaNameById.get(attachment.id)!}`,
        classification: mediaLevel,
      });
      // Companion file for classified media (issue #212): the binary itself
      // cannot carry the marking, so a sibling text file states it — it
      // survives unpacking and copying, where the manifest may be dropped.
      const mediaMarking = classificationMarking(mediaLevel);
      if (mediaMarking) {
        archive.append(`${mediaMarking}\n`, {
          name: `${prefix}media/${mediaNameById.get(attachment.id)!}.classification.txt`,
        });
      }
    }
    // The archive-level manifest (#210): every file with its level, and the
    // highest level contained stated once — the bulk-egress channel stays
    // machine-checkable even after the ZIP is unpacked and copied onward.
    archive.append(
      JSON.stringify(
        {
          classification: highestClassification(manifestFiles.map((f) => f.classification)),
          files: manifestFiles,
        },
        null,
        2,
      ),
      { name: `${prefix}manifest.json` },
    );
    for (const attachment of attachments) {
      const stream = this.storage.createReadStream(pond.id, attachment.id);
      // Defence in depth: a file removed between the existence check and the
      // read must not throw an unhandled stream error — let the archive skip it.
      stream.on('error', (error) =>
        this.logger.warn(
          { pondId, fileId: attachment.id, err: error.message },
          'export: media read failed',
        ),
      );
      archive.append(stream, { name: `${prefix}media/${mediaNameById.get(attachment.id)!}` });
    }
    this.logger.info(
      { pondId, pages: readablePages.length, media: attachments.length, userId: user.id },
      'audit: pond exported as markdown zip',
    );
  }

  /**
   * Enqueue a `markdown → pandoc → .docx/.odt` conversion for one page (issue
   * #65). Embedded images are inlined as `data:` URIs so the sidecar embeds
   * them; wikilinks become plain text (a standalone document has no targets).
   * The client polls `GET /jobs/:id` and downloads `GET /jobs/:id/result`.
   */
  async enqueuePageExport(
    user: User,
    pageId: string,
    format: ExportFormat,
    read: ReadActor,
  ): Promise<ConversionJobView> {
    if (format === 'pdf') return this.enqueuePdfExport(user, pageId, read);

    const page = await this.prisma.page.findFirst({
      where: { id: pageId, deletedAt: null },
      include: { contentCache: { select: { markdown: true } } },
    });
    if (!page) throw new NotFoundException();
    // Read trail (issue #222): recorded at enqueue — the user's action; the
    // worker's later conversion is machinery, not a second read.
    await this.recordClassifiedExport(page, read, format);

    // Plugin blocks degrade to their fallback text and sections to quoted
    // blocks first (#79) — GFM knows neither construct, and pandoc would
    // otherwise emit the literal fences into the .docx/.odt.
    const markdown = await this.fallbacks.applyToMarkdown(page.contentCache?.markdown ?? '');
    const dataUriById = await this.inlineImages(page.pondId, imageFileIds(markdown));
    const document = markdownForDocument(markdown, dataUriById);

    // A classified page's export records its marking as a job option (#209):
    // the worker then hands pandoc the classified reference document whose
    // header/footer carry the marking on every page in Word/LibreOffice.
    const marking = classificationMarking(page.classification.toLowerCase() as PageClassification);
    const job = await this.jobs.enqueue({
      ownerId: user.id,
      kind: `export_${format}`,
      from: 'gfm',
      to: format,
      input: Buffer.from(document, 'utf8'),
      standalone: true,
      ...(marking ? { options: { marking } } : {}),
    });
    this.logger.info(
      { jobId: job.id, pageId, format, userId: user.id },
      'audit: page export enqueued',
    );
    return this.jobs.viewOf(job);
  }

  /**
   * Enqueue a PDF export (issue #67). The self-contained export HTML — content
   * with images inlined, the pond's fonts inlined as base64 `@font-face`, print
   * CSS — is built here (permission already checked by the guard) and stored as
   * the job input; the worker sends it to Gotenberg (`html → pdf`). Building it
   * up front keeps the job a plain byte→byte render the worker can retry.
   */
  /** One `export` event for a classified page leaving as a document (#222). */
  private async recordClassifiedExport(
    page: { id: string; pondId: string; classification: string },
    read: ReadActor,
    format: ExportFormat,
  ): Promise<void> {
    if (page.classification !== 'VS_NFD') return;
    await this.readTrail.record({
      ...read,
      pageId: page.id,
      pondId: page.pondId,
      channel: 'export',
      details: { format },
    });
  }

  private async enqueuePdfExport(
    user: User,
    pageId: string,
    read: ReadActor,
  ): Promise<ConversionJobView> {
    const page = await this.prisma.page.findFirst({
      where: { id: pageId, deletedAt: null },
      include: {
        pond: { select: { name: true, settings: true } },
        contentCache: { select: { html: true } },
      },
    });
    if (!page) throw new NotFoundException();
    await this.recordClassifiedExport(page, read, 'pdf');

    const fonts = pondSettingsSchema.parse(page.pond.settings ?? {}).fonts;
    // Plugin blocks first become their best static form (#79: stored SVG
    // snapshot → manifest fallback → neutral marker), then images inline.
    const withFallbacks = await this.fallbacks.applyToHtml(page.contentCache?.html ?? '');
    const bodyHtml = await this.inlineHtmlImages(page.pondId, withFallbacks);
    const html = buildPdfHtml({
      title: page.title,
      pondName: page.pond.name,
      bodyHtml,
      fonts,
      fontFaceCss: await this.fontFaceCss(fonts),
      // Styled sections keep their look in the PDF (#75); a pond without
      // active style plugins contributes an empty string.
      sectionStyleCss: await this.plugins.sectionStyleCssForPond(page.pondId),
    });

    // The VS-NfD marking (issue #208, ADR 0022) travels as a job option so
    // the worker can hand it to Gotenberg's per-page header/footer templates
    // — an unclassified page carries none and renders exactly as before.
    const marking = classificationMarking(page.classification.toLowerCase() as PageClassification);
    const job = await this.jobs.enqueue({
      ownerId: user.id,
      kind: 'export_pdf',
      from: 'html',
      to: 'pdf',
      input: Buffer.from(html, 'utf8'),
      standalone: true,
      ...(marking ? { options: { marking } } : {}),
    });
    this.logger.info(
      { jobId: job.id, pageId, format: 'pdf', userId: user.id },
      'audit: page export enqueued',
    );
    return this.jobs.viewOf(job);
  }

  /** Replace each `<img data-file-id="X">` with an inlined `data:` URI so the
   * PDF render (isolated from the api) needs no network to fetch media. */
  private async inlineHtmlImages(pondId: string, html: string): Promise<string> {
    const ids = [...html.matchAll(/data-file-id="([A-Za-z0-9-]+)"/g)].map((m) => m[1]!);
    if (ids.length === 0) return html;
    const dataUriById = await this.inlineImages(pondId, ids);
    return html.replace(/data-file-id="([A-Za-z0-9-]+)"/g, (whole, id: string) => {
      const uri = dataUriById.get(id);
      return uri ? `src="${uri}" ${whole}` : whole;
    });
  }

  /** Base64 `@font-face` rules for the pond's three fonts, read from the
   * catalog baked into the image (ADR 0016). A font file that is absent (a
   * native dev run without `FONTS_DIR` populated) is skipped — the render falls
   * back to the system stack rather than failing. */
  private async fontFaceCss(fonts: PondFonts): Promise<string> {
    const slots = [fonts.heading, fonts.body, fonts.mono];
    // Dedup identical family+weight so a doc that repeats a font embeds it once.
    const seen = new Set<string>();
    const faces: string[] = [];
    for (const slot of slots) {
      const key = `${slot.family}:${slot.weight}`;
      if (seen.has(key)) continue;
      seen.add(key);
      const slug = fontSlug(slot.family);
      const file = join(this.config.env.FONTS_DIR, slug, `${slug}-${slot.weight}.woff2`);
      try {
        const bytes = await readFile(file);
        faces.push(
          `@font-face { font-family: '${slot.family}'; font-style: normal; font-weight: ${slot.weight};` +
            ` src: url('data:font/woff2;base64,${bytes.toString('base64')}') format('woff2'); }`,
        );
      } catch {
        this.logger.warn({ font: key }, 'pdf export: catalog font file missing, using fallback');
      }
    }
    return faces.join('\n');
  }

  /** Read each referenced attachment's bytes into a `data:` URI (a page has few
   * images, so holding them briefly is fine — unlike the streamed pond ZIP). */
  private async inlineImages(pondId: string, ids: string[]): Promise<Map<string, string>> {
    const dataUriById = new Map<string, string>();
    if (ids.length === 0) return dataUriById;
    const attachments = await this.prisma.attachment.findMany({
      where: { id: { in: ids }, pondId },
      select: { id: true, mimeType: true },
    });
    for (const attachment of attachments) {
      try {
        const bytes = await readStream(this.storage.createReadStream(pondId, attachment.id));
        dataUriById.set(
          attachment.id,
          `data:${attachment.mimeType};base64,${bytes.toString('base64')}`,
        );
      } catch (error) {
        // A missing file (data drift) drops the image rather than failing the
        // whole export — the rest of the page still converts.
        this.logger.warn(
          { pondId, fileId: attachment.id, err: (error as Error).message },
          'export: inline image read failed',
        );
      }
    }
    return dataUriById;
  }
}

function readStream(stream: NodeJS.ReadableStream): Promise<Buffer> {
  return new Promise((resolve, reject) => {
    const chunks: Buffer[] = [];
    stream.on('data', (chunk: Buffer) => chunks.push(chunk));
    stream.on('end', () => resolve(Buffer.concat(chunks)));
    stream.on('error', reject);
  });
}