dorfteich/apps/api/src/import-export/export.service.db.test.ts

import { readFileSync } from 'node:fs';
import { join } from 'node:path';

import { INestApplication } from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import { unzipSync, zipSync } from 'fflate';
import request from 'supertest';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

import { AuthTokensService } from '../auth/auth-tokens.service';
import { FileStorageService } from '../files/file-storage.service';
import { FilesService } from '../files/files.service';
import { PluginsService } from '../plugins/plugins.service';
import { createTestApp, sessionCookieOf } from '../testing/test-app';
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
import { UsersService } from '../users/users.service';

import { ConversionWorker } from './conversion-worker.service';
import { GotenbergRenderer, RenderError } from './gotenberg.renderer';
import { ConversionRequest, ConversionResult, PandocConverter } from './pandoc.converter';

/** Export (issues #65/#67): pond ZIP of Markdown, per-page docx/odt (pandoc),
 * and per-page PDF (Gotenberg). The office/PDF paths use injected fakes that
 * record the input they are handed, so image-inlining / font-inlining is checked
 * without a live sidecar. */

const enc = (text: string) => new TextEncoder().encode(text);

const PNG_BASE64 =
  'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';

class RecordingConverter extends PandocConverter {
  lastInput = '';
  convert(request: ConversionRequest): Promise<ConversionResult> {
    this.lastInput = request.input.toString('utf8');
    return Promise.resolve({ output: Buffer.from('OFFICE-BYTES'), mimeType: 'application/x-test' });
  }
  reachable(): Promise<boolean> {
    return Promise.resolve(true);
  }
}

class RecordingRenderer extends GotenbergRenderer {
  lastHtml = '';
  failWith: RenderError | null = null;
  renderHtmlToPdf(html: string): Promise<Buffer> {
    this.lastHtml = html;
    if (this.failWith) return Promise.reject(this.failWith);
    return Promise.resolve(Buffer.from('%PDF-1.7 fake'));
  }
  reachable(): Promise<boolean> {
    return Promise.resolve(true);
  }
}

/** Filenames of every entry in a ZIP buffer. */
function zipEntries(buffer: Buffer): Record<string, Uint8Array> {
  return unzipSync(new Uint8Array(buffer));
}

describe.skipIf(!hasTestDb)('export (e2e, issue #65)', () => {
  let app: INestApplication;
  let prisma: PrismaClient;
  let worker: ConversionWorker;
  let files: FilesService;
  let fake: RecordingConverter;
  let renderer: RecordingRenderer;
  const suffix = uniqueSuffix();
  const password = 'exportiere meine sachen 1';

  const owner = { username: `ella-export-${suffix}`, displayName: `Ella Export ${suffix}` };
  let ownerId: string;
  let ownerCookie: string;
  let personalPondId: string;

  const api = () => request(app.getHttpServer());

  async function login(username: string): Promise<string> {
    const res = await api()
      .post('/api/v1/auth/login')
      .send({ usernameOrEmail: username, password })
      .expect(200);
    return sessionCookieOf(res);
  }

  /** Create a page with a crafted content-cache Markdown (the exporter reads the
   * cache, not the Yjs state), returning its slug. */
  async function seedPage(
    pondId: string,
    title: string,
    markdown: string,
    html = '',
  ): Promise<string> {
    const slug = title.toLowerCase().replace(/[^a-z0-9]+/g, '-');
    await prisma.page.create({
      data: {
        pondId,
        title,
        slug,
        ydocState: new Uint8Array(),
        sortKey: title,
        createdBy: ownerId,
        contentCache: { create: { plainText: markdown, markdown, html, outline: [] } },
      },
    });
    return slug;
  }

  beforeAll(async () => {
    prisma = createTestPrisma();
    await prisma.rateLimit.deleteMany({});
    fake = new RecordingConverter();
    renderer = new RecordingRenderer();
    app = await createTestApp((builder) =>
      builder
        .overrideProvider(PandocConverter)
        .useValue(fake)
        .overrideProvider(GotenbergRenderer)
        .useValue(renderer),
    );
    worker = app.get(ConversionWorker);
    files = app.get(FilesService);

    const users = app.get(UsersService);
    const tokens = app.get(AuthTokensService);
    const ownerUser = await users.createUser({
      username: owner.username,
      email: `${owner.username}@example.org`,
      displayName: owner.displayName,
      password,
      locale: 'en',
    });
    ownerId = ownerUser.id;
    const verify = await tokens.issue(ownerUser.id, 'EMAIL_VERIFICATION', 600);
    await api().post('/api/v1/auth/verify-email').send({ token: verify }).expect(204);
    ownerCookie = await login(owner.username);
    personalPondId = (await prisma.pond.findFirstOrThrow({ where: { ownerId, type: 'PERSONAL' } }))
      .id;
  });

  afterAll(async () => {
    await prisma.conversionJob.deleteMany({ where: { owner: { username: { contains: suffix } } } });
    const where = { pond: { owner: { username: { contains: suffix } } } };
    await prisma.attachment.deleteMany({ where });
    await prisma.pageLabel.deleteMany({ where: { page: where } });
    await prisma.label.deleteMany({ where });
    await prisma.page.deleteMany({ where });
    await prisma.roleGrant.deleteMany({ where });
    await prisma.pond.deleteMany({ where: { owner: { username: { contains: suffix } } } });
    await prisma.user.deleteMany({ where: { username: { contains: suffix } } });
    await prisma.$disconnect();
    await app.close();
  });

  it('exports a pond ZIP with one .md per page, a media dir, and relative links', async () => {
    // A real stored image so the media stream has bytes.
    const image = await files.upload({ id: ownerId } as never, personalPondId, {
      buffer: Buffer.from(PNG_BASE64, 'base64'),
      size: 70,
      originalname: 'dot.png',
    });
    await seedPage(personalPondId, 'Zip Intro', 'Welcome. See [[zip-target]].');
    await seedPage(personalPondId, 'Zip Target', `# Zip Target\n\nAn image: ![dot](${image.id})`);

    const res = await api()
      .get(`/api/v1/ponds/${personalPondId}/export/markdown`)
      .set('Cookie', ownerCookie)
      .buffer(true)
      .parse((r, cb) => {
        const chunks: Buffer[] = [];
        r.on('data', (c: Buffer) => chunks.push(c));
        r.on('end', () => cb(null, Buffer.concat(chunks)));
      })
      .expect(200);

    expect(res.headers['content-type']).toContain('application/zip');
    expect(res.headers['content-disposition']).toContain('attachment');
    const entries = zipEntries(res.body as Buffer);
    const names = Object.keys(entries);
    expect(names).toContain('zip-intro.md');
    expect(names).toContain('zip-target.md');
    expect(names).toContain(`media/${image.id}.png`);

    const intro = Buffer.from(entries['zip-intro.md']!).toString('utf8');
    // The wikilink became a relative link to the readable target's .md.
    expect(intro).toContain('[zip-target](zip-target.md)');
    const target = Buffer.from(entries['zip-target.md']!).toString('utf8');
    // The image src became a working relative path into media/.
    expect(target).toContain(`![dot](media/${image.id}.png)`);
  });

  it('skips an attachment whose bytes are missing on disk instead of crashing', async () => {
    // An attachment row with no file (data drift): upload then remove the bytes.
    const image = await files.upload({ id: ownerId } as never, personalPondId, {
      buffer: Buffer.from(PNG_BASE64, 'base64'),
      size: 70,
      originalname: 'ghost.png',
    });
    await app.get(FileStorageService).delete(personalPondId, image.id);
    await seedPage(personalPondId, 'Ghost Image', `# Ghost\n\n![gone](${image.id})`);

    const res = await api()
      .get(`/api/v1/ponds/${personalPondId}/export/markdown`)
      .set('Cookie', ownerCookie)
      .buffer(true)
      .parse((r, cb) => {
        const chunks: Buffer[] = [];
        r.on('data', (c: Buffer) => chunks.push(c));
        r.on('end', () => cb(null, Buffer.concat(chunks)));
      })
      .expect(200);

    const names = Object.keys(zipEntries(res.body as Buffer));
    // The page is still exported; the missing media file is simply not included.
    expect(names).toContain('ghost-image.md');
    expect(names).not.toContain(`media/${image.id}.png`);
  });

  it('omits pages a label-restricted reader cannot read from their pond ZIP', async () => {
    // A shared pond with a public page and a `secret`-labelled page; a reader is
    // allowed pond-wide but denied on the secret label.
    const shared = await prisma.pond.create({
      data: {
        slug: `shared-export-${suffix}`,
        name: 'Shared Export',
        type: 'SHARED',
        ownerId,
        usage: { create: {} },
      },
    });
    const reader = await app.get(UsersService).createUser({
      username: `rudy-reader-${suffix}`,
      email: `rudy-reader-${suffix}@example.org`,
      displayName: `Rudy Reader ${suffix}`,
      password,
      locale: 'en',
    });
    await app.get(UsersService).markEmailVerified(reader.id);
    const secret = await prisma.label.create({
      data: { pondId: shared.id, name: 'secret', color: '#334455' },
    });
    const publicSlug = await seedPageIn(shared.id, 'Public Page', 'Everyone may read this.');
    const secretSlug = await seedPageIn(shared.id, 'Secret Page', 'Classified.');
    const secretPage = await prisma.page.findFirstOrThrow({
      where: { pondId: shared.id, slug: secretSlug },
    });
    await prisma.pageLabel.create({ data: { pageId: secretPage.id, labelId: secret.id } });
    // Reader: pond-wide reader ALLOW + secret-label DENY.
    await prisma.roleGrant.createMany({
      data: [
        {
          pondId: shared.id,
          subjectType: 'USER',
          subjectId: reader.id,
          role: 'READER',
          scopeType: 'POND',
          effect: 'ALLOW',
          createdBy: ownerId,
        },
        {
          pondId: shared.id,
          subjectType: 'USER',
          subjectId: reader.id,
          role: 'READER',
          scopeType: 'LABEL',
          scopeId: secret.id,
          effect: 'DENY',
          createdBy: ownerId,
        },
      ],
    });

    const readerCookie = await login(`rudy-reader-${suffix}`);
    const res = await api()
      .get(`/api/v1/ponds/${shared.id}/export/markdown`)
      .set('Cookie', readerCookie)
      .buffer(true)
      .parse((r, cb) => {
        const chunks: Buffer[] = [];
        r.on('data', (c: Buffer) => chunks.push(c));
        r.on('end', () => cb(null, Buffer.concat(chunks)));
      })
      .expect(200);

    const names = Object.keys(zipEntries(res.body as Buffer));
    expect(names).toContain(`${publicSlug}.md`);
    expect(names).not.toContain(`${secretSlug}.md`);
  });

  it('exports a page to .docx: images inlined as data URIs, wikilinks flattened', async () => {
    const image = await files.upload({ id: ownerId } as never, personalPondId, {
      buffer: Buffer.from(PNG_BASE64, 'base64'),
      size: 70,
      originalname: 'inl.png',
    });
    const slug = await seedPage(
      personalPondId,
      'Docx Source',
      `# Docx Source\n\n![pic](${image.id}) and [[other|Other Page]].`,
    );
    const page = await prisma.page.findFirstOrThrow({
      where: { pondId: personalPondId, slug },
    });

    const enqueued = await api()
      .post(`/api/v1/pages/${page.id}/export`)
      .set('Cookie', ownerCookie)
      .send({ format: 'docx' })
      .expect(201);
    expect(enqueued.body.status).toBe('pending');
    expect(enqueued.body.kind).toBe('export_docx');

    await worker.drain();

    // The Markdown handed to pandoc inlined the image and flattened the wikilink.
    expect(fake.lastInput).toContain('data:image/png;base64,');
    expect(fake.lastInput).toContain('Other Page');
    expect(fake.lastInput).not.toContain('[[');

    const done = await api()
      .get(`/api/v1/jobs/${enqueued.body.id}`)
      .set('Cookie', ownerCookie)
      .expect(200);
    expect(done.body.status).toBe('succeeded');
    const result = await api()
      .get(`/api/v1/jobs/${enqueued.body.id}/result`)
      .set('Cookie', ownerCookie)
      .expect(200);
    expect(result.text).toBe('OFFICE-BYTES');
  });

  it('exports a page to PDF: content + image inlined, font CSS, via Gotenberg', async () => {
    const image = await files.upload({ id: ownerId } as never, personalPondId, {
      buffer: Buffer.from(PNG_BASE64, 'base64'),
      size: 70,
      originalname: 'pdf.png',
    });
    const slug = await seedPage(
      personalPondId,
      'Pdf Source',
      '# Pdf Source\n\nbody',
      `<p>A PDF body paragraph.</p><img data-file-id="${image.id}" alt="pic">`,
    );
    const page = await prisma.page.findFirstOrThrow({
      where: { pondId: personalPondId, slug },
    });

    const enqueued = await api()
      .post(`/api/v1/pages/${page.id}/export`)
      .set('Cookie', ownerCookie)
      .send({ format: 'pdf' })
      .expect(201);
    expect(enqueued.body.kind).toBe('export_pdf');

    await worker.drain();

    // The HTML handed to Gotenberg carries the title, the pond's font stack as a
    // CSS variable, and the image inlined as a data URI (no network needed).
    expect(renderer.lastHtml).toContain('Pdf Source');
    expect(renderer.lastHtml).toContain("--font-body: 'Roboto'");
    expect(renderer.lastHtml).toContain('src="data:image/png;base64,');

    const done = await api()
      .get(`/api/v1/jobs/${enqueued.body.id}`)
      .set('Cookie', ownerCookie)
      .expect(200);
    expect(done.body.status).toBe('succeeded');
    const result = await api()
      .get(`/api/v1/jobs/${enqueued.body.id}/result`)
      .set('Cookie', ownerCookie)
      .buffer(true)
      .parse((r, cb) => {
        const chunks: Buffer[] = [];
        r.on('data', (c: Buffer) => chunks.push(c));
        r.on('end', () => cb(null, Buffer.concat(chunks)));
      })
      .expect(200);
    expect(result.headers['content-type']).toContain('application/pdf');
    expect((result.body as Buffer).toString('utf8')).toContain('%PDF');
  });

  it('inlines active section-style plugin CSS into the PDF html (#75)', async () => {
    // Install the real reference plugin and make it active everywhere, so the
    // export path exercises the same package users get.
    const pluginDir = join(__dirname, '../../../../packages/plugins/section-styles-basic');
    const plugins = app.get(PluginsService);
    // A `required` plugin refuses uninstall, and the local dev DB may carry
    // state from an aborted earlier run — always demote + remove, both ways.
    async function removeIfInstalled(): Promise<void> {
      await plugins.setMode('section-styles-basic', 'disabled').catch(() => undefined);
      await plugins.uninstall('section-styles-basic').catch(() => undefined);
    }
    await removeIfInstalled();
    await plugins.install(
      Buffer.from(
        zipSync({
          'manifest.json': new Uint8Array(readFileSync(join(pluginDir, 'manifest.json'))),
          'styles.css': new Uint8Array(readFileSync(join(pluginDir, 'styles.css'))),
        }),
      ),
    );
    try {
      await plugins.setMode('section-styles-basic', 'required');

      const slug = await seedPage(
        personalPondId,
        'Sectioned Pdf',
        'body',
        '<div class="dt-section dt-style-section-styles-basic-callout"><p>boxed</p></div>',
      );
      const page = await prisma.page.findFirstOrThrow({
        where: { pondId: personalPondId, slug },
      });

      await api()
        .post(`/api/v1/pages/${page.id}/export`)
        .set('Cookie', ownerCookie)
        .send({ format: 'pdf' })
        .expect(201);
      await worker.drain();

      // The Gotenberg HTML carries both the section markup and the plugin's
      // scoped CSS, so the box renders in the (network-isolated) PDF.
      expect(renderer.lastHtml).toContain('dt-style-section-styles-basic-callout"');
      expect(renderer.lastHtml).toContain('.dt-style-section-styles-basic-callout {');
      expect(renderer.lastHtml).toContain('/* section-styles-basic@');
    } finally {
      await removeIfInstalled();
    }
  });

  it('degrades plugin blocks in exports: snapshot SVG, manifest text, tombstone (#79)', async () => {
    const plugins = app.get(PluginsService);
    async function removeIfInstalled(id: string): Promise<void> {
      await plugins.setMode(id, 'disabled').catch(() => undefined);
      await plugins.uninstall(id).catch(() => undefined);
    }
    // A block plugin with a text fallback; a second one that gets uninstalled.
    const blockManifest = (id: string, fallback: string) => ({
      id,
      name: `Fixture ${id}`,
      version: '1.0.0',
      apiVersion: '1',
      kind: 'code',
      extensionPoints: [{ type: 'block', id: 'main', title: { de: id, en: id } }],
      permissions: [],
      fallback: { type: 'text', value: fallback },
      license: 'MIT',
    });
    await removeIfInstalled('fx-toc');
    await removeIfInstalled('fx-gone');
    await plugins.install(
      Buffer.from(
        zipSync({
          'manifest.json': enc(JSON.stringify(blockManifest('fx-toc', '[Table of contents]'))),
          'plugin.js': enc('export default {}'),
        }),
      ),
    );
    await plugins.install(
      Buffer.from(
        zipSync({
          'manifest.json': enc(JSON.stringify(blockManifest('fx-gone', '[Gone but text]'))),
          'plugin.js': enc('export default {}'),
        }),
      ),
    );
    await plugins.uninstall('fx-gone'); // tombstone: manifest snapshot remains

    try {
      // Content cache as the shared renderer emits it: a diagram block with a
      // stored SVG snapshot (hostile bits included), a toc block without one,
      // and a block of the uninstalled plugin.
      const svg =
        '<svg xmlns="http://www.w3.org/2000/svg"><script>alert(1)</script><rect width="5" height="5"></rect></svg>';
      // Exactly the attribute encoding the shared HTML renderer applies.
      const escapeAttr = (value: string) =>
        value
          .replace(/&/g, '&amp;')
          .replace(/</g, '&lt;')
          .replace(/>/g, '&gt;')
          .replace(/"/g, '&quot;');
      const svgData = escapeAttr(JSON.stringify({ source: 'graph', svg }));
      const html =
        `<div class="dt-plugin-block" data-plugin-block="mermaid/diagram" data-plugin-data="${svgData}">[mermaid/diagram]</div>` +
        '<div class="dt-plugin-block" data-plugin-block="fx-toc/main" data-plugin-data="{}">[fx-toc/main]</div>' +
        '<div class="dt-plugin-block" data-plugin-block="fx-gone/main" data-plugin-data="{}">[fx-gone/main]</div>';
      const markdown = [
        '```dorfteich-plugin fx-toc/main',
        '{}',
        '```',
        '',
        '::: {data-section-style="p/callout"}',
        'Sectioned text.',
        ':::',
      ].join('\n');
      const slug = await seedPage(personalPondId, 'Fallback Exports', markdown, html);
      const page = await prisma.page.findFirstOrThrow({
        where: { pondId: personalPondId, slug },
      });

      // PDF: snapshot SVG (sanitized!), manifest text, tombstone text.
      await api()
        .post(`/api/v1/pages/${page.id}/export`)
        .set('Cookie', ownerCookie)
        .send({ format: 'pdf' })
        .expect(201);
      await worker.drain();
      expect(renderer.lastHtml).toContain('<rect');
      expect(renderer.lastHtml).not.toContain('<script>');
      expect(renderer.lastHtml).toContain('[Table of contents]');
      expect(renderer.lastHtml).toContain('[Gone but text]');
      expect(renderer.lastHtml).not.toContain('dt-plugin-block"');

      // docx: fallback text, quoted section, no fence artifacts.
      await api()
        .post(`/api/v1/pages/${page.id}/export`)
        .set('Cookie', ownerCookie)
        .send({ format: 'docx' })
        .expect(201);
      await worker.drain();
      expect(fake.lastInput).toContain('\\[Table of contents\\]');
      expect(fake.lastInput).toContain('> Sectioned text.');
      expect(fake.lastInput).not.toContain('dorfteich-plugin');
      expect(fake.lastInput).not.toContain(':::');
    } finally {
      await removeIfInstalled('fx-toc');
      await removeIfInstalled('fx-gone');
    }
  });

  it('fails a PDF export when the renderer is down', async () => {
    renderer.failWith = new RenderError('render_failed', false, 'gotenberg exploded');
    const slug = await seedPage(personalPondId, 'Pdf Fails', 'x', '<p>x</p>');
    const page = await prisma.page.findFirstOrThrow({
      where: { pondId: personalPondId, slug },
    });

    const enqueued = await api()
      .post(`/api/v1/pages/${page.id}/export`)
      .set('Cookie', ownerCookie)
      .send({ format: 'pdf' })
      .expect(201);
    await worker.drain();
    renderer.failWith = null;

    const done = await api()
      .get(`/api/v1/jobs/${enqueued.body.id}`)
      .set('Cookie', ownerCookie)
      .expect(200);
    expect(done.body.status).toBe('failed');
    expect(done.body.errorCode).toBe('render_failed');
  });

  it('streams a large pond export (500 pages) without buffering it all', async () => {
    const big = await prisma.pond.create({
      data: {
        slug: `big-export-${suffix}`,
        name: 'Big Export',
        type: 'SHARED',
        ownerId,
        usage: { create: {} },
      },
    });
    await prisma.roleGrant.create({
      data: {
        pondId: big.id,
        subjectType: 'USER',
        subjectId: ownerId,
        role: 'POND_ADMIN',
        scopeType: 'POND',
        effect: 'ALLOW',
        createdBy: ownerId,
      },
    });
    const rows = Array.from({ length: 500 }, (_, i) => ({
      pondId: big.id,
      title: `Page ${i}`,
      slug: `page-${i}`,
      ydocState: new Uint8Array(),
      sortKey: String(i).padStart(4, '0'),
      createdBy: ownerId,
    }));
    await prisma.page.createMany({ data: rows });
    const created = await prisma.page.findMany({ where: { pondId: big.id }, select: { id: true } });
    await prisma.pageContentCache.createMany({
      data: created.map((p, i) => ({
        pageId: p.id,
        plainText: `Body ${i}`,
        markdown: `# Page ${i}\n\nBody ${i}.`,
        html: '',
        outline: [],
      })),
    });

    const res = await api()
      .get(`/api/v1/ponds/${big.id}/export/markdown`)
      .set('Cookie', ownerCookie)
      .buffer(true)
      .parse((r, cb) => {
        const chunks: Buffer[] = [];
        r.on('data', (c: Buffer) => chunks.push(c));
        r.on('end', () => cb(null, Buffer.concat(chunks)));
      })
      .expect(200);

    const mdEntries = Object.keys(zipEntries(res.body as Buffer)).filter((n) => n.endsWith('.md'));
    expect(mdEntries).toHaveLength(500);
  });

  /** Same as {@link seedPage} but for an arbitrary pond. */
  async function seedPageIn(pondId: string, title: string, markdown: string): Promise<string> {
    return seedPage(pondId, title, markdown);
  }
});