dorfteich/apps/api/src/import-export/import.service.db.test.ts

import { INestApplication } from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import request from 'supertest';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';

import { AuthTokensService } from '../auth/auth-tokens.service';
import { createTestApp, sessionCookieOf } from '../testing/test-app';
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
import { UsersService } from '../users/users.service';

import { ConversionWorker } from './conversion-worker.service';
import {
  ConversionError,
  ConversionRequest,
  ConversionResult,
  PandocConverter,
} from './pandoc.converter';

/**
 * Document import pipeline (issue #63): upload → conversion job → media stored
 * as pond files → new page. Driven by an injected fake converter that returns a
 * chosen Markdown for the structural pass, so the orchestration (media
 * extraction, quota, title, rollback, failure codes) is tested deterministically
 * without a live pandoc sidecar. The real conversion of the fixture corpus is
 * covered against a running sidecar by import.fixtures.test.ts.
 */

/** A 1×1 PNG — passes the raster magic-byte sniff, so it is stored as an image. */
const PNG_BASE64 =
  'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';

/** A converter whose `markdown` each test sets. The import pipeline calls it
 * twice (source→html, then html→gfm); the html pass is echoed, the gfm pass
 * returns the chosen Markdown (its input is irrelevant to these tests). A test
 * may instead make a pass throw to exercise a conversion failure. */
class FakeConverter extends PandocConverter {
  markdown = '# Untitled\n\nBody.';
  failHtmlWith: ConversionError | null = null;

  convert(request: ConversionRequest): Promise<ConversionResult> {
    if (request.to === 'html') {
      if (this.failHtmlWith) return Promise.reject(this.failHtmlWith);
      return Promise.resolve({ output: Buffer.from('<html/>'), mimeType: 'text/html' });
    }
    return Promise.resolve({ output: Buffer.from(this.markdown), mimeType: 'text/markdown' });
  }
  reachable(): Promise<boolean> {
    return Promise.resolve(true);
  }
}

describe.skipIf(!hasTestDb)('document import (e2e, issue #63)', () => {
  let app: INestApplication;
  let prisma: PrismaClient;
  let worker: ConversionWorker;
  let fake: FakeConverter;
  const suffix = uniqueSuffix();
  const password = 'importiere meine dokumente 1';

  const owner = { username: `iris-import-${suffix}`, displayName: `Iris Import ${suffix}` };
  const outsider = { username: `otis-out-${suffix}`, displayName: `Otis Out ${suffix}` };
  let ownerId: string;
  let pondId: string;
  let ownerCookie: string;
  let outsiderCookie: string;

  const api = () => request(app.getHttpServer());

  async function loginOf(username: string): Promise<string> {
    const res = await api()
      .post('/api/v1/auth/login')
      .send({ usernameOrEmail: username, password })
      .expect(200);
    return sessionCookieOf(res);
  }

  /** Upload a document and drain the queue; returns the finished job view. */
  async function importDoc(fileName: string, cookie = ownerCookie): Promise<request.Response> {
    const enqueued = await api()
      .post(`/api/v1/ponds/${pondId}/import`)
      .set('Cookie', cookie)
      .attach('file', Buffer.from('source-document-bytes'), fileName)
      .expect(201);
    await worker.drain();
    return api().get(`/api/v1/jobs/${enqueued.body.id}`).set('Cookie', cookie).expect(200);
  }

  async function markdownOf(pageId: string): Promise<string> {
    const cache = await prisma.pageContentCache.findUniqueOrThrow({ where: { pageId } });
    return cache.markdown;
  }

  beforeAll(async () => {
    prisma = createTestPrisma();
    await prisma.rateLimit.deleteMany({});
    fake = new FakeConverter();
    app = await createTestApp((builder) =>
      builder.overrideProvider(PandocConverter).useValue(fake),
    );
    worker = app.get(ConversionWorker);

    const users = app.get(UsersService);
    const tokens = app.get(AuthTokensService);
    const ownerUser = await users.createUser({
      username: owner.username,
      email: `${owner.username}@example.org`,
      displayName: owner.displayName,
      password,
      locale: 'en',
    });
    ownerId = ownerUser.id;
    // Verifying the e-mail creates the owner's personal pond (+ owner-admin
    // grant), which the owner may import into.
    const verify = await tokens.issue(ownerUser.id, 'EMAIL_VERIFICATION', 600);
    await api().post('/api/v1/auth/verify-email').send({ token: verify }).expect(204);
    ownerCookie = await loginOf(owner.username);
    pondId = (await prisma.pond.findFirstOrThrow({ where: { ownerId, type: 'PERSONAL' } })).id;

    const outsiderUser = await users.createUser({
      username: outsider.username,
      email: `${outsider.username}@example.org`,
      displayName: outsider.displayName,
      password,
      locale: 'en',
    });
    await users.markEmailVerified(outsiderUser.id);
    outsiderCookie = await loginOf(outsider.username);
  });

  afterAll(async () => {
    await prisma.conversionJob.deleteMany({ where: { owner: { username: { contains: suffix } } } });
    await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } });
    const where = { pond: { owner: { username: { contains: suffix } } } };
    await prisma.attachment.deleteMany({ where });
    // Imports created pages; remove them before their pond (pages restrict it).
    await prisma.page.deleteMany({ where });
    await prisma.roleGrant.deleteMany({ where });
    await prisma.pond.deleteMany({ where: { owner: { username: { contains: suffix } } } });
    await prisma.user.deleteMany({ where: { username: { contains: suffix } } });
    await prisma.$disconnect();
    await app.close();
  });

  it('imports a document as a new page, titled from the leading heading', async () => {
    fake.markdown = '# Imported Report\n\nA paragraph with **bold** text.\n\n## Section\n\nMore.';
    const job = await importDoc('report.docx');

    expect(job.body.status).toBe('succeeded');
    expect(job.body.errorCode).toBeNull();
    expect(job.body.resultPageId).toBeTruthy();

    const page = await prisma.page.findUniqueOrThrow({ where: { id: job.body.resultPageId } });
    expect(page.title).toBe('Imported Report');
    expect(page.pondId).toBe(pondId);
    // The leading H1 became the title and was removed from the body; the rest
    // of the structure survives.
    const markdown = await markdownOf(page.id);
    expect(markdown).not.toMatch(/^# Imported Report/);
    expect(markdown).toContain('**bold**');
    expect(markdown).toContain('## Section');
  });

  it('falls back to the file name (without extension) when there is no heading', async () => {
    fake.markdown = 'Just a paragraph, no heading.';
    const job = await importDoc('meeting-notes.odt');

    const page = await prisma.page.findUniqueOrThrow({ where: { id: job.body.resultPageId } });
    expect(page.title).toBe('meeting-notes');
  });

  it('stores an embedded image as a pond file with quota accounting', async () => {
    fake.markdown = `# With Image\n\nBefore.\n\n![a dot](data:image/png;base64,${PNG_BASE64})\n\nAfter.`;
    const before = await prisma.pondUsage.findUnique({ where: { pondId } });
    const usedBefore = Number(before?.storageBytesUsed ?? 0n);

    const job = await importDoc('illustrated.docx');
    const pageId = job.body.resultPageId as string;

    const files = await prisma.attachment.findMany({ where: { pageId } });
    expect(files).toHaveLength(1);
    expect(files[0]!.mimeType).toBe('image/png');
    expect(files[0]!.sizeBytes).toBeGreaterThan(0);

    // Usage grew by exactly the stored file's size (quota accounting).
    const after = await prisma.pondUsage.findUniqueOrThrow({ where: { pondId } });
    expect(Number(after.storageBytesUsed) - usedBefore).toBe(files[0]!.sizeBytes);

    // The image node now references the stored file id, not the data URI.
    const markdown = await markdownOf(pageId);
    expect(markdown).toContain(`(${files[0]!.id})`);
    expect(markdown).not.toContain('data:image');
  });

  it('drops an image whose type the upload pipeline rejects, keeping the rest', async () => {
    // A vector image Word can embed — not a raster type, not on the allowlist.
    fake.markdown = '# Vector\n\nText before.\n\n![v](data:image/x-emf;base64,AAAA)\n\nText after.';
    const job = await importDoc('vector.docx');
    const pageId = job.body.resultPageId as string;

    expect(await prisma.attachment.count({ where: { pageId } })).toBe(0);
    const markdown = await markdownOf(pageId);
    expect(markdown).toContain('Text before.');
    expect(markdown).toContain('Text after.');
    expect(markdown).not.toContain('data:image');
  });

  it('fails the job with a meaningful error for a corrupt/unsupported document', async () => {
    fake.failHtmlWith = new ConversionError('conversion_failed', false, 'not a real docx');
    const pagesBefore = await prisma.page.count({ where: { pondId } });

    const job = await importDoc('broken.docx');
    fake.failHtmlWith = null;

    expect(job.body.status).toBe('failed');
    expect(job.body.errorCode).toBe('conversion_failed');
    expect(job.body.resultPageId).toBeNull();
    // No page is created for a failed import.
    expect(await prisma.page.count({ where: { pondId } })).toBe(pagesBefore);
  });

  it('fails and rolls back media when the pond runs out of storage', async () => {
    // Cap this pond's storage below one image so the media store fails.
    await prisma.quotaOverride.upsert({
      where: {
        subjectType_subjectId_quotaKey: {
          subjectType: 'POND',
          subjectId: pondId,
          quotaKey: 'storage_bytes',
        },
      },
      create: { subjectType: 'POND', subjectId: pondId, quotaKey: 'storage_bytes', value: 10 },
      update: { value: 10 },
    });
    const usedBefore = Number(
      (await prisma.pondUsage.findUnique({ where: { pondId } }))?.storageBytesUsed ?? 0n,
    );
    const pagesBefore = await prisma.page.count({ where: { pondId } });
    fake.markdown = `# Too big\n\n![x](data:image/png;base64,${PNG_BASE64})`;

    const job = await importDoc('too-big.docx');

    expect(job.body.status).toBe('failed');
    expect(job.body.errorCode).toBe('quota_exceeded');
    // Neither a page nor any stored bytes survive the failed attempt.
    expect(await prisma.page.count({ where: { pondId } })).toBe(pagesBefore);
    const usedAfter = Number(
      (await prisma.pondUsage.findUnique({ where: { pondId } }))?.storageBytesUsed ?? 0n,
    );
    expect(usedAfter).toBe(usedBefore);

    await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } });
  });

  it('imports a Markdown file directly (no job): succeeded with a page at once', async () => {
    // A .md upload never touches the fake converter — it is parsed in-process.
    fake.markdown = 'SHOULD NOT BE USED';
    const enqueued = await api()
      .post(`/api/v1/ponds/${pondId}/import`)
      .set('Cookie', ownerCookie)
      .attach('file', Buffer.from('# Direct Markdown\n\nA paragraph.'), 'note.md')
      .expect(201);

    // No polling needed — the response is already terminal.
    expect(enqueued.body.status).toBe('succeeded');
    expect(enqueued.body.sourceFormat).toBe('md');
    const pageId = enqueued.body.resultPageId as string;
    expect(pageId).toBeTruthy();

    const page = await prisma.page.findUniqueOrThrow({ where: { id: pageId } });
    expect(page.title).toBe('Direct Markdown');
    expect(await markdownOf(page.id)).toContain('A paragraph.');
  });

  it('rejects an unsupported upload extension (400) before enqueuing', async () => {
    const res = await api()
      .post(`/api/v1/ponds/${pondId}/import`)
      .set('Cookie', ownerCookie)
      .attach('file', Buffer.from('hello'), 'notes.txt')
      .expect(400);
    expect(res.body.code).toBe('import_unsupported_format');
  });

  it('hides the pond from a non-member (404, not an import)', async () => {
    await api()
      .post(`/api/v1/ponds/${pondId}/import`)
      .set('Cookie', outsiderCookie)
      .attach('file', Buffer.from('x'), 'doc.docx')
      .expect(404);
  });
});