import { INestApplication } from '@nestjs/common'; import { PrismaClient } from '@prisma/client'; import request from 'supertest'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { AuthTokensService } from '../auth/auth-tokens.service'; import { createTestApp, sessionCookieOf } from '../testing/test-app'; import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db'; import { UsersService } from '../users/users.service'; import { ConversionWorker } from './conversion-worker.service'; import { ConversionError, ConversionRequest, ConversionResult, PandocConverter, } from './pandoc.converter'; /** * Document import pipeline (issue #63): upload → conversion job → media stored * as pond files → new page. Driven by an injected fake converter that returns a * chosen Markdown for the structural pass, so the orchestration (media * extraction, quota, title, rollback, failure codes) is tested deterministically * without a live pandoc sidecar. The real conversion of the fixture corpus is * covered against a running sidecar by import.fixtures.test.ts. */ /** A 1×1 PNG — passes the raster magic-byte sniff, so it is stored as an image. */ const PNG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; /** A converter whose `markdown` each test sets. The import pipeline calls it * twice (source→html, then html→gfm); the html pass is echoed, the gfm pass * returns the chosen Markdown (its input is irrelevant to these tests). A test * may instead make a pass throw to exercise a conversion failure. */ class FakeConverter extends PandocConverter { markdown = '# Untitled\n\nBody.'; failHtmlWith: ConversionError | null = null; convert(request: ConversionRequest): Promise { if (request.to === 'html') { if (this.failHtmlWith) return Promise.reject(this.failHtmlWith); return Promise.resolve({ output: Buffer.from(''), mimeType: 'text/html' }); } return Promise.resolve({ output: Buffer.from(this.markdown), mimeType: 'text/markdown' }); } reachable(): Promise { return Promise.resolve(true); } } describe.skipIf(!hasTestDb)('document import (e2e, issue #63)', () => { let app: INestApplication; let prisma: PrismaClient; let worker: ConversionWorker; let fake: FakeConverter; const suffix = uniqueSuffix(); const password = 'importiere meine dokumente 1'; const owner = { username: `iris-import-${suffix}`, displayName: `Iris Import ${suffix}` }; const outsider = { username: `otis-out-${suffix}`, displayName: `Otis Out ${suffix}` }; let ownerId: string; let pondId: string; let ownerCookie: string; let outsiderCookie: string; const api = () => request(app.getHttpServer()); async function loginOf(username: string): Promise { const res = await api() .post('/api/v1/auth/login') .send({ usernameOrEmail: username, password }) .expect(200); return sessionCookieOf(res); } /** Upload a document and drain the queue; returns the finished job view. */ async function importDoc(fileName: string, cookie = ownerCookie): Promise { const enqueued = await api() .post(`/api/v1/ponds/${pondId}/import`) .set('Cookie', cookie) .attach('file', Buffer.from('source-document-bytes'), fileName) .expect(201); await worker.drain(); return api().get(`/api/v1/jobs/${enqueued.body.id}`).set('Cookie', cookie).expect(200); } async function markdownOf(pageId: string): Promise { const cache = await prisma.pageContentCache.findUniqueOrThrow({ where: { pageId } }); return cache.markdown; } beforeAll(async () => { prisma = createTestPrisma(); await prisma.rateLimit.deleteMany({}); fake = new FakeConverter(); app = await createTestApp((builder) => builder.overrideProvider(PandocConverter).useValue(fake), ); worker = app.get(ConversionWorker); const users = app.get(UsersService); const tokens = app.get(AuthTokensService); const ownerUser = await users.createUser({ username: owner.username, email: `${owner.username}@example.org`, displayName: owner.displayName, password, locale: 'en', }); ownerId = ownerUser.id; // Verifying the e-mail creates the owner's personal pond (+ owner-admin // grant), which the owner may import into. const verify = await tokens.issue(ownerUser.id, 'EMAIL_VERIFICATION', 600); await api().post('/api/v1/auth/verify-email').send({ token: verify }).expect(204); ownerCookie = await loginOf(owner.username); pondId = (await prisma.pond.findFirstOrThrow({ where: { ownerId, type: 'PERSONAL' } })).id; const outsiderUser = await users.createUser({ username: outsider.username, email: `${outsider.username}@example.org`, displayName: outsider.displayName, password, locale: 'en', }); await users.markEmailVerified(outsiderUser.id); outsiderCookie = await loginOf(outsider.username); }); afterAll(async () => { await prisma.conversionJob.deleteMany({ where: { owner: { username: { contains: suffix } } } }); await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } }); const where = { pond: { owner: { username: { contains: suffix } } } }; await prisma.attachment.deleteMany({ where }); // Imports created pages; remove them before their pond (pages restrict it). await prisma.page.deleteMany({ where }); await prisma.roleGrant.deleteMany({ where }); await prisma.pond.deleteMany({ where: { owner: { username: { contains: suffix } } } }); await prisma.user.deleteMany({ where: { username: { contains: suffix } } }); await prisma.$disconnect(); await app.close(); }); it('imports a document as a new page, titled from the leading heading', async () => { fake.markdown = '# Imported Report\n\nA paragraph with **bold** text.\n\n## Section\n\nMore.'; const job = await importDoc('report.docx'); expect(job.body.status).toBe('succeeded'); expect(job.body.errorCode).toBeNull(); expect(job.body.resultPageId).toBeTruthy(); const page = await prisma.page.findUniqueOrThrow({ where: { id: job.body.resultPageId } }); expect(page.title).toBe('Imported Report'); expect(page.pondId).toBe(pondId); // The leading H1 became the title and was removed from the body; the rest // of the structure survives. const markdown = await markdownOf(page.id); expect(markdown).not.toMatch(/^# Imported Report/); expect(markdown).toContain('**bold**'); expect(markdown).toContain('## Section'); }); it('falls back to the file name (without extension) when there is no heading', async () => { fake.markdown = 'Just a paragraph, no heading.'; const job = await importDoc('meeting-notes.odt'); const page = await prisma.page.findUniqueOrThrow({ where: { id: job.body.resultPageId } }); expect(page.title).toBe('meeting-notes'); }); it('stores an embedded image as a pond file with quota accounting', async () => { fake.markdown = `# With Image\n\nBefore.\n\n![a dot](data:image/png;base64,${PNG_BASE64})\n\nAfter.`; const before = await prisma.pondUsage.findUnique({ where: { pondId } }); const usedBefore = Number(before?.storageBytesUsed ?? 0n); const job = await importDoc('illustrated.docx'); const pageId = job.body.resultPageId as string; const files = await prisma.attachment.findMany({ where: { pageId } }); expect(files).toHaveLength(1); expect(files[0]!.mimeType).toBe('image/png'); expect(files[0]!.sizeBytes).toBeGreaterThan(0); // Usage grew by exactly the stored file's size (quota accounting). const after = await prisma.pondUsage.findUniqueOrThrow({ where: { pondId } }); expect(Number(after.storageBytesUsed) - usedBefore).toBe(files[0]!.sizeBytes); // The image node now references the stored file id, not the data URI. const markdown = await markdownOf(pageId); expect(markdown).toContain(`(${files[0]!.id})`); expect(markdown).not.toContain('data:image'); }); it('drops an image whose type the upload pipeline rejects, keeping the rest', async () => { // A vector image Word can embed — not a raster type, not on the allowlist. fake.markdown = '# Vector\n\nText before.\n\n![v](data:image/x-emf;base64,AAAA)\n\nText after.'; const job = await importDoc('vector.docx'); const pageId = job.body.resultPageId as string; expect(await prisma.attachment.count({ where: { pageId } })).toBe(0); const markdown = await markdownOf(pageId); expect(markdown).toContain('Text before.'); expect(markdown).toContain('Text after.'); expect(markdown).not.toContain('data:image'); }); it('fails the job with a meaningful error for a corrupt/unsupported document', async () => { fake.failHtmlWith = new ConversionError('conversion_failed', false, 'not a real docx'); const pagesBefore = await prisma.page.count({ where: { pondId } }); const job = await importDoc('broken.docx'); fake.failHtmlWith = null; expect(job.body.status).toBe('failed'); expect(job.body.errorCode).toBe('conversion_failed'); expect(job.body.resultPageId).toBeNull(); // No page is created for a failed import. expect(await prisma.page.count({ where: { pondId } })).toBe(pagesBefore); }); it('fails and rolls back media when the pond runs out of storage', async () => { // Cap this pond's storage below one image so the media store fails. await prisma.quotaOverride.upsert({ where: { subjectType_subjectId_quotaKey: { subjectType: 'POND', subjectId: pondId, quotaKey: 'storage_bytes', }, }, create: { subjectType: 'POND', subjectId: pondId, quotaKey: 'storage_bytes', value: 10 }, update: { value: 10 }, }); const usedBefore = Number( (await prisma.pondUsage.findUnique({ where: { pondId } }))?.storageBytesUsed ?? 0n, ); const pagesBefore = await prisma.page.count({ where: { pondId } }); fake.markdown = `# Too big\n\n![x](data:image/png;base64,${PNG_BASE64})`; const job = await importDoc('too-big.docx'); expect(job.body.status).toBe('failed'); expect(job.body.errorCode).toBe('quota_exceeded'); // Neither a page nor any stored bytes survive the failed attempt. expect(await prisma.page.count({ where: { pondId } })).toBe(pagesBefore); const usedAfter = Number( (await prisma.pondUsage.findUnique({ where: { pondId } }))?.storageBytesUsed ?? 0n, ); expect(usedAfter).toBe(usedBefore); await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } }); }); it('imports a Markdown file directly (no job): succeeded with a page at once', async () => { // A .md upload never touches the fake converter — it is parsed in-process. fake.markdown = 'SHOULD NOT BE USED'; const enqueued = await api() .post(`/api/v1/ponds/${pondId}/import`) .set('Cookie', ownerCookie) .attach('file', Buffer.from('# Direct Markdown\n\nA paragraph.'), 'note.md') .expect(201); // No polling needed — the response is already terminal. expect(enqueued.body.status).toBe('succeeded'); expect(enqueued.body.sourceFormat).toBe('md'); const pageId = enqueued.body.resultPageId as string; expect(pageId).toBeTruthy(); const page = await prisma.page.findUniqueOrThrow({ where: { id: pageId } }); expect(page.title).toBe('Direct Markdown'); expect(await markdownOf(page.id)).toContain('A paragraph.'); }); it('rejects an unsupported upload extension (400) before enqueuing', async () => { const res = await api() .post(`/api/v1/ponds/${pondId}/import`) .set('Cookie', ownerCookie) .attach('file', Buffer.from('hello'), 'notes.txt') .expect(400); expect(res.body.code).toBe('import_unsupported_format'); }); it('hides the pond from a non-member (404, not an import)', async () => { await api() .post(`/api/v1/ponds/${pondId}/import`) .set('Cookie', outsiderCookie) .attach('file', Buffer.from('x'), 'doc.docx') .expect(404); }); });