import { INestApplication } from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import request from 'supertest';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { AuthTokensService } from '../auth/auth-tokens.service';
import { createTestApp, sessionCookieOf } from '../testing/test-app';
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
import { UsersService } from '../users/users.service';
import { ConversionWorker } from './conversion-worker.service';
import {
ConversionError,
ConversionRequest,
ConversionResult,
PandocConverter,
} from './pandoc.converter';
/**
* Document import pipeline (issue #63): upload → conversion job → media stored
* as pond files → new page. Driven by an injected fake converter that returns a
* chosen Markdown for the structural pass, so the orchestration (media
* extraction, quota, title, rollback, failure codes) is tested deterministically
* without a live pandoc sidecar. The real conversion of the fixture corpus is
* covered against a running sidecar by import.fixtures.test.ts.
*/
/** A 1×1 PNG — passes the raster magic-byte sniff, so it is stored as an image. */
const PNG_BASE64 =
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';
/** A converter whose `markdown` each test sets. The import pipeline calls it
* twice (source→html, then html→gfm); the html pass is echoed, the gfm pass
* returns the chosen Markdown (its input is irrelevant to these tests). A test
* may instead make a pass throw to exercise a conversion failure. */
class FakeConverter extends PandocConverter {
markdown = '# Untitled\n\nBody.';
failHtmlWith: ConversionError | null = null;
convert(request: ConversionRequest): Promise {
if (request.to === 'html') {
if (this.failHtmlWith) return Promise.reject(this.failHtmlWith);
return Promise.resolve({ output: Buffer.from(''), mimeType: 'text/html' });
}
return Promise.resolve({ output: Buffer.from(this.markdown), mimeType: 'text/markdown' });
}
reachable(): Promise {
return Promise.resolve(true);
}
}
describe.skipIf(!hasTestDb)('document import (e2e, issue #63)', () => {
let app: INestApplication;
let prisma: PrismaClient;
let worker: ConversionWorker;
let fake: FakeConverter;
const suffix = uniqueSuffix();
const password = 'importiere meine dokumente 1';
const owner = { username: `iris-import-${suffix}`, displayName: `Iris Import ${suffix}` };
const outsider = { username: `otis-out-${suffix}`, displayName: `Otis Out ${suffix}` };
let ownerId: string;
let pondId: string;
let ownerCookie: string;
let outsiderCookie: string;
const api = () => request(app.getHttpServer());
async function loginOf(username: string): Promise {
const res = await api()
.post('/api/v1/auth/login')
.send({ usernameOrEmail: username, password })
.expect(200);
return sessionCookieOf(res);
}
/** Upload a document and drain the queue; returns the finished job view. */
async function importDoc(fileName: string, cookie = ownerCookie): Promise {
const enqueued = await api()
.post(`/api/v1/ponds/${pondId}/import`)
.set('Cookie', cookie)
.attach('file', Buffer.from('source-document-bytes'), fileName)
.expect(201);
await worker.drain();
return api().get(`/api/v1/jobs/${enqueued.body.id}`).set('Cookie', cookie).expect(200);
}
async function markdownOf(pageId: string): Promise {
const cache = await prisma.pageContentCache.findUniqueOrThrow({ where: { pageId } });
return cache.markdown;
}
beforeAll(async () => {
prisma = createTestPrisma();
await prisma.rateLimit.deleteMany({});
fake = new FakeConverter();
app = await createTestApp((builder) =>
builder.overrideProvider(PandocConverter).useValue(fake),
);
worker = app.get(ConversionWorker);
const users = app.get(UsersService);
const tokens = app.get(AuthTokensService);
const ownerUser = await users.createUser({
username: owner.username,
email: `${owner.username}@example.org`,
displayName: owner.displayName,
password,
locale: 'en',
});
ownerId = ownerUser.id;
// Verifying the e-mail creates the owner's personal pond (+ owner-admin
// grant), which the owner may import into.
const verify = await tokens.issue(ownerUser.id, 'EMAIL_VERIFICATION', 600);
await api().post('/api/v1/auth/verify-email').send({ token: verify }).expect(204);
ownerCookie = await loginOf(owner.username);
pondId = (await prisma.pond.findFirstOrThrow({ where: { ownerId, type: 'PERSONAL' } })).id;
const outsiderUser = await users.createUser({
username: outsider.username,
email: `${outsider.username}@example.org`,
displayName: outsider.displayName,
password,
locale: 'en',
});
await users.markEmailVerified(outsiderUser.id);
outsiderCookie = await loginOf(outsider.username);
});
afterAll(async () => {
await prisma.conversionJob.deleteMany({ where: { owner: { username: { contains: suffix } } } });
await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } });
const where = { pond: { owner: { username: { contains: suffix } } } };
await prisma.attachment.deleteMany({ where });
// Imports created pages; remove them before their pond (pages restrict it).
await prisma.page.deleteMany({ where });
await prisma.roleGrant.deleteMany({ where });
await prisma.pond.deleteMany({ where: { owner: { username: { contains: suffix } } } });
await prisma.user.deleteMany({ where: { username: { contains: suffix } } });
await prisma.$disconnect();
await app.close();
});
it('imports a document as a new page, titled from the leading heading', async () => {
fake.markdown = '# Imported Report\n\nA paragraph with **bold** text.\n\n## Section\n\nMore.';
const job = await importDoc('report.docx');
expect(job.body.status).toBe('succeeded');
expect(job.body.errorCode).toBeNull();
expect(job.body.resultPageId).toBeTruthy();
const page = await prisma.page.findUniqueOrThrow({ where: { id: job.body.resultPageId } });
expect(page.title).toBe('Imported Report');
expect(page.pondId).toBe(pondId);
// The leading H1 became the title and was removed from the body; the rest
// of the structure survives.
const markdown = await markdownOf(page.id);
expect(markdown).not.toMatch(/^# Imported Report/);
expect(markdown).toContain('**bold**');
expect(markdown).toContain('## Section');
});
it('falls back to the file name (without extension) when there is no heading', async () => {
fake.markdown = 'Just a paragraph, no heading.';
const job = await importDoc('meeting-notes.odt');
const page = await prisma.page.findUniqueOrThrow({ where: { id: job.body.resultPageId } });
expect(page.title).toBe('meeting-notes');
});
it('stores an embedded image as a pond file with quota accounting', async () => {
fake.markdown = `# With Image\n\nBefore.\n\n\n\nAfter.`;
const before = await prisma.pondUsage.findUnique({ where: { pondId } });
const usedBefore = Number(before?.storageBytesUsed ?? 0n);
const job = await importDoc('illustrated.docx');
const pageId = job.body.resultPageId as string;
const files = await prisma.attachment.findMany({ where: { pageId } });
expect(files).toHaveLength(1);
expect(files[0]!.mimeType).toBe('image/png');
expect(files[0]!.sizeBytes).toBeGreaterThan(0);
// Usage grew by exactly the stored file's size (quota accounting).
const after = await prisma.pondUsage.findUniqueOrThrow({ where: { pondId } });
expect(Number(after.storageBytesUsed) - usedBefore).toBe(files[0]!.sizeBytes);
// The image node now references the stored file id, not the data URI.
const markdown = await markdownOf(pageId);
expect(markdown).toContain(`(${files[0]!.id})`);
expect(markdown).not.toContain('data:image');
});
it('drops an image whose type the upload pipeline rejects, keeping the rest', async () => {
// A vector image Word can embed — not a raster type, not on the allowlist.
fake.markdown = '# Vector\n\nText before.\n\n\n\nText after.';
const job = await importDoc('vector.docx');
const pageId = job.body.resultPageId as string;
expect(await prisma.attachment.count({ where: { pageId } })).toBe(0);
const markdown = await markdownOf(pageId);
expect(markdown).toContain('Text before.');
expect(markdown).toContain('Text after.');
expect(markdown).not.toContain('data:image');
});
it('fails the job with a meaningful error for a corrupt/unsupported document', async () => {
fake.failHtmlWith = new ConversionError('conversion_failed', false, 'not a real docx');
const pagesBefore = await prisma.page.count({ where: { pondId } });
const job = await importDoc('broken.docx');
fake.failHtmlWith = null;
expect(job.body.status).toBe('failed');
expect(job.body.errorCode).toBe('conversion_failed');
expect(job.body.resultPageId).toBeNull();
// No page is created for a failed import.
expect(await prisma.page.count({ where: { pondId } })).toBe(pagesBefore);
});
it('fails and rolls back media when the pond runs out of storage', async () => {
// Cap this pond's storage below one image so the media store fails.
await prisma.quotaOverride.upsert({
where: {
subjectType_subjectId_quotaKey: {
subjectType: 'POND',
subjectId: pondId,
quotaKey: 'storage_bytes',
},
},
create: { subjectType: 'POND', subjectId: pondId, quotaKey: 'storage_bytes', value: 10 },
update: { value: 10 },
});
const usedBefore = Number(
(await prisma.pondUsage.findUnique({ where: { pondId } }))?.storageBytesUsed ?? 0n,
);
const pagesBefore = await prisma.page.count({ where: { pondId } });
fake.markdown = `# Too big\n\n`;
const job = await importDoc('too-big.docx');
expect(job.body.status).toBe('failed');
expect(job.body.errorCode).toBe('quota_exceeded');
// Neither a page nor any stored bytes survive the failed attempt.
expect(await prisma.page.count({ where: { pondId } })).toBe(pagesBefore);
const usedAfter = Number(
(await prisma.pondUsage.findUnique({ where: { pondId } }))?.storageBytesUsed ?? 0n,
);
expect(usedAfter).toBe(usedBefore);
await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } });
});
it('imports a Markdown file directly (no job): succeeded with a page at once', async () => {
// A .md upload never touches the fake converter — it is parsed in-process.
fake.markdown = 'SHOULD NOT BE USED';
const enqueued = await api()
.post(`/api/v1/ponds/${pondId}/import`)
.set('Cookie', ownerCookie)
.attach('file', Buffer.from('# Direct Markdown\n\nA paragraph.'), 'note.md')
.expect(201);
// No polling needed — the response is already terminal.
expect(enqueued.body.status).toBe('succeeded');
expect(enqueued.body.sourceFormat).toBe('md');
const pageId = enqueued.body.resultPageId as string;
expect(pageId).toBeTruthy();
const page = await prisma.page.findUniqueOrThrow({ where: { id: pageId } });
expect(page.title).toBe('Direct Markdown');
expect(await markdownOf(page.id)).toContain('A paragraph.');
});
it('rejects an unsupported upload extension (400) before enqueuing', async () => {
const res = await api()
.post(`/api/v1/ponds/${pondId}/import`)
.set('Cookie', ownerCookie)
.attach('file', Buffer.from('hello'), 'notes.txt')
.expect(400);
expect(res.body.code).toBe('import_unsupported_format');
});
it('hides the pond from a non-member (404, not an import)', async () => {
await api()
.post(`/api/v1/ponds/${pondId}/import`)
.set('Cookie', outsiderCookie)
.attach('file', Buffer.from('x'), 'doc.docx')
.expect(404);
});
});