diff --git a/.prettierignore b/.prettierignore index 6ae2ada..35e1aed 100644 --- a/.prettierignore +++ b/.prettierignore @@ -6,3 +6,8 @@ pnpm-lock.yaml # Must stay byte-identical to docToMarkdown's own output (issue #32) — # Prettier's Markdown table/list opinions would break that fixed point. apps/api/prisma/fixtures/content-page.md +# Import fixture corpus (issue #63): the expected-Markdown snapshots and the +# HTML sources must stay exactly as pandoc produces/consumes them — Prettier's +# Markdown/HTML opinions would break the regression tests. +fixtures/import/*.expected.md +fixtures/import/*.src.html diff --git a/apps/api/prisma/migrations/20260710041215_import_pages_conversion/migration.sql b/apps/api/prisma/migrations/20260710041215_import_pages_conversion/migration.sql new file mode 100644 index 0000000..1c49877 --- /dev/null +++ b/apps/api/prisma/migrations/20260710041215_import_pages_conversion/migration.sql @@ -0,0 +1,10 @@ +-- AlterTable +ALTER TABLE "conversion_jobs" ADD COLUMN "pond_id" TEXT, +ADD COLUMN "result_page_id" TEXT, +ADD COLUMN "source_name" TEXT; + +-- AddForeignKey +ALTER TABLE "conversion_jobs" ADD CONSTRAINT "conversion_jobs_pond_id_fkey" FOREIGN KEY ("pond_id") REFERENCES "ponds"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "conversion_jobs" ADD CONSTRAINT "conversion_jobs_result_page_id_fkey" FOREIGN KEY ("result_page_id") REFERENCES "pages"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index a54cb94..e77ccf4 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -81,6 +81,7 @@ model Pond { attachments Attachment[] labels Label[] grants RoleGrant[] + conversionJobs ConversionJob[] @@index([ownerId]) @@map("ponds") @@ -164,6 +165,7 @@ model Page { labels PageLabel[] outgoingLinks PageLink[] @relation("outgoingLinks") incomingLinks PageLink[] @relation("incomingLinks") + conversionJobs ConversionJob[] @@unique([pondId, slug]) @@index([pondId]) @@ -551,7 +553,16 @@ model ConversionJob { createdAt DateTime @default(now()) @map("created_at") updatedAt DateTime @updatedAt @map("updated_at") - owner User @relation(fields: [ownerId], references: [id], onDelete: Cascade) + /// Import jobs (#63) carry the pond the document is imported into, the + /// original upload file name (title fallback), and the page they produced. + /// All null for a plain byte→byte conversion (export, #62/#65). + pondId String? @map("pond_id") + sourceName String? @map("source_name") + resultPageId String? @map("result_page_id") + + owner User @relation(fields: [ownerId], references: [id], onDelete: Cascade) + pond Pond? @relation(fields: [pondId], references: [id], onDelete: Cascade) + page Page? @relation(fields: [resultPageId], references: [id], onDelete: SetNull) @@index([status, createdAt]) @@map("conversion_jobs") diff --git a/apps/api/src/files/files.module.ts b/apps/api/src/files/files.module.ts index 26665e0..1fa378a 100644 --- a/apps/api/src/files/files.module.ts +++ b/apps/api/src/files/files.module.ts @@ -11,6 +11,6 @@ import { FilesService } from './files.service'; imports: [PondsModule, QuotasModule], controllers: [FilesController], providers: [FilesService, FileStorageService], - exports: [FileStorageService], + exports: [FileStorageService, FilesService], }) export class FilesModule {} diff --git a/apps/api/src/files/files.service.ts b/apps/api/src/files/files.service.ts index 742a751..da943be 100644 --- a/apps/api/src/files/files.service.ts +++ b/apps/api/src/files/files.service.ts @@ -169,6 +169,21 @@ export class FilesService { } } + /** + * Point a set of just-uploaded attachments at the page that now embeds them + * (issue #63 import). The import worker stores a document's media before the + * page exists (the page's Yjs state references their ids), then calls this so + * they list under the page and are purged with it (#31), the same invariant + * the collab persistence hook maintains for pasted images. + */ + async linkAttachmentsToPage(attachmentIds: string[], pageId: string): Promise { + if (attachmentIds.length === 0) return; + await this.prisma.attachment.updateMany({ + where: { id: { in: attachmentIds } }, + data: { pageId }, + }); + } + /** Upload against the pond of `pageId`, linked to that page (#61). */ async uploadToPage( user: User, diff --git a/apps/api/src/import-export/conversion-job.service.ts b/apps/api/src/import-export/conversion-job.service.ts index 4667f25..5713f3b 100644 --- a/apps/api/src/import-export/conversion-job.service.ts +++ b/apps/api/src/import-export/conversion-job.service.ts @@ -15,6 +15,10 @@ export interface EnqueueConversion { to: string; input: Buffer; standalone?: boolean; + /** Set for import jobs (#63): the pond the document is imported into and the + * original upload file name (a title fallback). */ + pondId?: string; + sourceName?: string; } export interface ConversionResultPayload { @@ -55,6 +59,8 @@ export class ConversionJobService { const job = await this.prisma.conversionJob.create({ data: { ownerId: request.ownerId, + pondId: request.pondId ?? null, + sourceName: request.sourceName ?? null, kind: request.kind, sourceFormat: request.from, targetFormat: request.to, @@ -103,6 +109,8 @@ export class ConversionJobService { sourceFormat: job.sourceFormat, targetFormat: job.targetFormat, errorCode: job.errorCode, + // Set once an import job succeeds (#63) so the client can open the page. + resultPageId: job.resultPageId, createdAt: job.createdAt.toISOString(), updatedAt: job.updatedAt.toISOString(), }; diff --git a/apps/api/src/import-export/conversion-worker.service.ts b/apps/api/src/import-export/conversion-worker.service.ts index aaa9639..97d4964 100644 --- a/apps/api/src/import-export/conversion-worker.service.ts +++ b/apps/api/src/import-export/conversion-worker.service.ts @@ -1,10 +1,12 @@ import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { ModuleRef } from '@nestjs/core'; import { ConversionJob } from '@prisma/client'; import { PinoLogger } from 'nestjs-pino'; import { AppConfig } from '../config/app-config.service'; import { PrismaService } from '../prisma/prisma.service'; +import { IMPORT_PROCESSOR, ImportProcessor, isImportKind } from './import.constants'; import { ConversionError, PandocConverter } from './pandoc.converter'; /** How often the worker sweeps for pending jobs on its own — the safety net @@ -38,6 +40,9 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy { private readonly converter: PandocConverter, private readonly config: AppConfig, private readonly logger: PinoLogger, + // Resolved lazily to break the construction cycle (the import service + // enqueues via the job service, which wakes this worker). + private readonly moduleRef: ModuleRef, ) { this.logger.setContext(ConversionWorker.name); } @@ -103,6 +108,13 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy { private async process(job: ConversionJob): Promise { try { + if (isImportKind(job.kind)) { + // Import runs a multi-step pipeline and records its own success (the + // created page id) on the job (#63). Resolved lazily via a token so the + // worker's file never imports the import service's (avoids a cycle). + await this.moduleRef.get(IMPORT_PROCESSOR, { strict: false }).run(job); + return; + } const result = await this.converter.convert({ from: job.sourceFormat, to: job.targetFormat, diff --git a/apps/api/src/import-export/import-export.module.ts b/apps/api/src/import-export/import-export.module.ts index 7bf16f3..28d0557 100644 --- a/apps/api/src/import-export/import-export.module.ts +++ b/apps/api/src/import-export/import-export.module.ts @@ -1,20 +1,32 @@ import { Module } from '@nestjs/common'; +import { FilesModule } from '../files/files.module'; +import { PagesModule } from '../pages/pages.module'; + import { ConversionJobService } from './conversion-job.service'; import { ConversionWorker } from './conversion-worker.service'; +import { IMPORT_PROCESSOR } from './import.constants'; +import { ImportController } from './import.controller'; +import { ImportService } from './import.service'; import { JobsController } from './jobs.controller'; import { PandocConverter, PandocServerConverter } from './pandoc.converter'; /** - * Import/export orchestration (ADR 0009, issue #62): the conversion job queue, - * its worker, and the pandoc-server client. Later stories (#63 import, #65 - * export) add the feature endpoints that enqueue jobs here. + * Import/export orchestration (ADR 0009): the conversion job queue, its worker, + * the pandoc-server client (#62), and the document import pipeline (#63, which + * turns an uploaded `.docx`/`.odt` into a new page). Export (#65) will add its + * feature endpoints here too. */ @Module({ - controllers: [JobsController], + imports: [FilesModule, PagesModule], + controllers: [JobsController, ImportController], providers: [ ConversionJobService, ConversionWorker, + ImportService, + // The worker resolves the import pipeline through this token (never the + // class), so its file does not import the import service's (avoids a cycle). + { provide: IMPORT_PROCESSOR, useExisting: ImportService }, // Bind the abstract converter to the HTTP implementation; tests override // this provider with a fake so the queue mechanics need no live sidecar. { provide: PandocConverter, useClass: PandocServerConverter }, diff --git a/apps/api/src/import-export/import.constants.ts b/apps/api/src/import-export/import.constants.ts new file mode 100644 index 0000000..30d7aa2 --- /dev/null +++ b/apps/api/src/import-export/import.constants.ts @@ -0,0 +1,21 @@ +import { ConversionJob } from '@prisma/client'; + +/** + * Contract between the shared conversion worker and the import pipeline, kept in + * its own module so neither imports the other's file (the worker resolves the + * processor lazily via this token, breaking what would otherwise be a cycle: + * import.service → conversion-job.service → conversion-worker.service). + */ +export const IMPORT_PROCESSOR = Symbol('IMPORT_PROCESSOR'); + +export interface ImportProcessor { + /** Run one import job to completion, recording the created page on the job or + * throwing a ConversionError for the worker's retry/fail policy. */ + run(job: ConversionJob): Promise; +} + +/** True for a job the import pipeline processes rather than the plain pandoc + * byte→byte path (export, #62/#65). */ +export function isImportKind(kind: string): boolean { + return kind.startsWith('import_'); +} diff --git a/apps/api/src/import-export/import.controller.ts b/apps/api/src/import-export/import.controller.ts new file mode 100644 index 0000000..185cb26 --- /dev/null +++ b/apps/api/src/import-export/import.controller.ts @@ -0,0 +1,39 @@ +import { + BadRequestException, + Controller, + Param, + Post, + Req, + UploadedFile, + UseInterceptors, +} from '@nestjs/common'; +import { FileInterceptor } from '@nestjs/platform-express'; +import { ConversionJobView, MAX_UPLOAD_PARSE_BYTES } from '@dorfteich/shared'; + +import { AuthedRequest } from '../auth/auth.guard'; +import { RequiresPondRole } from '../permissions/permission.decorators'; + +import { ImportService } from './import.service'; + +/** + * Document import (ADR 0009, issue #63). Uploading a `.docx`/`.odt` enqueues a + * conversion job (#62 queue) that produces a new page in the pond; the client + * polls `GET /jobs/:id` for `resultPageId`. Creating a page requires pond-wide + * editor access (permissions.md, same rule as `POST /ponds/:id/pages`). + */ +@Controller('ponds/:pondId') +export class ImportController { + constructor(private readonly imports: ImportService) {} + + @Post('import') + @RequiresPondRole('editor', { idParam: 'pondId' }) + @UseInterceptors(FileInterceptor('file', { limits: { fileSize: MAX_UPLOAD_PARSE_BYTES } })) + async import( + @Param('pondId') pondId: string, + @UploadedFile() file: Express.Multer.File | undefined, + @Req() request: AuthedRequest, + ): Promise { + if (!file) throw new BadRequestException({ code: 'bad_request' }); + return this.imports.enqueue(request.user!, pondId, file); + } +} diff --git a/apps/api/src/import-export/import.fixtures.test.ts b/apps/api/src/import-export/import.fixtures.test.ts new file mode 100644 index 0000000..735a3d0 --- /dev/null +++ b/apps/api/src/import-export/import.fixtures.test.ts @@ -0,0 +1,78 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { markdownToDoc } from '@dorfteich/shared'; +import { beforeAll, describe, expect, it, TestContext } from 'vitest'; + +import { AppConfig } from '../config/app-config.service'; + +import { convertImportedDocument } from './import.service'; +import { CONVERSION_TIMEOUT_MS, PandocServerConverter } from './pandoc.converter'; + +/** + * Import fidelity regression (issue #63, ADR 0009): runs the real two-pass + * pandoc conversion over the committed `.docx`/`.odt` corpus and asserts each + * produces its expected Markdown. Needs a reachable pandoc sidecar (the pinned + * `pandoc/core:3.6`, so output matches the snapshots) — each test skips itself + * when none is configured, and CI starts one and points `PANDOC_URL` at it. + */ +const PANDOC_URL = process.env.PANDOC_URL ?? 'http://localhost:3030'; +// The test runner's cwd is `apps/api`; the corpus lives at the repo root. +const FIXTURES = join(process.cwd(), '../../fixtures/import'); + +const converter = new PandocServerConverter({ + env: { PANDOC_URL }, +} as unknown as AppConfig); + +let reachable = false; + +/** Normalise embedded image `data:` URIs to the stable token the snapshots use + * (the base64 payload is volatile and not what we are pinning). */ +function normalize(markdown: string): string { + return markdown.replace( + /data:image\/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g, + 'data:embedded-image', + ); +} + +const CORPUS = ['article.docx', 'article.odt', 'formatting.docx', 'formatting.odt']; + +describe('import fixture corpus (real pandoc, issue #63)', () => { + beforeAll(async () => { + reachable = await converter.reachable().catch(() => false); + }); + + for (const fixture of CORPUS) { + it(`converts ${fixture} to its expected Markdown`, async (ctx: TestContext) => { + if (!reachable) ctx.skip(); + const format = fixture.endsWith('.odt') ? 'odt' : 'docx'; + const document = readFileSync(join(FIXTURES, fixture)); + const expected = readFileSync(join(FIXTURES, `${fixture}.expected.md`), 'utf8'); + + const markdown = await convertImportedDocument(converter, format, document); + expect(normalize(markdown)).toBe(expected); + + // The Markdown must also parse into a valid editor document (no schema + // surprises from real-world structure). + expect(() => markdownToDoc(markdown)).not.toThrow(); + }); + } + + it('imports a 50-page document within the conversion timeout', async (ctx: TestContext) => { + if (!reachable) ctx.skip(); + // Build a ~50-page document by repeating a page of structured content, then + // convert it to docx once and time the import conversion of that document. + const onePage = + '# Section\n\n' + 'A paragraph of survey notes about the pond. '.repeat(20) + '\n\n'; + const large = Array.from({ length: 50 }, () => onePage).join('\n---\n\n'); + const docx = await converter.convert({ from: 'gfm', to: 'docx', input: Buffer.from(large) }); + + const started = Date.now(); + const markdown = await convertImportedDocument(converter, 'docx', docx.output); + const elapsed = Date.now() - started; + + expect(markdown.length).toBeGreaterThan(1000); + // Comfortably inside the documented 60 s per-conversion ceiling (ADR 0009). + expect(elapsed).toBeLessThan(CONVERSION_TIMEOUT_MS); + }); +}); diff --git a/apps/api/src/import-export/import.service.db.test.ts b/apps/api/src/import-export/import.service.db.test.ts new file mode 100644 index 0000000..04cb74d --- /dev/null +++ b/apps/api/src/import-export/import.service.db.test.ts @@ -0,0 +1,270 @@ +import { INestApplication } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; +import request from 'supertest'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +import { AuthTokensService } from '../auth/auth-tokens.service'; +import { createTestApp, sessionCookieOf } from '../testing/test-app'; +import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db'; +import { UsersService } from '../users/users.service'; + +import { ConversionWorker } from './conversion-worker.service'; +import { + ConversionError, + ConversionRequest, + ConversionResult, + PandocConverter, +} from './pandoc.converter'; + +/** + * Document import pipeline (issue #63): upload → conversion job → media stored + * as pond files → new page. Driven by an injected fake converter that returns a + * chosen Markdown for the structural pass, so the orchestration (media + * extraction, quota, title, rollback, failure codes) is tested deterministically + * without a live pandoc sidecar. The real conversion of the fixture corpus is + * covered against a running sidecar by import.fixtures.test.ts. + */ + +/** A 1×1 PNG — passes the raster magic-byte sniff, so it is stored as an image. */ +const PNG_BASE64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + +/** A converter whose `markdown` each test sets. The import pipeline calls it + * twice (source→html, then html→gfm); the html pass is echoed, the gfm pass + * returns the chosen Markdown (its input is irrelevant to these tests). A test + * may instead make a pass throw to exercise a conversion failure. */ +class FakeConverter extends PandocConverter { + markdown = '# Untitled\n\nBody.'; + failHtmlWith: ConversionError | null = null; + + convert(request: ConversionRequest): Promise { + if (request.to === 'html') { + if (this.failHtmlWith) return Promise.reject(this.failHtmlWith); + return Promise.resolve({ output: Buffer.from(''), mimeType: 'text/html' }); + } + return Promise.resolve({ output: Buffer.from(this.markdown), mimeType: 'text/markdown' }); + } + reachable(): Promise { + return Promise.resolve(true); + } +} + +describe.skipIf(!hasTestDb)('document import (e2e, issue #63)', () => { + let app: INestApplication; + let prisma: PrismaClient; + let worker: ConversionWorker; + let fake: FakeConverter; + const suffix = uniqueSuffix(); + const password = 'importiere meine dokumente 1'; + + const owner = { username: `iris-import-${suffix}`, displayName: `Iris Import ${suffix}` }; + const outsider = { username: `otis-out-${suffix}`, displayName: `Otis Out ${suffix}` }; + let ownerId: string; + let pondId: string; + let ownerCookie: string; + let outsiderCookie: string; + + const api = () => request(app.getHttpServer()); + + async function loginOf(username: string): Promise { + const res = await api() + .post('/api/v1/auth/login') + .send({ usernameOrEmail: username, password }) + .expect(200); + return sessionCookieOf(res); + } + + /** Upload a document and drain the queue; returns the finished job view. */ + async function importDoc(fileName: string, cookie = ownerCookie): Promise { + const enqueued = await api() + .post(`/api/v1/ponds/${pondId}/import`) + .set('Cookie', cookie) + .attach('file', Buffer.from('source-document-bytes'), fileName) + .expect(201); + await worker.drain(); + return api().get(`/api/v1/jobs/${enqueued.body.id}`).set('Cookie', cookie).expect(200); + } + + async function markdownOf(pageId: string): Promise { + const cache = await prisma.pageContentCache.findUniqueOrThrow({ where: { pageId } }); + return cache.markdown; + } + + beforeAll(async () => { + prisma = createTestPrisma(); + await prisma.rateLimit.deleteMany({}); + fake = new FakeConverter(); + app = await createTestApp((builder) => + builder.overrideProvider(PandocConverter).useValue(fake), + ); + worker = app.get(ConversionWorker); + + const users = app.get(UsersService); + const tokens = app.get(AuthTokensService); + const ownerUser = await users.createUser({ + username: owner.username, + email: `${owner.username}@example.org`, + displayName: owner.displayName, + password, + locale: 'en', + }); + ownerId = ownerUser.id; + // Verifying the e-mail creates the owner's personal pond (+ owner-admin + // grant), which the owner may import into. + const verify = await tokens.issue(ownerUser.id, 'EMAIL_VERIFICATION', 600); + await api().post('/api/v1/auth/verify-email').send({ token: verify }).expect(204); + ownerCookie = await loginOf(owner.username); + pondId = (await prisma.pond.findFirstOrThrow({ where: { ownerId, type: 'PERSONAL' } })).id; + + const outsiderUser = await users.createUser({ + username: outsider.username, + email: `${outsider.username}@example.org`, + displayName: outsider.displayName, + password, + locale: 'en', + }); + await users.markEmailVerified(outsiderUser.id); + outsiderCookie = await loginOf(outsider.username); + }); + + afterAll(async () => { + await prisma.conversionJob.deleteMany({ where: { owner: { username: { contains: suffix } } } }); + await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } }); + const where = { pond: { owner: { username: { contains: suffix } } } }; + await prisma.attachment.deleteMany({ where }); + // Imports created pages; remove them before their pond (pages restrict it). + await prisma.page.deleteMany({ where }); + await prisma.roleGrant.deleteMany({ where }); + await prisma.pond.deleteMany({ where: { owner: { username: { contains: suffix } } } }); + await prisma.user.deleteMany({ where: { username: { contains: suffix } } }); + await prisma.$disconnect(); + await app.close(); + }); + + it('imports a document as a new page, titled from the leading heading', async () => { + fake.markdown = '# Imported Report\n\nA paragraph with **bold** text.\n\n## Section\n\nMore.'; + const job = await importDoc('report.docx'); + + expect(job.body.status).toBe('succeeded'); + expect(job.body.errorCode).toBeNull(); + expect(job.body.resultPageId).toBeTruthy(); + + const page = await prisma.page.findUniqueOrThrow({ where: { id: job.body.resultPageId } }); + expect(page.title).toBe('Imported Report'); + expect(page.pondId).toBe(pondId); + // The leading H1 became the title and was removed from the body; the rest + // of the structure survives. + const markdown = await markdownOf(page.id); + expect(markdown).not.toMatch(/^# Imported Report/); + expect(markdown).toContain('**bold**'); + expect(markdown).toContain('## Section'); + }); + + it('falls back to the file name (without extension) when there is no heading', async () => { + fake.markdown = 'Just a paragraph, no heading.'; + const job = await importDoc('meeting-notes.odt'); + + const page = await prisma.page.findUniqueOrThrow({ where: { id: job.body.resultPageId } }); + expect(page.title).toBe('meeting-notes'); + }); + + it('stores an embedded image as a pond file with quota accounting', async () => { + fake.markdown = `# With Image\n\nBefore.\n\n![a dot](data:image/png;base64,${PNG_BASE64})\n\nAfter.`; + const before = await prisma.pondUsage.findUnique({ where: { pondId } }); + const usedBefore = Number(before?.storageBytesUsed ?? 0n); + + const job = await importDoc('illustrated.docx'); + const pageId = job.body.resultPageId as string; + + const files = await prisma.attachment.findMany({ where: { pageId } }); + expect(files).toHaveLength(1); + expect(files[0]!.mimeType).toBe('image/png'); + expect(files[0]!.sizeBytes).toBeGreaterThan(0); + + // Usage grew by exactly the stored file's size (quota accounting). + const after = await prisma.pondUsage.findUniqueOrThrow({ where: { pondId } }); + expect(Number(after.storageBytesUsed) - usedBefore).toBe(files[0]!.sizeBytes); + + // The image node now references the stored file id, not the data URI. + const markdown = await markdownOf(pageId); + expect(markdown).toContain(`(${files[0]!.id})`); + expect(markdown).not.toContain('data:image'); + }); + + it('drops an image whose type the upload pipeline rejects, keeping the rest', async () => { + // A vector image Word can embed — not a raster type, not on the allowlist. + fake.markdown = '# Vector\n\nText before.\n\n![v](data:image/x-emf;base64,AAAA)\n\nText after.'; + const job = await importDoc('vector.docx'); + const pageId = job.body.resultPageId as string; + + expect(await prisma.attachment.count({ where: { pageId } })).toBe(0); + const markdown = await markdownOf(pageId); + expect(markdown).toContain('Text before.'); + expect(markdown).toContain('Text after.'); + expect(markdown).not.toContain('data:image'); + }); + + it('fails the job with a meaningful error for a corrupt/unsupported document', async () => { + fake.failHtmlWith = new ConversionError('conversion_failed', false, 'not a real docx'); + const pagesBefore = await prisma.page.count({ where: { pondId } }); + + const job = await importDoc('broken.docx'); + fake.failHtmlWith = null; + + expect(job.body.status).toBe('failed'); + expect(job.body.errorCode).toBe('conversion_failed'); + expect(job.body.resultPageId).toBeNull(); + // No page is created for a failed import. + expect(await prisma.page.count({ where: { pondId } })).toBe(pagesBefore); + }); + + it('fails and rolls back media when the pond runs out of storage', async () => { + // Cap this pond's storage below one image so the media store fails. + await prisma.quotaOverride.upsert({ + where: { + subjectType_subjectId_quotaKey: { + subjectType: 'POND', + subjectId: pondId, + quotaKey: 'storage_bytes', + }, + }, + create: { subjectType: 'POND', subjectId: pondId, quotaKey: 'storage_bytes', value: 10 }, + update: { value: 10 }, + }); + const usedBefore = Number( + (await prisma.pondUsage.findUnique({ where: { pondId } }))?.storageBytesUsed ?? 0n, + ); + const pagesBefore = await prisma.page.count({ where: { pondId } }); + fake.markdown = `# Too big\n\n![x](data:image/png;base64,${PNG_BASE64})`; + + const job = await importDoc('too-big.docx'); + + expect(job.body.status).toBe('failed'); + expect(job.body.errorCode).toBe('quota_exceeded'); + // Neither a page nor any stored bytes survive the failed attempt. + expect(await prisma.page.count({ where: { pondId } })).toBe(pagesBefore); + const usedAfter = Number( + (await prisma.pondUsage.findUnique({ where: { pondId } }))?.storageBytesUsed ?? 0n, + ); + expect(usedAfter).toBe(usedBefore); + + await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } }); + }); + + it('rejects an unsupported upload extension (400) before enqueuing', async () => { + const res = await api() + .post(`/api/v1/ponds/${pondId}/import`) + .set('Cookie', ownerCookie) + .attach('file', Buffer.from('hello'), 'notes.txt') + .expect(400); + expect(res.body.code).toBe('import_unsupported_format'); + }); + + it('hides the pond from a non-member (404, not an import)', async () => { + await api() + .post(`/api/v1/ponds/${pondId}/import`) + .set('Cookie', outsiderCookie) + .attach('file', Buffer.from('x'), 'doc.docx') + .expect(404); + }); +}); diff --git a/apps/api/src/import-export/import.service.ts b/apps/api/src/import-export/import.service.ts new file mode 100644 index 0000000..cf9f785 --- /dev/null +++ b/apps/api/src/import-export/import.service.ts @@ -0,0 +1,322 @@ +import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/common'; +import { ConversionJob, User } from '@prisma/client'; +import { ConversionJobView, editorSchema, markdownToDoc } from '@dorfteich/shared'; +import { Node } from 'prosemirror-model'; +import { PinoLogger } from 'nestjs-pino'; + +import { FilesService } from '../files/files.service'; +import { PagesService } from '../pages/pages.service'; +import { docToState } from '../pages/yjs-content'; +import { PrismaService } from '../prisma/prisma.service'; + +import { ImportProcessor } from './import.constants'; +import { ConversionError, PandocConverter } from './pandoc.converter'; +import { ConversionJobService } from './conversion-job.service'; + +/** pandoc source format per accepted upload extension (ADR 0009). */ +const IMPORT_FORMATS: Readonly> = { + docx: 'docx', + odt: 'odt', +}; + +/** Job `kind` per source format, so the worker can route the job to the import + * pipeline (a plain byte→byte conversion has a different kind, #62/#65). */ +function importKind(format: string): string { + return `import_${format}`; +} + +/** + * Writer format for the structural pass. GFM matches our editor schema + * (tables, task lists, strikethrough); `-implicit_figures` keeps images as + * inline `![alt](src)` instead of wrapping them in a figure with a duplicated + * caption; `-raw_html` stops pandoc emitting raw HTML for constructs it can't + * represent (our Markdown parser runs with `html:false` and would drop it + * anyway — per ADR 0009 we preserve structure, not layout). + */ +const IMPORT_MARKDOWN_FORMAT = 'gfm-implicit_figures-raw_html'; + +/** + * The two-pass conversion at the heart of import, extracted so tests can run it + * over the fixture corpus against a real sidecar. pandoc-server is stateless and + * hands back a document's media no other way, so we first inline every image as + * a `data:` URI (`source → html`, embed-resources), then produce clean + * structural Markdown that still carries those data URIs inline at the right + * positions (`html → gfm`, no wrapping). + */ +export async function convertImportedDocument( + converter: PandocConverter, + sourceFormat: string, + input: Buffer, +): Promise { + const html = await converter.convert({ + from: sourceFormat, + to: 'html', + input, + standalone: false, + embedResources: true, + }); + const md = await converter.convert({ + from: 'html', + to: IMPORT_MARKDOWN_FORMAT, + input: html.output, + standalone: false, + wrap: 'none', + }); + return md.output.toString('utf8'); +} + +/** Minimal shape of a ProseMirror document as JSON — enough to walk it for + * image nodes and the leading heading without pulling in prosemirror types. */ +interface PmNode { + type: string; + attrs?: Record; + content?: PmNode[]; + text?: string; + marks?: unknown[]; +} + +interface DecodedImage { + buffer: Buffer; + extension: string; +} + +/** + * Imports `.docx`/`.odt` documents as new pages (ADR 0009, issue #63). Enqueue + * is synchronous and cheap (validate + persist a job on the #62 queue); the + * heavy conversion runs out of band in {@link run}, invoked by the shared + * {@link ConversionWorker} for import-kind jobs so it inherits the queue's + * locking, retry, and restart-survival. + * + * Pipeline: the pandoc sidecar is stateless and will not hand back a + * document's embedded media any other way, so we convert in two passes — + * `docx/odt → html` with `embed-resources` inlines every image as a `data:` + * URI, then `html → gfm` produces clean structural Markdown with those data + * URIs still inline at the right positions. We parse that to an editor + * document, store each embedded image as a pond file (with quota accounting) + * and rewrite its reference to the file id, derive the title from a leading + * top-level heading (else the file name), and create the page from the + * resulting Yjs state. + */ +@Injectable() +export class ImportService implements ImportProcessor { + constructor( + private readonly prisma: PrismaService, + private readonly jobs: ConversionJobService, + private readonly converter: PandocConverter, + private readonly files: FilesService, + private readonly pages: PagesService, + private readonly logger: PinoLogger, + ) { + this.logger.setContext(ImportService.name); + } + + /** Validate and enqueue an import; the returned job id is polled via + * `GET /jobs/:id` until it reports the created `resultPageId` (#62). */ + async enqueue( + user: User, + pondId: string, + file: { buffer: Buffer; originalname: string }, + ): Promise { + const extension = fileExtension(file.originalname); + const format = extension ? IMPORT_FORMATS[extension] : undefined; + if (!format) { + throw new BadRequestException({ code: 'import_unsupported_format' }); + } + const job = await this.jobs.enqueue({ + ownerId: user.id, + pondId, + kind: importKind(format), + from: format, + to: 'page', + input: file.buffer, + sourceName: file.originalname, + standalone: false, + }); + return this.jobs.viewOf(job); + } + + /** + * Run one import job to completion (called by the worker). Throws a + * {@link ConversionError} on failure so the worker applies the queue's + * retry/fail policy; on success it records the created page on the job. + * Media stored during a failed attempt is rolled back so a retry does not + * leak files or double-count quota. + */ + async run(job: ConversionJob): Promise { + if (!job.pondId) { + throw new ConversionError('conversion_failed', false, 'import job has no pond'); + } + const user = await this.prisma.user.findUnique({ where: { id: job.ownerId } }); + if (!user) { + throw new ConversionError('conversion_failed', false, 'import job owner is gone'); + } + + const rawMarkdown = await convertImportedDocument( + this.converter, + job.sourceFormat, + Buffer.from(job.input), + ); + + // Media is stored before the page exists (the page's state references the + // file ids); track what we create so a later failure can be rolled back. + const storedFileIds: string[] = []; + try { + const markdown = await this.storeEmbeddedImages(rawMarkdown, user, job.pondId, storedFileIds); + const json = markdownToDoc(markdown).toJSON() as unknown as PmNode; + const { title, doc } = this.splitTitle(json, job.sourceName); + const state = docToState(Node.fromJSON(editorSchema, doc)); + + const page = await this.pages.createWithState(user, job.pondId, title, state); + await this.files.linkAttachmentsToPage(storedFileIds, page.id); + await this.prisma.conversionJob.update({ + where: { id: job.id }, + data: { status: 'SUCCEEDED', resultPageId: page.id, errorCode: null }, + }); + this.logger.info( + { jobId: job.id, pageId: page.id, pondId: job.pondId, images: storedFileIds.length }, + 'audit: document imported', + ); + } catch (error) { + await this.rollbackMedia(user, storedFileIds); + throw error; + } + } + + /** + * Store each embedded image (an inline `data:` URI, from the embed-resources + * pass) as a pond file and rewrite the Markdown reference to the stored file + * id. Done on the Markdown text, before parsing, because the editor's parser + * only admits a whitelist of `data:` image types (png/jpeg/gif/webp) and + * would leave any other inline as a huge literal base64 string. base64 has no + * `)`, so the image regex is unambiguous. An image whose bytes the upload + * pipeline rejects (e.g. a vector `image/x-emf` Word can embed) is dropped + * rather than failing the whole import — structure over layout (ADR 0009); a + * pond that runs out of storage fails the import. + */ + private async storeEmbeddedImages( + markdown: string, + user: User, + pondId: string, + storedFileIds: string[], + ): Promise { + const image = /!\[([^\]]*)\]\((data:[^)\s]+)\)/g; + const uris = new Set(); + for (const match of markdown.matchAll(image)) uris.add(match[2]!); + if (uris.size === 0) return markdown; + + // Dedup identical images so a document that repeats one stores it once. + const resolved = new Map(); + let index = 0; + for (const uri of uris) { + resolved.set(uri, await this.storeImage(uri, user, pondId, ++index, storedFileIds)); + } + return markdown.replace(image, (_whole, alt: string, uri: string) => { + const fileId = resolved.get(uri); + return fileId ? `![${alt}](${fileId})` : ''; + }); + } + + private async storeImage( + uri: string, + user: User, + pondId: string, + index: number, + storedFileIds: string[], + ): Promise { + const decoded = decodeDataUri(uri); + if (!decoded) return null; + try { + const view = await this.files.upload(user, pondId, { + buffer: decoded.buffer, + size: decoded.buffer.length, + originalname: `import-${index}.${decoded.extension}`, + }); + storedFileIds.push(view.id); + return view.id; + } catch (error) { + if (error instanceof ForbiddenException && errorCode(error) === 'quota_exceeded') { + throw new ConversionError('quota_exceeded', false, 'pond storage exhausted during import'); + } + // Unsupported image type / rejected bytes — drop this one, keep going. + this.logger.warn({ pondId, code: errorCode(error) }, 'import: dropped an image'); + return null; + } + } + + /** + * Use a leading top-level heading as the page title (and remove it from the + * body so it is not duplicated), else fall back to the upload file name + * without its extension, else a generic title. + */ + private splitTitle(root: PmNode, sourceName: string | null): { title: string; doc: PmNode } { + const content = root.content ?? []; + const first = content[0]; + if (first && first.type === 'heading') { + const headingText = textOf(first).trim(); + if (headingText) { + return { title: headingText, doc: { ...root, content: content.slice(1) } }; + } + } + const fallback = baseName(sourceName) || 'Imported document'; + return { title: fallback, doc: root }; + } + + private async rollbackMedia(user: User, fileIds: string[]): Promise { + for (const id of fileIds) { + try { + await this.files.remove(user, id); + } catch (error) { + this.logger.warn({ fileId: id, code: errorCode(error) }, 'import: media rollback failed'); + } + } + } +} + +/** Concatenated text of a node's inline content (headings have no nesting). */ +function textOf(node: PmNode): string { + if (typeof node.text === 'string') return node.text; + return (node.content ?? []).map(textOf).join(''); +} + +const DATA_URI = /^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/; +const EXTENSION_BY_MIME: Readonly> = { + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'image/gif': 'gif', + 'image/webp': 'webp', + 'image/svg+xml': 'svg', +}; + +function decodeDataUri(value: string): DecodedImage | null { + const match = DATA_URI.exec(value); + if (!match) return null; + const mime = match[1]!; + const buffer = Buffer.from(match[2]!, 'base64'); + if (buffer.length === 0) return null; + return { buffer, extension: EXTENSION_BY_MIME[mime] ?? 'bin' }; +} + +/** Lowercased extension without the dot, or '' when there is none. */ +function fileExtension(fileName: string): string { + const dot = fileName.lastIndexOf('.'); + return dot >= 0 ? fileName.slice(dot + 1).toLowerCase() : ''; +} + +/** File name without its extension. */ +function baseName(fileName: string | null): string { + if (!fileName) return ''; + const dot = fileName.lastIndexOf('.'); + return (dot > 0 ? fileName.slice(0, dot) : fileName).trim(); +} + +/** The `code` from a Nest HttpException response body, if any. */ +function errorCode(error: unknown): string | undefined { + if (error instanceof BadRequestException || error instanceof ForbiddenException) { + const body = error.getResponse(); + if (body && typeof body === 'object' && 'code' in body) { + const code = (body as { code?: unknown }).code; + return typeof code === 'string' ? code : undefined; + } + } + return undefined; +} diff --git a/apps/api/src/import-export/pandoc.converter.ts b/apps/api/src/import-export/pandoc.converter.ts index 1ee23d9..8993436 100644 --- a/apps/api/src/import-export/pandoc.converter.ts +++ b/apps/api/src/import-export/pandoc.converter.ts @@ -10,6 +10,13 @@ export interface ConversionRequest { to: string; input: Buffer; standalone?: boolean; + /** Inline referenced/embedded resources (images) as `data:` URIs in the + * output. Used by the import pipeline (#63): pandoc-server is stateless and + * will not hand back a document's media bytes any other way. */ + embedResources?: boolean; + /** Line-wrapping of the writer's output. Import uses `none` so a paragraph + * stays on one line (no soft breaks inside image alt text or links). */ + wrap?: 'none' | 'auto' | 'preserve'; } export interface ConversionResult { @@ -18,7 +25,12 @@ export interface ConversionResult { } export type ConversionErrorCode = - 'converter_unavailable' | 'converter_timeout' | 'conversion_failed'; + | 'converter_unavailable' + | 'converter_timeout' + | 'conversion_failed' + // The pond ran out of storage while an import stored the document's media + // (#63) — final, and surfaced the same way through the worker. + | 'quota_exceeded'; /** A conversion failure with a stable, localizable code. `retryable` marks * the transient causes (sidecar down / timed out) the worker retries before @@ -111,6 +123,10 @@ export class PandocServerConverter extends PandocConverter { from: request.from, to: request.to, standalone: request.standalone ?? true, + // pandoc-server uses hyphenated option keys; unknown keys are ignored, + // so these are only present when the import pipeline sets them. + ...(request.embedResources ? { 'embed-resources': true } : {}), + ...(request.wrap ? { wrap: request.wrap } : {}), }), signal: controller.signal, }); diff --git a/apps/api/src/pages/pages.service.ts b/apps/api/src/pages/pages.service.ts index 7e8d540..4a0832f 100644 --- a/apps/api/src/pages/pages.service.ts +++ b/apps/api/src/pages/pages.service.ts @@ -130,23 +130,49 @@ export class PagesService { } async create(user: User, pondId: string, input: CreatePageInput): Promise { + const page = await this.insertPage(user, pondId, input.title, emptyPageState()); + return this.viewOf(page); + } + + /** + * Create a page from a prepared Yjs state (issue #63 import): the imported + * document is already a full Yjs state whose fragment the editor binds to, so + * an opening client sees the converted content immediately. Same invariants + * as {@link create} — unique slug, appended sort key, derived content cache, + * phantom-link resolution, search indexing. Returns the persisted row so the + * caller (the import worker) can link the document's media to it. + */ + async createWithState( + user: User, + pondId: string, + title: string, + state: Uint8Array, + ): Promise { + return this.insertPage(user, pondId, title, state); + } + + private async insertPage( + user: User, + pondId: string, + title: string, + state: Uint8Array, + ): Promise { const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } }); if (!pond) throw new NotFoundException(); - const slug = await this.generateUniqueSlugInPond(pond.id, input.title); + const slug = await this.generateUniqueSlugInPond(pond.id, title); const last = await this.prisma.page.findFirst({ where: { pondId: pond.id }, orderBy: { sortKey: 'desc' }, select: { sortKey: true }, }); const sortKey = generateKeyBetween(last?.sortKey ?? null, null); - const state = emptyPageState(); const content = deriveContent(state); const page = await this.prisma.page.create({ data: { pondId: pond.id, - title: input.title, + title, slug, sortKey, ydocState: state, @@ -156,10 +182,10 @@ export class PagesService { }); // A new page may satisfy phantom wikilinks that referenced its slug (#47). await this.resolvePhantomLinks(pond.id, slug, page.id); - // Index the (empty) page so a title-only match is findable immediately (#49). + // Index the page so a title-only match is findable immediately (#49). await this.search.indexPage(page.id); this.logger.info({ pageId: page.id, pondId: pond.id, userId: user.id }, 'audit: page created'); - return this.viewOf(page); + return page; } /** diff --git a/apps/api/src/pages/yjs-content.ts b/apps/api/src/pages/yjs-content.ts index ff82f5b..cc8dd4e 100644 --- a/apps/api/src/pages/yjs-content.ts +++ b/apps/api/src/pages/yjs-content.ts @@ -32,13 +32,17 @@ function docFromState(state: Uint8Array): Node { } } -/** A fresh Yjs state containing a single empty paragraph. */ -export function emptyPageState(): Uint8Array { +/** + * Encode a ProseMirror document as the initial Yjs state a page is created + * with. Used both for a fresh empty page and for importing a converted + * document (#63) as a page's starting content — the editor binds to the same + * {@link FRAGMENT_NAME}, so an opening client sees exactly this document. + */ +export function docToState(doc: Node): Uint8Array { const ydoc = new Y.Doc(); try { const fragment = ydoc.getXmlFragment(FRAGMENT_NAME); - const emptyDoc = editorSchema.node('doc', null, [editorSchema.node('paragraph')]); - prosemirrorJSONToYXmlFragment(editorSchema, emptyDoc.toJSON(), fragment); + prosemirrorJSONToYXmlFragment(editorSchema, doc.toJSON(), fragment); // Copy into a plain ArrayBuffer-backed view — yjs's own return type is // the wider `Uint8Array`, which Prisma's Bytes input // (`Uint8Array`) does not accept directly. @@ -48,6 +52,11 @@ export function emptyPageState(): Uint8Array { } } +/** A fresh Yjs state containing a single empty paragraph. */ +export function emptyPageState(): Uint8Array { + return docToState(editorSchema.node('doc', null, [editorSchema.node('paragraph')])); +} + export interface DerivedPageContent { plainText: string; markdown: string; diff --git a/eslint.config.mjs b/eslint.config.mjs index 821c14e..d4a6bbf 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -22,7 +22,13 @@ export default tseslint.config( // Plain-Node maintenance scripts (no TypeScript, no bundler). files: ['scripts/**/*.mjs'], languageOptions: { - globals: { console: 'readonly', process: 'readonly', URL: 'readonly' }, + globals: { + console: 'readonly', + process: 'readonly', + URL: 'readonly', + fetch: 'readonly', + Buffer: 'readonly', + }, }, }, { diff --git a/fixtures/import/README.md b/fixtures/import/README.md new file mode 100644 index 0000000..99b3e18 --- /dev/null +++ b/fixtures/import/README.md @@ -0,0 +1,42 @@ +# Import fixture corpus (issue #63, ADR 0009) + +Representative `.docx`/`.odt` documents and the Markdown our import pipeline is +expected to produce from them. `import.fixtures.test.ts` runs the real two-pass +pandoc conversion over each and asserts the result, so a change in behaviour +(ours or pandoc's) surfaces as a snapshot diff to review — not a silent +regression. + +## Files + +For each document `` and format `` (`docx`, `odt`): + +- `.` — the source document. +- `..expected.md` — the Markdown the pipeline produces, with every + embedded image's `data:` URI normalised to the literal `data:embedded-image` + (the base64 payload is volatile and not what the snapshot is pinning). + +`article` covers headings, paragraphs, **bold**/_italic_, a link, a bullet list +with nesting, an ordered list, an embedded image, and a table. `formatting` +covers strikethrough, inline code, a blockquote, and three levels of nesting. + +## Fidelity notes (structure, not layout — ADR 0009) + +- **ODT images lose their alt text** and **ODT tables lose their header row**: + pandoc's HTML→ODT _writer_ does not encode either, so the source documents + genuinely lack them. The DOCX variants keep both. This is a pinned pandoc + limitation, not an import bug — extend the corpus rather than chasing it. + +## Limits (ADR 0009) + +An uploaded document may be at most **25 MiB** (`MAX_CONVERSION_INPUT_BYTES`), +and each of the two conversion passes may run for at most **60 s** +(`CONVERSION_TIMEOUT_MS`). A ~50-page document converts well inside that ceiling +(`import.fixtures.test.ts` asserts it); a document that exceeds the timeout fails +the job with `converter_timeout`. + +## Regenerating + +The documents and snapshots are generated with the **pinned** `pandoc/core:3.6` +(the production sidecar) so the snapshots match CI; a different pandoc version +wraps lists and pads tables differently. `scripts/gen-import-fixtures.mjs` +regenerates everything against a reachable pandoc sidecar (`PANDOC_URL`). diff --git a/fixtures/import/article.docx b/fixtures/import/article.docx new file mode 100644 index 0000000..e39b190 Binary files /dev/null and b/fixtures/import/article.docx differ diff --git a/fixtures/import/article.docx.expected.md b/fixtures/import/article.docx.expected.md new file mode 100644 index 0000000..32e1945 --- /dev/null +++ b/fixtures/import/article.docx.expected.md @@ -0,0 +1,29 @@ +# Field Notes + +An introduction with **bold**, *italic*, and a [link to the pond](https://example.com/pond). + +## Observations + +The pond hosts several species. Notable ones include an image below. + +![A calm pond](data:embedded-image) + +## Lists + +- Amphibians + - Frogs + - Newts +- Insects + - Dragonflies + - Water striders + +1. Spring survey +2. Summer survey +3. Autumn survey + +## Measurements + +| Month | Depth (cm) | +|-------|------------| +| April | 120 | +| July | 95 | diff --git a/fixtures/import/article.odt b/fixtures/import/article.odt new file mode 100644 index 0000000..5c0adbc Binary files /dev/null and b/fixtures/import/article.odt differ diff --git a/fixtures/import/article.odt.expected.md b/fixtures/import/article.odt.expected.md new file mode 100644 index 0000000..3a4d13f --- /dev/null +++ b/fixtures/import/article.odt.expected.md @@ -0,0 +1,32 @@ +# Field Notes + +An introduction with **bold**, *italic*, and a [link to the pond](https://example.com/pond). + +## Observations + +The pond hosts several species. Notable ones include an image below. + +![](data:embedded-image) + +## Lists + +- Amphibians + + - Frogs + - Newts + +- Insects + + - Dragonflies + - Water striders + +1. Spring survey +2. Summer survey +3. Autumn survey + +## Measurements + +| | | +|-------|-----| +| April | 120 | +| July | 95 | diff --git a/fixtures/import/article.src.html b/fixtures/import/article.src.html new file mode 100644 index 0000000..6822bbf --- /dev/null +++ b/fixtures/import/article.src.html @@ -0,0 +1,21 @@ +

Field Notes

+

An introduction with bold, italic, and a link to the pond.

+

Observations

+

The pond hosts several species. Notable ones include an image below.

+

A calm pond

+

Lists

+
    +
  • Amphibians +
    • Frogs
    • Newts
  • +
  • Insects +
    • Dragonflies
    • Water striders
  • +
+
  1. Spring survey
  2. Summer survey
  3. Autumn survey
+

Measurements

+ + + + + + +
MonthDepth (cm)
April120
July95
diff --git a/fixtures/import/formatting.docx b/fixtures/import/formatting.docx new file mode 100644 index 0000000..1859b1b Binary files /dev/null and b/fixtures/import/formatting.docx differ diff --git a/fixtures/import/formatting.docx.expected.md b/fixtures/import/formatting.docx.expected.md new file mode 100644 index 0000000..754fd21 --- /dev/null +++ b/fixtures/import/formatting.docx.expected.md @@ -0,0 +1,13 @@ +# Formatting Sampler + +This paragraph mixes **bold**, *italic*, ~~strikethrough~~, and `inline code`. + +> A quoted remark about still water. + +### Nested detail + +- Top level + - Second level + - Third level + +A closing line with another [external link](https://example.org). diff --git a/fixtures/import/formatting.odt b/fixtures/import/formatting.odt new file mode 100644 index 0000000..c3f9824 Binary files /dev/null and b/fixtures/import/formatting.odt differ diff --git a/fixtures/import/formatting.odt.expected.md b/fixtures/import/formatting.odt.expected.md new file mode 100644 index 0000000..dde485e --- /dev/null +++ b/fixtures/import/formatting.odt.expected.md @@ -0,0 +1,15 @@ +# Formatting Sampler + +This paragraph mixes **bold**, *italic*, ~~strikethrough~~, and `inline code`. + +> A quoted remark about still water. + +### Nested detail + +- Top level + + - Second level + + - Third level + +A closing line with another [external link](https://example.org). diff --git a/fixtures/import/formatting.src.html b/fixtures/import/formatting.src.html new file mode 100644 index 0000000..6695e81 --- /dev/null +++ b/fixtures/import/formatting.src.html @@ -0,0 +1,6 @@ +

Formatting Sampler

+

This paragraph mixes bold, italic, strikethrough, and inline code.

+

A quoted remark about still water.

+

Nested detail

+
  • Top level
    • Second level
      • Third level
+

A closing line with another external link.

diff --git a/packages/shared/i18n/de/errors.json b/packages/shared/i18n/de/errors.json index 1e99c26..bb0fd2f 100644 --- a/packages/shared/i18n/de/errors.json +++ b/packages/shared/i18n/de/errors.json @@ -34,6 +34,7 @@ "converter_unavailable": "Der Dokument-Konverter ist derzeit nicht verfügbar. Bitte versuche es später erneut.", "converter_timeout": "Die Konvertierung hat zu lange gedauert und wurde abgebrochen.", "conversion_failed": "Dieses Dokument konnte nicht konvertiert werden.", + "import_unsupported_format": "Nur Word- (.docx) und OpenDocument-Dokumente (.odt) können importiert werden.", "network": "Der Server war nicht erreichbar.", "grant_exists": "Diese Berechtigung existiert bereits.", "grant_pond_admin_scope": "Eine Teich-Admin-Berechtigung muss für den ganzen Teich und eine bestimmte Person gelten.", diff --git a/packages/shared/i18n/en/errors.json b/packages/shared/i18n/en/errors.json index 6e1258b..6f90499 100644 --- a/packages/shared/i18n/en/errors.json +++ b/packages/shared/i18n/en/errors.json @@ -34,6 +34,7 @@ "converter_unavailable": "The document converter is currently unavailable. Please try again later.", "converter_timeout": "The conversion took too long and was cancelled.", "conversion_failed": "This document could not be converted.", + "import_unsupported_format": "Only Word (.docx) and OpenDocument (.odt) documents can be imported.", "network": "The server could not be reached.", "grant_exists": "This grant already exists.", "grant_pond_admin_scope": "A Pond Admin grant must apply to the whole pond and a specific user.", diff --git a/packages/shared/src/conversion.ts b/packages/shared/src/conversion.ts index c42b9ea..c80a8bc 100644 --- a/packages/shared/src/conversion.ts +++ b/packages/shared/src/conversion.ts @@ -12,8 +12,17 @@ export interface ConversionJobView { sourceFormat: string; targetFormat: string; /** Set only when `status` is `failed` — a code from the errors namespace - * (`converter_unavailable` | `converter_timeout` | `conversion_failed`). */ + * (`converter_unavailable` | `converter_timeout` | `conversion_failed` | + * `quota_exceeded`). */ errorCode: string | null; + /** Set once an import job (#63) succeeds: the id of the page it created, so + * the client can navigate to it. `null` for a pending/failed import and for + * plain byte→byte conversions (export). */ + resultPageId: string | null; createdAt: string; updatedAt: string; } + +/** Extensions the import endpoint accepts (ADR 0009, issue #63). */ +export const IMPORT_EXTENSIONS = ['docx', 'odt'] as const; +export type ImportExtension = (typeof IMPORT_EXTENSIONS)[number]; diff --git a/scripts/gen-import-fixtures.mjs b/scripts/gen-import-fixtures.mjs new file mode 100644 index 0000000..13b2397 --- /dev/null +++ b/scripts/gen-import-fixtures.mjs @@ -0,0 +1,69 @@ +#!/usr/bin/env node +// Regenerate the import fixture corpus (issue #63, ADR 0009) from the committed +// `*.src.html` sources: for each source, write `.docx`, `.odt`, and +// the expected Markdown snapshot for each format. Run against a reachable pandoc +// sidecar (the pinned `pandoc/core:3.6`, so snapshots match CI): +// +// docker run --rm -p 3030:3030 pandoc/core:3.6 server +// PANDOC_URL=http://localhost:3030 node scripts/gen-import-fixtures.mjs +// +// The pipeline mirrors ImportService: `html → ` builds the document, then +// ` → html` (embed-resources) → `html → gfm` reproduces what an import +// would parse, with image data URIs normalised to a stable token. + +import { readFileSync, writeFileSync, readdirSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const FIXTURES = join(dirname(fileURLToPath(import.meta.url)), '..', 'fixtures', 'import'); +const PANDOC_URL = process.env.PANDOC_URL ?? 'http://localhost:3030'; +const MARKDOWN_FORMAT = 'gfm-implicit_figures-raw_html'; + +/** POST a conversion to pandoc-server. Without an `Accept` header the server + * returns text for text writers and raw bytes for binary ones (docx/odt). */ +async function pandoc(params, binaryOut = false) { + const res = await fetch(`${PANDOC_URL}/`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(params), + }); + if (!res.ok) throw new Error(`pandoc ${res.status}: ${await res.text()}`); + return binaryOut ? Buffer.from(await res.arrayBuffer()) : res.text(); +} + +async function buildDocument(html, ext) { + return pandoc({ text: html, from: 'html', to: ext, standalone: true }, true); +} + +async function toExpectedMarkdown(documentBytes, ext) { + const embedded = await pandoc({ + text: documentBytes.toString('base64'), + from: ext, + to: 'html', + standalone: false, + 'embed-resources': true, + }); + const md = await pandoc({ + text: embedded, + from: 'html', + to: MARKDOWN_FORMAT, + standalone: false, + wrap: 'none', + }); + return md.replace(/data:image\/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g, 'data:embedded-image'); +} + +const sources = readdirSync(FIXTURES).filter((f) => f.endsWith('.src.html')); +for (const source of sources) { + const name = source.replace(/\.src\.html$/, ''); + const html = readFileSync(join(FIXTURES, source), 'utf8'); + for (const ext of ['docx', 'odt']) { + const bytes = await buildDocument(html, ext); + writeFileSync(join(FIXTURES, `${name}.${ext}`), bytes); + writeFileSync( + join(FIXTURES, `${name}.${ext}.expected.md`), + await toExpectedMarkdown(bytes, ext), + ); + console.log(`wrote ${name}.${ext} (+ expected.md)`); + } +}