import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/common'; import { ConversionJob, User } from '@prisma/client'; import { ConversionJobView, editorSchema, markdownToDoc } from '@dorfteich/shared'; import { Node } from 'prosemirror-model'; import { PinoLogger } from 'nestjs-pino'; import { FilesService } from '../files/files.service'; import { PagesService } from '../pages/pages.service'; import { docToState } from '../pages/yjs-content'; import { PrismaService } from '../prisma/prisma.service'; import { ImportProcessor } from './import.constants'; import { ConversionError, PandocConverter } from './pandoc.converter'; import { ConversionJobService } from './conversion-job.service'; /** pandoc source format per accepted upload extension (ADR 0009). */ const IMPORT_FORMATS: Readonly> = { docx: 'docx', odt: 'odt', }; /** Job `kind` per source format, so the worker can route the job to the import * pipeline (a plain byte→byte conversion has a different kind, #62/#65). */ function importKind(format: string): string { return `import_${format}`; } /** * Writer format for the structural pass. GFM matches our editor schema * (tables, task lists, strikethrough); `-implicit_figures` keeps images as * inline `![alt](src)` instead of wrapping them in a figure with a duplicated * caption; `-raw_html` stops pandoc emitting raw HTML for constructs it can't * represent (our Markdown parser runs with `html:false` and would drop it * anyway — per ADR 0009 we preserve structure, not layout). */ const IMPORT_MARKDOWN_FORMAT = 'gfm-implicit_figures-raw_html'; /** * The two-pass conversion at the heart of import, extracted so tests can run it * over the fixture corpus against a real sidecar. pandoc-server is stateless and * hands back a document's media no other way, so we first inline every image as * a `data:` URI (`source → html`, embed-resources), then produce clean * structural Markdown that still carries those data URIs inline at the right * positions (`html → gfm`, no wrapping). */ export async function convertImportedDocument( converter: PandocConverter, sourceFormat: string, input: Buffer, ): Promise { const html = await converter.convert({ from: sourceFormat, to: 'html', input, standalone: false, embedResources: true, }); const md = await converter.convert({ from: 'html', to: IMPORT_MARKDOWN_FORMAT, input: html.output, standalone: false, wrap: 'none', }); return md.output.toString('utf8'); } /** Minimal shape of a ProseMirror document as JSON — enough to walk it for * image nodes and the leading heading without pulling in prosemirror types. */ interface PmNode { type: string; attrs?: Record; content?: PmNode[]; text?: string; marks?: unknown[]; } interface DecodedImage { buffer: Buffer; extension: string; } /** * Imports `.docx`/`.odt` documents as new pages (ADR 0009, issue #63). Enqueue * is synchronous and cheap (validate + persist a job on the #62 queue); the * heavy conversion runs out of band in {@link run}, invoked by the shared * {@link ConversionWorker} for import-kind jobs so it inherits the queue's * locking, retry, and restart-survival. * * Pipeline: the pandoc sidecar is stateless and will not hand back a * document's embedded media any other way, so we convert in two passes — * `docx/odt → html` with `embed-resources` inlines every image as a `data:` * URI, then `html → gfm` produces clean structural Markdown with those data * URIs still inline at the right positions. We parse that to an editor * document, store each embedded image as a pond file (with quota accounting) * and rewrite its reference to the file id, derive the title from a leading * top-level heading (else the file name), and create the page from the * resulting Yjs state. */ @Injectable() export class ImportService implements ImportProcessor { constructor( private readonly prisma: PrismaService, private readonly jobs: ConversionJobService, private readonly converter: PandocConverter, private readonly files: FilesService, private readonly pages: PagesService, private readonly logger: PinoLogger, ) { this.logger.setContext(ImportService.name); } /** Validate and enqueue an import; the returned job id is polled via * `GET /jobs/:id` until it reports the created `resultPageId` (#62). */ async enqueue( user: User, pondId: string, file: { buffer: Buffer; originalname: string }, ): Promise { const extension = fileExtension(file.originalname); const format = extension ? IMPORT_FORMATS[extension] : undefined; if (!format) { throw new BadRequestException({ code: 'import_unsupported_format' }); } const job = await this.jobs.enqueue({ ownerId: user.id, pondId, kind: importKind(format), from: format, to: 'page', input: file.buffer, sourceName: file.originalname, standalone: false, }); return this.jobs.viewOf(job); } /** * Run one import job to completion (called by the worker). Throws a * {@link ConversionError} on failure so the worker applies the queue's * retry/fail policy; on success it records the created page on the job. * Media stored during a failed attempt is rolled back so a retry does not * leak files or double-count quota. */ async run(job: ConversionJob): Promise { if (!job.pondId) { throw new ConversionError('conversion_failed', false, 'import job has no pond'); } const user = await this.prisma.user.findUnique({ where: { id: job.ownerId } }); if (!user) { throw new ConversionError('conversion_failed', false, 'import job owner is gone'); } const rawMarkdown = await convertImportedDocument( this.converter, job.sourceFormat, Buffer.from(job.input), ); // Media is stored before the page exists (the page's state references the // file ids); track what we create so a later failure can be rolled back. const storedFileIds: string[] = []; try { const markdown = await this.storeEmbeddedImages(rawMarkdown, user, job.pondId, storedFileIds); const json = markdownToDoc(markdown).toJSON() as unknown as PmNode; const { title, doc } = this.splitTitle(json, job.sourceName); const state = docToState(Node.fromJSON(editorSchema, doc)); const page = await this.pages.createWithState(user, job.pondId, title, state); await this.files.linkAttachmentsToPage(storedFileIds, page.id); await this.prisma.conversionJob.update({ where: { id: job.id }, data: { status: 'SUCCEEDED', resultPageId: page.id, errorCode: null }, }); this.logger.info( { jobId: job.id, pageId: page.id, pondId: job.pondId, images: storedFileIds.length }, 'audit: document imported', ); } catch (error) { await this.rollbackMedia(user, storedFileIds); throw error; } } /** * Store each embedded image (an inline `data:` URI, from the embed-resources * pass) as a pond file and rewrite the Markdown reference to the stored file * id. Done on the Markdown text, before parsing, because the editor's parser * only admits a whitelist of `data:` image types (png/jpeg/gif/webp) and * would leave any other inline as a huge literal base64 string. base64 has no * `)`, so the image regex is unambiguous. An image whose bytes the upload * pipeline rejects (e.g. a vector `image/x-emf` Word can embed) is dropped * rather than failing the whole import — structure over layout (ADR 0009); a * pond that runs out of storage fails the import. */ private async storeEmbeddedImages( markdown: string, user: User, pondId: string, storedFileIds: string[], ): Promise { const image = /!\[([^\]]*)\]\((data:[^)\s]+)\)/g; const uris = new Set(); for (const match of markdown.matchAll(image)) uris.add(match[2]!); if (uris.size === 0) return markdown; // Dedup identical images so a document that repeats one stores it once. const resolved = new Map(); let index = 0; for (const uri of uris) { resolved.set(uri, await this.storeImage(uri, user, pondId, ++index, storedFileIds)); } return markdown.replace(image, (_whole, alt: string, uri: string) => { const fileId = resolved.get(uri); return fileId ? `![${alt}](${fileId})` : ''; }); } private async storeImage( uri: string, user: User, pondId: string, index: number, storedFileIds: string[], ): Promise { const decoded = decodeDataUri(uri); if (!decoded) return null; try { const view = await this.files.upload(user, pondId, { buffer: decoded.buffer, size: decoded.buffer.length, originalname: `import-${index}.${decoded.extension}`, }); storedFileIds.push(view.id); return view.id; } catch (error) { if (error instanceof ForbiddenException && errorCode(error) === 'quota_exceeded') { throw new ConversionError('quota_exceeded', false, 'pond storage exhausted during import'); } // Unsupported image type / rejected bytes — drop this one, keep going. this.logger.warn({ pondId, code: errorCode(error) }, 'import: dropped an image'); return null; } } /** * Use a leading top-level heading as the page title (and remove it from the * body so it is not duplicated), else fall back to the upload file name * without its extension, else a generic title. */ private splitTitle(root: PmNode, sourceName: string | null): { title: string; doc: PmNode } { const content = root.content ?? []; const first = content[0]; if (first && first.type === 'heading') { const headingText = textOf(first).trim(); if (headingText) { return { title: headingText, doc: { ...root, content: content.slice(1) } }; } } const fallback = baseName(sourceName) || 'Imported document'; return { title: fallback, doc: root }; } private async rollbackMedia(user: User, fileIds: string[]): Promise { for (const id of fileIds) { try { await this.files.remove(user, id); } catch (error) { this.logger.warn({ fileId: id, code: errorCode(error) }, 'import: media rollback failed'); } } } } /** Concatenated text of a node's inline content (headings have no nesting). */ function textOf(node: PmNode): string { if (typeof node.text === 'string') return node.text; return (node.content ?? []).map(textOf).join(''); } const DATA_URI = /^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/; const EXTENSION_BY_MIME: Readonly> = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/gif': 'gif', 'image/webp': 'webp', 'image/svg+xml': 'svg', }; function decodeDataUri(value: string): DecodedImage | null { const match = DATA_URI.exec(value); if (!match) return null; const mime = match[1]!; const buffer = Buffer.from(match[2]!, 'base64'); if (buffer.length === 0) return null; return { buffer, extension: EXTENSION_BY_MIME[mime] ?? 'bin' }; } /** Lowercased extension without the dot, or '' when there is none. */ function fileExtension(fileName: string): string { const dot = fileName.lastIndexOf('.'); return dot >= 0 ? fileName.slice(dot + 1).toLowerCase() : ''; } /** File name without its extension. */ function baseName(fileName: string | null): string { if (!fileName) return ''; const dot = fileName.lastIndexOf('.'); return (dot > 0 ? fileName.slice(0, dot) : fileName).trim(); } /** The `code` from a Nest HttpException response body, if any. */ function errorCode(error: unknown): string | undefined { if (error instanceof BadRequestException || error instanceof ForbiddenException) { const body = error.getResponse(); if (body && typeof body === 'object' && 'code' in body) { const code = (body as { code?: unknown }).code; return typeof code === 'string' ? code : undefined; } } return undefined; }