Import .docx and .odt documents as new pages (#63)
All checks were successful
CD / Build and push images (push) Successful in 3m19s
CI / Lint, typecheck, test (push) Successful in 2m55s
CI / Auth e2e pack (push) Successful in 3m45s
CI / Build container images (push) Has been skipped
CD / Deploy to Test (push) Successful in 9s
CD / Smoke tests against Test (push) Successful in 1m13s
CD / Promote to Int (push) Successful in 11s
All checks were successful
CD / Build and push images (push) Successful in 3m19s
CI / Lint, typecheck, test (push) Successful in 2m55s
CI / Auth e2e pack (push) Successful in 3m45s
CI / Build container images (push) Has been skipped
CD / Deploy to Test (push) Successful in 9s
CD / Smoke tests against Test (push) Successful in 1m13s
CD / Promote to Int (push) Successful in 11s
Uploading a Word/OpenOffice document to POST /ponds/:id/import enqueues a conversion job (the #62 queue) that produces a new page in the pond; the client polls GET /jobs/:id for the created resultPageId. Pipeline (ImportService, ADR 0009): pandoc-server is stateless and hands back a document's media no other way, so we convert in two passes — docx/odt → html with embed-resources inlines every image as a data: URI, then html → gfm produces clean structural Markdown with those data URIs still inline. Embedded images are stored as pond files (with quota accounting) and their references rewritten to file ids on the Markdown text before parsing (the editor parser only admits png/jpeg/gif/webp data URIs); an image whose bytes the upload pipeline rejects is dropped, not fatal. The title comes from a leading top-level heading (removed from the body) else the file name. The page is created from the resulting Yjs state. The shared conversion worker routes import-kind jobs to the pipeline via a token (breaking a module cycle), so import inherits the queue's locking, retry, and restart-survival. Media stored during a failed attempt is rolled back; a pond that runs out of storage fails the job with quota_exceeded. - schema: ConversionJob gains pond_id / source_name / result_page_id (migration 20260710041215_import_pages_conversion); ConversionJobView gains resultPageId. - PagesService.createWithState / yjs-content docToState build a page from a prepared document; FilesService.linkAttachmentsToPage links import media. - fixtures/import/: representative .docx/.odt corpus (headings, lists, nested lists, tables, images, links, bold/italic) with expected-Markdown snapshots; scripts/gen-import-fixtures.mjs regenerates them. - tests: import.service.db.test.ts drives the full pipeline with a fake converter (CI); import.fixtures.test.ts runs the real two-pass conversion over the corpus and a 50-page timing check against a reachable sidecar. - i18n: import_unsupported_format (de+en). Limits documented (25 MiB input, 60 s per pass). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1
This commit is contained in:
parent
4755c18ef5
commit
546e8279ac
@ -6,3 +6,8 @@ pnpm-lock.yaml
|
||||
# Must stay byte-identical to docToMarkdown's own output (issue #32) —
|
||||
# Prettier's Markdown table/list opinions would break that fixed point.
|
||||
apps/api/prisma/fixtures/content-page.md
|
||||
# Import fixture corpus (issue #63): the expected-Markdown snapshots and the
|
||||
# HTML sources must stay exactly as pandoc produces/consumes them — Prettier's
|
||||
# Markdown/HTML opinions would break the regression tests.
|
||||
fixtures/import/*.expected.md
|
||||
fixtures/import/*.src.html
|
||||
|
||||
@ -0,0 +1,10 @@
|
||||
-- AlterTable
|
||||
ALTER TABLE "conversion_jobs" ADD COLUMN "pond_id" TEXT,
|
||||
ADD COLUMN "result_page_id" TEXT,
|
||||
ADD COLUMN "source_name" TEXT;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "conversion_jobs" ADD CONSTRAINT "conversion_jobs_pond_id_fkey" FOREIGN KEY ("pond_id") REFERENCES "ponds"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "conversion_jobs" ADD CONSTRAINT "conversion_jobs_result_page_id_fkey" FOREIGN KEY ("result_page_id") REFERENCES "pages"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
@ -81,6 +81,7 @@ model Pond {
|
||||
attachments Attachment[]
|
||||
labels Label[]
|
||||
grants RoleGrant[]
|
||||
conversionJobs ConversionJob[]
|
||||
|
||||
@@index([ownerId])
|
||||
@@map("ponds")
|
||||
@ -164,6 +165,7 @@ model Page {
|
||||
labels PageLabel[]
|
||||
outgoingLinks PageLink[] @relation("outgoingLinks")
|
||||
incomingLinks PageLink[] @relation("incomingLinks")
|
||||
conversionJobs ConversionJob[]
|
||||
|
||||
@@unique([pondId, slug])
|
||||
@@index([pondId])
|
||||
@ -551,7 +553,16 @@ model ConversionJob {
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
owner User @relation(fields: [ownerId], references: [id], onDelete: Cascade)
|
||||
/// Import jobs (#63) carry the pond the document is imported into, the
|
||||
/// original upload file name (title fallback), and the page they produced.
|
||||
/// All null for a plain byte→byte conversion (export, #62/#65).
|
||||
pondId String? @map("pond_id")
|
||||
sourceName String? @map("source_name")
|
||||
resultPageId String? @map("result_page_id")
|
||||
|
||||
owner User @relation(fields: [ownerId], references: [id], onDelete: Cascade)
|
||||
pond Pond? @relation(fields: [pondId], references: [id], onDelete: Cascade)
|
||||
page Page? @relation(fields: [resultPageId], references: [id], onDelete: SetNull)
|
||||
|
||||
@@index([status, createdAt])
|
||||
@@map("conversion_jobs")
|
||||
|
||||
@ -11,6 +11,6 @@ import { FilesService } from './files.service';
|
||||
imports: [PondsModule, QuotasModule],
|
||||
controllers: [FilesController],
|
||||
providers: [FilesService, FileStorageService],
|
||||
exports: [FileStorageService],
|
||||
exports: [FileStorageService, FilesService],
|
||||
})
|
||||
export class FilesModule {}
|
||||
|
||||
@ -169,6 +169,21 @@ export class FilesService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Point a set of just-uploaded attachments at the page that now embeds them
|
||||
* (issue #63 import). The import worker stores a document's media before the
|
||||
* page exists (the page's Yjs state references their ids), then calls this so
|
||||
* they list under the page and are purged with it (#31), the same invariant
|
||||
* the collab persistence hook maintains for pasted images.
|
||||
*/
|
||||
async linkAttachmentsToPage(attachmentIds: string[], pageId: string): Promise<void> {
|
||||
if (attachmentIds.length === 0) return;
|
||||
await this.prisma.attachment.updateMany({
|
||||
where: { id: { in: attachmentIds } },
|
||||
data: { pageId },
|
||||
});
|
||||
}
|
||||
|
||||
/** Upload against the pond of `pageId`, linked to that page (#61). */
|
||||
async uploadToPage(
|
||||
user: User,
|
||||
|
||||
@ -15,6 +15,10 @@ export interface EnqueueConversion {
|
||||
to: string;
|
||||
input: Buffer;
|
||||
standalone?: boolean;
|
||||
/** Set for import jobs (#63): the pond the document is imported into and the
|
||||
* original upload file name (a title fallback). */
|
||||
pondId?: string;
|
||||
sourceName?: string;
|
||||
}
|
||||
|
||||
export interface ConversionResultPayload {
|
||||
@ -55,6 +59,8 @@ export class ConversionJobService {
|
||||
const job = await this.prisma.conversionJob.create({
|
||||
data: {
|
||||
ownerId: request.ownerId,
|
||||
pondId: request.pondId ?? null,
|
||||
sourceName: request.sourceName ?? null,
|
||||
kind: request.kind,
|
||||
sourceFormat: request.from,
|
||||
targetFormat: request.to,
|
||||
@ -103,6 +109,8 @@ export class ConversionJobService {
|
||||
sourceFormat: job.sourceFormat,
|
||||
targetFormat: job.targetFormat,
|
||||
errorCode: job.errorCode,
|
||||
// Set once an import job succeeds (#63) so the client can open the page.
|
||||
resultPageId: job.resultPageId,
|
||||
createdAt: job.createdAt.toISOString(),
|
||||
updatedAt: job.updatedAt.toISOString(),
|
||||
};
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
|
||||
import { ModuleRef } from '@nestjs/core';
|
||||
import { ConversionJob } from '@prisma/client';
|
||||
import { PinoLogger } from 'nestjs-pino';
|
||||
|
||||
import { AppConfig } from '../config/app-config.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
|
||||
import { IMPORT_PROCESSOR, ImportProcessor, isImportKind } from './import.constants';
|
||||
import { ConversionError, PandocConverter } from './pandoc.converter';
|
||||
|
||||
/** How often the worker sweeps for pending jobs on its own — the safety net
|
||||
@ -38,6 +40,9 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly converter: PandocConverter,
|
||||
private readonly config: AppConfig,
|
||||
private readonly logger: PinoLogger,
|
||||
// Resolved lazily to break the construction cycle (the import service
|
||||
// enqueues via the job service, which wakes this worker).
|
||||
private readonly moduleRef: ModuleRef,
|
||||
) {
|
||||
this.logger.setContext(ConversionWorker.name);
|
||||
}
|
||||
@ -103,6 +108,13 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy {
|
||||
|
||||
private async process(job: ConversionJob): Promise<void> {
|
||||
try {
|
||||
if (isImportKind(job.kind)) {
|
||||
// Import runs a multi-step pipeline and records its own success (the
|
||||
// created page id) on the job (#63). Resolved lazily via a token so the
|
||||
// worker's file never imports the import service's (avoids a cycle).
|
||||
await this.moduleRef.get<ImportProcessor>(IMPORT_PROCESSOR, { strict: false }).run(job);
|
||||
return;
|
||||
}
|
||||
const result = await this.converter.convert({
|
||||
from: job.sourceFormat,
|
||||
to: job.targetFormat,
|
||||
|
||||
@ -1,20 +1,32 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { FilesModule } from '../files/files.module';
|
||||
import { PagesModule } from '../pages/pages.module';
|
||||
|
||||
import { ConversionJobService } from './conversion-job.service';
|
||||
import { ConversionWorker } from './conversion-worker.service';
|
||||
import { IMPORT_PROCESSOR } from './import.constants';
|
||||
import { ImportController } from './import.controller';
|
||||
import { ImportService } from './import.service';
|
||||
import { JobsController } from './jobs.controller';
|
||||
import { PandocConverter, PandocServerConverter } from './pandoc.converter';
|
||||
|
||||
/**
|
||||
* Import/export orchestration (ADR 0009, issue #62): the conversion job queue,
|
||||
* its worker, and the pandoc-server client. Later stories (#63 import, #65
|
||||
* export) add the feature endpoints that enqueue jobs here.
|
||||
* Import/export orchestration (ADR 0009): the conversion job queue, its worker,
|
||||
* the pandoc-server client (#62), and the document import pipeline (#63, which
|
||||
* turns an uploaded `.docx`/`.odt` into a new page). Export (#65) will add its
|
||||
* feature endpoints here too.
|
||||
*/
|
||||
@Module({
|
||||
controllers: [JobsController],
|
||||
imports: [FilesModule, PagesModule],
|
||||
controllers: [JobsController, ImportController],
|
||||
providers: [
|
||||
ConversionJobService,
|
||||
ConversionWorker,
|
||||
ImportService,
|
||||
// The worker resolves the import pipeline through this token (never the
|
||||
// class), so its file does not import the import service's (avoids a cycle).
|
||||
{ provide: IMPORT_PROCESSOR, useExisting: ImportService },
|
||||
// Bind the abstract converter to the HTTP implementation; tests override
|
||||
// this provider with a fake so the queue mechanics need no live sidecar.
|
||||
{ provide: PandocConverter, useClass: PandocServerConverter },
|
||||
|
||||
21
apps/api/src/import-export/import.constants.ts
Normal file
21
apps/api/src/import-export/import.constants.ts
Normal file
@ -0,0 +1,21 @@
|
||||
import { ConversionJob } from '@prisma/client';
|
||||
|
||||
/**
|
||||
* Contract between the shared conversion worker and the import pipeline, kept in
|
||||
* its own module so neither imports the other's file (the worker resolves the
|
||||
* processor lazily via this token, breaking what would otherwise be a cycle:
|
||||
* import.service → conversion-job.service → conversion-worker.service).
|
||||
*/
|
||||
export const IMPORT_PROCESSOR = Symbol('IMPORT_PROCESSOR');
|
||||
|
||||
export interface ImportProcessor {
|
||||
/** Run one import job to completion, recording the created page on the job or
|
||||
* throwing a ConversionError for the worker's retry/fail policy. */
|
||||
run(job: ConversionJob): Promise<void>;
|
||||
}
|
||||
|
||||
/** True for a job the import pipeline processes rather than the plain pandoc
|
||||
* byte→byte path (export, #62/#65). */
|
||||
export function isImportKind(kind: string): boolean {
|
||||
return kind.startsWith('import_');
|
||||
}
|
||||
39
apps/api/src/import-export/import.controller.ts
Normal file
39
apps/api/src/import-export/import.controller.ts
Normal file
@ -0,0 +1,39 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Controller,
|
||||
Param,
|
||||
Post,
|
||||
Req,
|
||||
UploadedFile,
|
||||
UseInterceptors,
|
||||
} from '@nestjs/common';
|
||||
import { FileInterceptor } from '@nestjs/platform-express';
|
||||
import { ConversionJobView, MAX_UPLOAD_PARSE_BYTES } from '@dorfteich/shared';
|
||||
|
||||
import { AuthedRequest } from '../auth/auth.guard';
|
||||
import { RequiresPondRole } from '../permissions/permission.decorators';
|
||||
|
||||
import { ImportService } from './import.service';
|
||||
|
||||
/**
|
||||
* Document import (ADR 0009, issue #63). Uploading a `.docx`/`.odt` enqueues a
|
||||
* conversion job (#62 queue) that produces a new page in the pond; the client
|
||||
* polls `GET /jobs/:id` for `resultPageId`. Creating a page requires pond-wide
|
||||
* editor access (permissions.md, same rule as `POST /ponds/:id/pages`).
|
||||
*/
|
||||
@Controller('ponds/:pondId')
|
||||
export class ImportController {
|
||||
constructor(private readonly imports: ImportService) {}
|
||||
|
||||
@Post('import')
|
||||
@RequiresPondRole('editor', { idParam: 'pondId' })
|
||||
@UseInterceptors(FileInterceptor('file', { limits: { fileSize: MAX_UPLOAD_PARSE_BYTES } }))
|
||||
async import(
|
||||
@Param('pondId') pondId: string,
|
||||
@UploadedFile() file: Express.Multer.File | undefined,
|
||||
@Req() request: AuthedRequest,
|
||||
): Promise<ConversionJobView> {
|
||||
if (!file) throw new BadRequestException({ code: 'bad_request' });
|
||||
return this.imports.enqueue(request.user!, pondId, file);
|
||||
}
|
||||
}
|
||||
78
apps/api/src/import-export/import.fixtures.test.ts
Normal file
78
apps/api/src/import-export/import.fixtures.test.ts
Normal file
@ -0,0 +1,78 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { markdownToDoc } from '@dorfteich/shared';
|
||||
import { beforeAll, describe, expect, it, TestContext } from 'vitest';
|
||||
|
||||
import { AppConfig } from '../config/app-config.service';
|
||||
|
||||
import { convertImportedDocument } from './import.service';
|
||||
import { CONVERSION_TIMEOUT_MS, PandocServerConverter } from './pandoc.converter';
|
||||
|
||||
/**
|
||||
* Import fidelity regression (issue #63, ADR 0009): runs the real two-pass
|
||||
* pandoc conversion over the committed `.docx`/`.odt` corpus and asserts each
|
||||
* produces its expected Markdown. Needs a reachable pandoc sidecar (the pinned
|
||||
* `pandoc/core:3.6`, so output matches the snapshots) — each test skips itself
|
||||
* when none is configured, and CI starts one and points `PANDOC_URL` at it.
|
||||
*/
|
||||
const PANDOC_URL = process.env.PANDOC_URL ?? 'http://localhost:3030';
|
||||
// The test runner's cwd is `apps/api`; the corpus lives at the repo root.
|
||||
const FIXTURES = join(process.cwd(), '../../fixtures/import');
|
||||
|
||||
const converter = new PandocServerConverter({
|
||||
env: { PANDOC_URL },
|
||||
} as unknown as AppConfig);
|
||||
|
||||
let reachable = false;
|
||||
|
||||
/** Normalise embedded image `data:` URIs to the stable token the snapshots use
|
||||
* (the base64 payload is volatile and not what we are pinning). */
|
||||
function normalize(markdown: string): string {
|
||||
return markdown.replace(
|
||||
/data:image\/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g,
|
||||
'data:embedded-image',
|
||||
);
|
||||
}
|
||||
|
||||
const CORPUS = ['article.docx', 'article.odt', 'formatting.docx', 'formatting.odt'];
|
||||
|
||||
describe('import fixture corpus (real pandoc, issue #63)', () => {
|
||||
beforeAll(async () => {
|
||||
reachable = await converter.reachable().catch(() => false);
|
||||
});
|
||||
|
||||
for (const fixture of CORPUS) {
|
||||
it(`converts ${fixture} to its expected Markdown`, async (ctx: TestContext) => {
|
||||
if (!reachable) ctx.skip();
|
||||
const format = fixture.endsWith('.odt') ? 'odt' : 'docx';
|
||||
const document = readFileSync(join(FIXTURES, fixture));
|
||||
const expected = readFileSync(join(FIXTURES, `${fixture}.expected.md`), 'utf8');
|
||||
|
||||
const markdown = await convertImportedDocument(converter, format, document);
|
||||
expect(normalize(markdown)).toBe(expected);
|
||||
|
||||
// The Markdown must also parse into a valid editor document (no schema
|
||||
// surprises from real-world structure).
|
||||
expect(() => markdownToDoc(markdown)).not.toThrow();
|
||||
});
|
||||
}
|
||||
|
||||
it('imports a 50-page document within the conversion timeout', async (ctx: TestContext) => {
|
||||
if (!reachable) ctx.skip();
|
||||
// Build a ~50-page document by repeating a page of structured content, then
|
||||
// convert it to docx once and time the import conversion of that document.
|
||||
const onePage =
|
||||
'# Section\n\n' + 'A paragraph of survey notes about the pond. '.repeat(20) + '\n\n';
|
||||
const large = Array.from({ length: 50 }, () => onePage).join('\n---\n\n');
|
||||
const docx = await converter.convert({ from: 'gfm', to: 'docx', input: Buffer.from(large) });
|
||||
|
||||
const started = Date.now();
|
||||
const markdown = await convertImportedDocument(converter, 'docx', docx.output);
|
||||
const elapsed = Date.now() - started;
|
||||
|
||||
expect(markdown.length).toBeGreaterThan(1000);
|
||||
// Comfortably inside the documented 60 s per-conversion ceiling (ADR 0009).
|
||||
expect(elapsed).toBeLessThan(CONVERSION_TIMEOUT_MS);
|
||||
});
|
||||
});
|
||||
270
apps/api/src/import-export/import.service.db.test.ts
Normal file
270
apps/api/src/import-export/import.service.db.test.ts
Normal file
@ -0,0 +1,270 @@
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import request from 'supertest';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
import { AuthTokensService } from '../auth/auth-tokens.service';
|
||||
import { createTestApp, sessionCookieOf } from '../testing/test-app';
|
||||
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
|
||||
import { UsersService } from '../users/users.service';
|
||||
|
||||
import { ConversionWorker } from './conversion-worker.service';
|
||||
import {
|
||||
ConversionError,
|
||||
ConversionRequest,
|
||||
ConversionResult,
|
||||
PandocConverter,
|
||||
} from './pandoc.converter';
|
||||
|
||||
/**
|
||||
* Document import pipeline (issue #63): upload → conversion job → media stored
|
||||
* as pond files → new page. Driven by an injected fake converter that returns a
|
||||
* chosen Markdown for the structural pass, so the orchestration (media
|
||||
* extraction, quota, title, rollback, failure codes) is tested deterministically
|
||||
* without a live pandoc sidecar. The real conversion of the fixture corpus is
|
||||
* covered against a running sidecar by import.fixtures.test.ts.
|
||||
*/
|
||||
|
||||
/** A 1×1 PNG — passes the raster magic-byte sniff, so it is stored as an image. */
|
||||
const PNG_BASE64 =
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';
|
||||
|
||||
/** A converter whose `markdown` each test sets. The import pipeline calls it
|
||||
* twice (source→html, then html→gfm); the html pass is echoed, the gfm pass
|
||||
* returns the chosen Markdown (its input is irrelevant to these tests). A test
|
||||
* may instead make a pass throw to exercise a conversion failure. */
|
||||
class FakeConverter extends PandocConverter {
|
||||
markdown = '# Untitled\n\nBody.';
|
||||
failHtmlWith: ConversionError | null = null;
|
||||
|
||||
convert(request: ConversionRequest): Promise<ConversionResult> {
|
||||
if (request.to === 'html') {
|
||||
if (this.failHtmlWith) return Promise.reject(this.failHtmlWith);
|
||||
return Promise.resolve({ output: Buffer.from('<html/>'), mimeType: 'text/html' });
|
||||
}
|
||||
return Promise.resolve({ output: Buffer.from(this.markdown), mimeType: 'text/markdown' });
|
||||
}
|
||||
reachable(): Promise<boolean> {
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
}
|
||||
|
||||
describe.skipIf(!hasTestDb)('document import (e2e, issue #63)', () => {
|
||||
let app: INestApplication;
|
||||
let prisma: PrismaClient;
|
||||
let worker: ConversionWorker;
|
||||
let fake: FakeConverter;
|
||||
const suffix = uniqueSuffix();
|
||||
const password = 'importiere meine dokumente 1';
|
||||
|
||||
const owner = { username: `iris-import-${suffix}`, displayName: `Iris Import ${suffix}` };
|
||||
const outsider = { username: `otis-out-${suffix}`, displayName: `Otis Out ${suffix}` };
|
||||
let ownerId: string;
|
||||
let pondId: string;
|
||||
let ownerCookie: string;
|
||||
let outsiderCookie: string;
|
||||
|
||||
const api = () => request(app.getHttpServer());
|
||||
|
||||
async function loginOf(username: string): Promise<string> {
|
||||
const res = await api()
|
||||
.post('/api/v1/auth/login')
|
||||
.send({ usernameOrEmail: username, password })
|
||||
.expect(200);
|
||||
return sessionCookieOf(res);
|
||||
}
|
||||
|
||||
/** Upload a document and drain the queue; returns the finished job view. */
|
||||
async function importDoc(fileName: string, cookie = ownerCookie): Promise<request.Response> {
|
||||
const enqueued = await api()
|
||||
.post(`/api/v1/ponds/${pondId}/import`)
|
||||
.set('Cookie', cookie)
|
||||
.attach('file', Buffer.from('source-document-bytes'), fileName)
|
||||
.expect(201);
|
||||
await worker.drain();
|
||||
return api().get(`/api/v1/jobs/${enqueued.body.id}`).set('Cookie', cookie).expect(200);
|
||||
}
|
||||
|
||||
async function markdownOf(pageId: string): Promise<string> {
|
||||
const cache = await prisma.pageContentCache.findUniqueOrThrow({ where: { pageId } });
|
||||
return cache.markdown;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
prisma = createTestPrisma();
|
||||
await prisma.rateLimit.deleteMany({});
|
||||
fake = new FakeConverter();
|
||||
app = await createTestApp((builder) =>
|
||||
builder.overrideProvider(PandocConverter).useValue(fake),
|
||||
);
|
||||
worker = app.get(ConversionWorker);
|
||||
|
||||
const users = app.get(UsersService);
|
||||
const tokens = app.get(AuthTokensService);
|
||||
const ownerUser = await users.createUser({
|
||||
username: owner.username,
|
||||
email: `${owner.username}@example.org`,
|
||||
displayName: owner.displayName,
|
||||
password,
|
||||
locale: 'en',
|
||||
});
|
||||
ownerId = ownerUser.id;
|
||||
// Verifying the e-mail creates the owner's personal pond (+ owner-admin
|
||||
// grant), which the owner may import into.
|
||||
const verify = await tokens.issue(ownerUser.id, 'EMAIL_VERIFICATION', 600);
|
||||
await api().post('/api/v1/auth/verify-email').send({ token: verify }).expect(204);
|
||||
ownerCookie = await loginOf(owner.username);
|
||||
pondId = (await prisma.pond.findFirstOrThrow({ where: { ownerId, type: 'PERSONAL' } })).id;
|
||||
|
||||
const outsiderUser = await users.createUser({
|
||||
username: outsider.username,
|
||||
email: `${outsider.username}@example.org`,
|
||||
displayName: outsider.displayName,
|
||||
password,
|
||||
locale: 'en',
|
||||
});
|
||||
await users.markEmailVerified(outsiderUser.id);
|
||||
outsiderCookie = await loginOf(outsider.username);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await prisma.conversionJob.deleteMany({ where: { owner: { username: { contains: suffix } } } });
|
||||
await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } });
|
||||
const where = { pond: { owner: { username: { contains: suffix } } } };
|
||||
await prisma.attachment.deleteMany({ where });
|
||||
// Imports created pages; remove them before their pond (pages restrict it).
|
||||
await prisma.page.deleteMany({ where });
|
||||
await prisma.roleGrant.deleteMany({ where });
|
||||
await prisma.pond.deleteMany({ where: { owner: { username: { contains: suffix } } } });
|
||||
await prisma.user.deleteMany({ where: { username: { contains: suffix } } });
|
||||
await prisma.$disconnect();
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it('imports a document as a new page, titled from the leading heading', async () => {
|
||||
fake.markdown = '# Imported Report\n\nA paragraph with **bold** text.\n\n## Section\n\nMore.';
|
||||
const job = await importDoc('report.docx');
|
||||
|
||||
expect(job.body.status).toBe('succeeded');
|
||||
expect(job.body.errorCode).toBeNull();
|
||||
expect(job.body.resultPageId).toBeTruthy();
|
||||
|
||||
const page = await prisma.page.findUniqueOrThrow({ where: { id: job.body.resultPageId } });
|
||||
expect(page.title).toBe('Imported Report');
|
||||
expect(page.pondId).toBe(pondId);
|
||||
// The leading H1 became the title and was removed from the body; the rest
|
||||
// of the structure survives.
|
||||
const markdown = await markdownOf(page.id);
|
||||
expect(markdown).not.toMatch(/^# Imported Report/);
|
||||
expect(markdown).toContain('**bold**');
|
||||
expect(markdown).toContain('## Section');
|
||||
});
|
||||
|
||||
it('falls back to the file name (without extension) when there is no heading', async () => {
|
||||
fake.markdown = 'Just a paragraph, no heading.';
|
||||
const job = await importDoc('meeting-notes.odt');
|
||||
|
||||
const page = await prisma.page.findUniqueOrThrow({ where: { id: job.body.resultPageId } });
|
||||
expect(page.title).toBe('meeting-notes');
|
||||
});
|
||||
|
||||
it('stores an embedded image as a pond file with quota accounting', async () => {
|
||||
fake.markdown = `# With Image\n\nBefore.\n\n\n\nAfter.`;
|
||||
const before = await prisma.pondUsage.findUnique({ where: { pondId } });
|
||||
const usedBefore = Number(before?.storageBytesUsed ?? 0n);
|
||||
|
||||
const job = await importDoc('illustrated.docx');
|
||||
const pageId = job.body.resultPageId as string;
|
||||
|
||||
const files = await prisma.attachment.findMany({ where: { pageId } });
|
||||
expect(files).toHaveLength(1);
|
||||
expect(files[0]!.mimeType).toBe('image/png');
|
||||
expect(files[0]!.sizeBytes).toBeGreaterThan(0);
|
||||
|
||||
// Usage grew by exactly the stored file's size (quota accounting).
|
||||
const after = await prisma.pondUsage.findUniqueOrThrow({ where: { pondId } });
|
||||
expect(Number(after.storageBytesUsed) - usedBefore).toBe(files[0]!.sizeBytes);
|
||||
|
||||
// The image node now references the stored file id, not the data URI.
|
||||
const markdown = await markdownOf(pageId);
|
||||
expect(markdown).toContain(`(${files[0]!.id})`);
|
||||
expect(markdown).not.toContain('data:image');
|
||||
});
|
||||
|
||||
it('drops an image whose type the upload pipeline rejects, keeping the rest', async () => {
|
||||
// A vector image Word can embed — not a raster type, not on the allowlist.
|
||||
fake.markdown = '# Vector\n\nText before.\n\n\n\nText after.';
|
||||
const job = await importDoc('vector.docx');
|
||||
const pageId = job.body.resultPageId as string;
|
||||
|
||||
expect(await prisma.attachment.count({ where: { pageId } })).toBe(0);
|
||||
const markdown = await markdownOf(pageId);
|
||||
expect(markdown).toContain('Text before.');
|
||||
expect(markdown).toContain('Text after.');
|
||||
expect(markdown).not.toContain('data:image');
|
||||
});
|
||||
|
||||
it('fails the job with a meaningful error for a corrupt/unsupported document', async () => {
|
||||
fake.failHtmlWith = new ConversionError('conversion_failed', false, 'not a real docx');
|
||||
const pagesBefore = await prisma.page.count({ where: { pondId } });
|
||||
|
||||
const job = await importDoc('broken.docx');
|
||||
fake.failHtmlWith = null;
|
||||
|
||||
expect(job.body.status).toBe('failed');
|
||||
expect(job.body.errorCode).toBe('conversion_failed');
|
||||
expect(job.body.resultPageId).toBeNull();
|
||||
// No page is created for a failed import.
|
||||
expect(await prisma.page.count({ where: { pondId } })).toBe(pagesBefore);
|
||||
});
|
||||
|
||||
it('fails and rolls back media when the pond runs out of storage', async () => {
|
||||
// Cap this pond's storage below one image so the media store fails.
|
||||
await prisma.quotaOverride.upsert({
|
||||
where: {
|
||||
subjectType_subjectId_quotaKey: {
|
||||
subjectType: 'POND',
|
||||
subjectId: pondId,
|
||||
quotaKey: 'storage_bytes',
|
||||
},
|
||||
},
|
||||
create: { subjectType: 'POND', subjectId: pondId, quotaKey: 'storage_bytes', value: 10 },
|
||||
update: { value: 10 },
|
||||
});
|
||||
const usedBefore = Number(
|
||||
(await prisma.pondUsage.findUnique({ where: { pondId } }))?.storageBytesUsed ?? 0n,
|
||||
);
|
||||
const pagesBefore = await prisma.page.count({ where: { pondId } });
|
||||
fake.markdown = `# Too big\n\n`;
|
||||
|
||||
const job = await importDoc('too-big.docx');
|
||||
|
||||
expect(job.body.status).toBe('failed');
|
||||
expect(job.body.errorCode).toBe('quota_exceeded');
|
||||
// Neither a page nor any stored bytes survive the failed attempt.
|
||||
expect(await prisma.page.count({ where: { pondId } })).toBe(pagesBefore);
|
||||
const usedAfter = Number(
|
||||
(await prisma.pondUsage.findUnique({ where: { pondId } }))?.storageBytesUsed ?? 0n,
|
||||
);
|
||||
expect(usedAfter).toBe(usedBefore);
|
||||
|
||||
await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } });
|
||||
});
|
||||
|
||||
it('rejects an unsupported upload extension (400) before enqueuing', async () => {
|
||||
const res = await api()
|
||||
.post(`/api/v1/ponds/${pondId}/import`)
|
||||
.set('Cookie', ownerCookie)
|
||||
.attach('file', Buffer.from('hello'), 'notes.txt')
|
||||
.expect(400);
|
||||
expect(res.body.code).toBe('import_unsupported_format');
|
||||
});
|
||||
|
||||
it('hides the pond from a non-member (404, not an import)', async () => {
|
||||
await api()
|
||||
.post(`/api/v1/ponds/${pondId}/import`)
|
||||
.set('Cookie', outsiderCookie)
|
||||
.attach('file', Buffer.from('x'), 'doc.docx')
|
||||
.expect(404);
|
||||
});
|
||||
});
|
||||
322
apps/api/src/import-export/import.service.ts
Normal file
322
apps/api/src/import-export/import.service.ts
Normal file
@ -0,0 +1,322 @@
|
||||
import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/common';
|
||||
import { ConversionJob, User } from '@prisma/client';
|
||||
import { ConversionJobView, editorSchema, markdownToDoc } from '@dorfteich/shared';
|
||||
import { Node } from 'prosemirror-model';
|
||||
import { PinoLogger } from 'nestjs-pino';
|
||||
|
||||
import { FilesService } from '../files/files.service';
|
||||
import { PagesService } from '../pages/pages.service';
|
||||
import { docToState } from '../pages/yjs-content';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
|
||||
import { ImportProcessor } from './import.constants';
|
||||
import { ConversionError, PandocConverter } from './pandoc.converter';
|
||||
import { ConversionJobService } from './conversion-job.service';
|
||||
|
||||
/** pandoc source format per accepted upload extension (ADR 0009). */
|
||||
const IMPORT_FORMATS: Readonly<Record<string, string>> = {
|
||||
docx: 'docx',
|
||||
odt: 'odt',
|
||||
};
|
||||
|
||||
/** Job `kind` per source format, so the worker can route the job to the import
|
||||
* pipeline (a plain byte→byte conversion has a different kind, #62/#65). */
|
||||
function importKind(format: string): string {
|
||||
return `import_${format}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writer format for the structural pass. GFM matches our editor schema
|
||||
* (tables, task lists, strikethrough); `-implicit_figures` keeps images as
|
||||
* inline `` instead of wrapping them in a figure with a duplicated
|
||||
* caption; `-raw_html` stops pandoc emitting raw HTML for constructs it can't
|
||||
* represent (our Markdown parser runs with `html:false` and would drop it
|
||||
* anyway — per ADR 0009 we preserve structure, not layout).
|
||||
*/
|
||||
const IMPORT_MARKDOWN_FORMAT = 'gfm-implicit_figures-raw_html';
|
||||
|
||||
/**
|
||||
* The two-pass conversion at the heart of import, extracted so tests can run it
|
||||
* over the fixture corpus against a real sidecar. pandoc-server is stateless and
|
||||
* hands back a document's media no other way, so we first inline every image as
|
||||
* a `data:` URI (`source → html`, embed-resources), then produce clean
|
||||
* structural Markdown that still carries those data URIs inline at the right
|
||||
* positions (`html → gfm`, no wrapping).
|
||||
*/
|
||||
export async function convertImportedDocument(
|
||||
converter: PandocConverter,
|
||||
sourceFormat: string,
|
||||
input: Buffer,
|
||||
): Promise<string> {
|
||||
const html = await converter.convert({
|
||||
from: sourceFormat,
|
||||
to: 'html',
|
||||
input,
|
||||
standalone: false,
|
||||
embedResources: true,
|
||||
});
|
||||
const md = await converter.convert({
|
||||
from: 'html',
|
||||
to: IMPORT_MARKDOWN_FORMAT,
|
||||
input: html.output,
|
||||
standalone: false,
|
||||
wrap: 'none',
|
||||
});
|
||||
return md.output.toString('utf8');
|
||||
}
|
||||
|
||||
/** Minimal shape of a ProseMirror document as JSON — enough to walk it for
|
||||
* image nodes and the leading heading without pulling in prosemirror types. */
|
||||
interface PmNode {
|
||||
type: string;
|
||||
attrs?: Record<string, unknown>;
|
||||
content?: PmNode[];
|
||||
text?: string;
|
||||
marks?: unknown[];
|
||||
}
|
||||
|
||||
interface DecodedImage {
|
||||
buffer: Buffer;
|
||||
extension: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports `.docx`/`.odt` documents as new pages (ADR 0009, issue #63). Enqueue
|
||||
* is synchronous and cheap (validate + persist a job on the #62 queue); the
|
||||
* heavy conversion runs out of band in {@link run}, invoked by the shared
|
||||
* {@link ConversionWorker} for import-kind jobs so it inherits the queue's
|
||||
* locking, retry, and restart-survival.
|
||||
*
|
||||
* Pipeline: the pandoc sidecar is stateless and will not hand back a
|
||||
* document's embedded media any other way, so we convert in two passes —
|
||||
* `docx/odt → html` with `embed-resources` inlines every image as a `data:`
|
||||
* URI, then `html → gfm` produces clean structural Markdown with those data
|
||||
* URIs still inline at the right positions. We parse that to an editor
|
||||
* document, store each embedded image as a pond file (with quota accounting)
|
||||
* and rewrite its reference to the file id, derive the title from a leading
|
||||
* top-level heading (else the file name), and create the page from the
|
||||
* resulting Yjs state.
|
||||
*/
|
||||
@Injectable()
|
||||
export class ImportService implements ImportProcessor {
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly jobs: ConversionJobService,
|
||||
private readonly converter: PandocConverter,
|
||||
private readonly files: FilesService,
|
||||
private readonly pages: PagesService,
|
||||
private readonly logger: PinoLogger,
|
||||
) {
|
||||
this.logger.setContext(ImportService.name);
|
||||
}
|
||||
|
||||
/** Validate and enqueue an import; the returned job id is polled via
|
||||
* `GET /jobs/:id` until it reports the created `resultPageId` (#62). */
|
||||
async enqueue(
|
||||
user: User,
|
||||
pondId: string,
|
||||
file: { buffer: Buffer; originalname: string },
|
||||
): Promise<ConversionJobView> {
|
||||
const extension = fileExtension(file.originalname);
|
||||
const format = extension ? IMPORT_FORMATS[extension] : undefined;
|
||||
if (!format) {
|
||||
throw new BadRequestException({ code: 'import_unsupported_format' });
|
||||
}
|
||||
const job = await this.jobs.enqueue({
|
||||
ownerId: user.id,
|
||||
pondId,
|
||||
kind: importKind(format),
|
||||
from: format,
|
||||
to: 'page',
|
||||
input: file.buffer,
|
||||
sourceName: file.originalname,
|
||||
standalone: false,
|
||||
});
|
||||
return this.jobs.viewOf(job);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one import job to completion (called by the worker). Throws a
|
||||
* {@link ConversionError} on failure so the worker applies the queue's
|
||||
* retry/fail policy; on success it records the created page on the job.
|
||||
* Media stored during a failed attempt is rolled back so a retry does not
|
||||
* leak files or double-count quota.
|
||||
*/
|
||||
async run(job: ConversionJob): Promise<void> {
|
||||
if (!job.pondId) {
|
||||
throw new ConversionError('conversion_failed', false, 'import job has no pond');
|
||||
}
|
||||
const user = await this.prisma.user.findUnique({ where: { id: job.ownerId } });
|
||||
if (!user) {
|
||||
throw new ConversionError('conversion_failed', false, 'import job owner is gone');
|
||||
}
|
||||
|
||||
const rawMarkdown = await convertImportedDocument(
|
||||
this.converter,
|
||||
job.sourceFormat,
|
||||
Buffer.from(job.input),
|
||||
);
|
||||
|
||||
// Media is stored before the page exists (the page's state references the
|
||||
// file ids); track what we create so a later failure can be rolled back.
|
||||
const storedFileIds: string[] = [];
|
||||
try {
|
||||
const markdown = await this.storeEmbeddedImages(rawMarkdown, user, job.pondId, storedFileIds);
|
||||
const json = markdownToDoc(markdown).toJSON() as unknown as PmNode;
|
||||
const { title, doc } = this.splitTitle(json, job.sourceName);
|
||||
const state = docToState(Node.fromJSON(editorSchema, doc));
|
||||
|
||||
const page = await this.pages.createWithState(user, job.pondId, title, state);
|
||||
await this.files.linkAttachmentsToPage(storedFileIds, page.id);
|
||||
await this.prisma.conversionJob.update({
|
||||
where: { id: job.id },
|
||||
data: { status: 'SUCCEEDED', resultPageId: page.id, errorCode: null },
|
||||
});
|
||||
this.logger.info(
|
||||
{ jobId: job.id, pageId: page.id, pondId: job.pondId, images: storedFileIds.length },
|
||||
'audit: document imported',
|
||||
);
|
||||
} catch (error) {
|
||||
await this.rollbackMedia(user, storedFileIds);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Store each embedded image (an inline `data:` URI, from the embed-resources
|
||||
* pass) as a pond file and rewrite the Markdown reference to the stored file
|
||||
* id. Done on the Markdown text, before parsing, because the editor's parser
|
||||
* only admits a whitelist of `data:` image types (png/jpeg/gif/webp) and
|
||||
* would leave any other inline as a huge literal base64 string. base64 has no
|
||||
* `)`, so the image regex is unambiguous. An image whose bytes the upload
|
||||
* pipeline rejects (e.g. a vector `image/x-emf` Word can embed) is dropped
|
||||
* rather than failing the whole import — structure over layout (ADR 0009); a
|
||||
* pond that runs out of storage fails the import.
|
||||
*/
|
||||
private async storeEmbeddedImages(
|
||||
markdown: string,
|
||||
user: User,
|
||||
pondId: string,
|
||||
storedFileIds: string[],
|
||||
): Promise<string> {
|
||||
const image = /!\[([^\]]*)\]\((data:[^)\s]+)\)/g;
|
||||
const uris = new Set<string>();
|
||||
for (const match of markdown.matchAll(image)) uris.add(match[2]!);
|
||||
if (uris.size === 0) return markdown;
|
||||
|
||||
// Dedup identical images so a document that repeats one stores it once.
|
||||
const resolved = new Map<string, string | null>();
|
||||
let index = 0;
|
||||
for (const uri of uris) {
|
||||
resolved.set(uri, await this.storeImage(uri, user, pondId, ++index, storedFileIds));
|
||||
}
|
||||
return markdown.replace(image, (_whole, alt: string, uri: string) => {
|
||||
const fileId = resolved.get(uri);
|
||||
return fileId ? `` : '';
|
||||
});
|
||||
}
|
||||
|
||||
private async storeImage(
|
||||
uri: string,
|
||||
user: User,
|
||||
pondId: string,
|
||||
index: number,
|
||||
storedFileIds: string[],
|
||||
): Promise<string | null> {
|
||||
const decoded = decodeDataUri(uri);
|
||||
if (!decoded) return null;
|
||||
try {
|
||||
const view = await this.files.upload(user, pondId, {
|
||||
buffer: decoded.buffer,
|
||||
size: decoded.buffer.length,
|
||||
originalname: `import-${index}.${decoded.extension}`,
|
||||
});
|
||||
storedFileIds.push(view.id);
|
||||
return view.id;
|
||||
} catch (error) {
|
||||
if (error instanceof ForbiddenException && errorCode(error) === 'quota_exceeded') {
|
||||
throw new ConversionError('quota_exceeded', false, 'pond storage exhausted during import');
|
||||
}
|
||||
// Unsupported image type / rejected bytes — drop this one, keep going.
|
||||
this.logger.warn({ pondId, code: errorCode(error) }, 'import: dropped an image');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Use a leading top-level heading as the page title (and remove it from the
|
||||
* body so it is not duplicated), else fall back to the upload file name
|
||||
* without its extension, else a generic title.
|
||||
*/
|
||||
private splitTitle(root: PmNode, sourceName: string | null): { title: string; doc: PmNode } {
|
||||
const content = root.content ?? [];
|
||||
const first = content[0];
|
||||
if (first && first.type === 'heading') {
|
||||
const headingText = textOf(first).trim();
|
||||
if (headingText) {
|
||||
return { title: headingText, doc: { ...root, content: content.slice(1) } };
|
||||
}
|
||||
}
|
||||
const fallback = baseName(sourceName) || 'Imported document';
|
||||
return { title: fallback, doc: root };
|
||||
}
|
||||
|
||||
private async rollbackMedia(user: User, fileIds: string[]): Promise<void> {
|
||||
for (const id of fileIds) {
|
||||
try {
|
||||
await this.files.remove(user, id);
|
||||
} catch (error) {
|
||||
this.logger.warn({ fileId: id, code: errorCode(error) }, 'import: media rollback failed');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Concatenated text of a node's inline content (headings have no nesting). */
|
||||
function textOf(node: PmNode): string {
|
||||
if (typeof node.text === 'string') return node.text;
|
||||
return (node.content ?? []).map(textOf).join('');
|
||||
}
|
||||
|
||||
const DATA_URI = /^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/;
|
||||
const EXTENSION_BY_MIME: Readonly<Record<string, string>> = {
|
||||
'image/png': 'png',
|
||||
'image/jpeg': 'jpg',
|
||||
'image/gif': 'gif',
|
||||
'image/webp': 'webp',
|
||||
'image/svg+xml': 'svg',
|
||||
};
|
||||
|
||||
function decodeDataUri(value: string): DecodedImage | null {
|
||||
const match = DATA_URI.exec(value);
|
||||
if (!match) return null;
|
||||
const mime = match[1]!;
|
||||
const buffer = Buffer.from(match[2]!, 'base64');
|
||||
if (buffer.length === 0) return null;
|
||||
return { buffer, extension: EXTENSION_BY_MIME[mime] ?? 'bin' };
|
||||
}
|
||||
|
||||
/** Lowercased extension without the dot, or '' when there is none. */
|
||||
function fileExtension(fileName: string): string {
|
||||
const dot = fileName.lastIndexOf('.');
|
||||
return dot >= 0 ? fileName.slice(dot + 1).toLowerCase() : '';
|
||||
}
|
||||
|
||||
/** File name without its extension. */
|
||||
function baseName(fileName: string | null): string {
|
||||
if (!fileName) return '';
|
||||
const dot = fileName.lastIndexOf('.');
|
||||
return (dot > 0 ? fileName.slice(0, dot) : fileName).trim();
|
||||
}
|
||||
|
||||
/** The `code` from a Nest HttpException response body, if any. */
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (error instanceof BadRequestException || error instanceof ForbiddenException) {
|
||||
const body = error.getResponse();
|
||||
if (body && typeof body === 'object' && 'code' in body) {
|
||||
const code = (body as { code?: unknown }).code;
|
||||
return typeof code === 'string' ? code : undefined;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@ -10,6 +10,13 @@ export interface ConversionRequest {
|
||||
to: string;
|
||||
input: Buffer;
|
||||
standalone?: boolean;
|
||||
/** Inline referenced/embedded resources (images) as `data:` URIs in the
|
||||
* output. Used by the import pipeline (#63): pandoc-server is stateless and
|
||||
* will not hand back a document's media bytes any other way. */
|
||||
embedResources?: boolean;
|
||||
/** Line-wrapping of the writer's output. Import uses `none` so a paragraph
|
||||
* stays on one line (no soft breaks inside image alt text or links). */
|
||||
wrap?: 'none' | 'auto' | 'preserve';
|
||||
}
|
||||
|
||||
export interface ConversionResult {
|
||||
@ -18,7 +25,12 @@ export interface ConversionResult {
|
||||
}
|
||||
|
||||
export type ConversionErrorCode =
|
||||
'converter_unavailable' | 'converter_timeout' | 'conversion_failed';
|
||||
| 'converter_unavailable'
|
||||
| 'converter_timeout'
|
||||
| 'conversion_failed'
|
||||
// The pond ran out of storage while an import stored the document's media
|
||||
// (#63) — final, and surfaced the same way through the worker.
|
||||
| 'quota_exceeded';
|
||||
|
||||
/** A conversion failure with a stable, localizable code. `retryable` marks
|
||||
* the transient causes (sidecar down / timed out) the worker retries before
|
||||
@ -111,6 +123,10 @@ export class PandocServerConverter extends PandocConverter {
|
||||
from: request.from,
|
||||
to: request.to,
|
||||
standalone: request.standalone ?? true,
|
||||
// pandoc-server uses hyphenated option keys; unknown keys are ignored,
|
||||
// so these are only present when the import pipeline sets them.
|
||||
...(request.embedResources ? { 'embed-resources': true } : {}),
|
||||
...(request.wrap ? { wrap: request.wrap } : {}),
|
||||
}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
@ -130,23 +130,49 @@ export class PagesService {
|
||||
}
|
||||
|
||||
async create(user: User, pondId: string, input: CreatePageInput): Promise<PageView> {
|
||||
const page = await this.insertPage(user, pondId, input.title, emptyPageState());
|
||||
return this.viewOf(page);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a page from a prepared Yjs state (issue #63 import): the imported
|
||||
* document is already a full Yjs state whose fragment the editor binds to, so
|
||||
* an opening client sees the converted content immediately. Same invariants
|
||||
* as {@link create} — unique slug, appended sort key, derived content cache,
|
||||
* phantom-link resolution, search indexing. Returns the persisted row so the
|
||||
* caller (the import worker) can link the document's media to it.
|
||||
*/
|
||||
async createWithState(
|
||||
user: User,
|
||||
pondId: string,
|
||||
title: string,
|
||||
state: Uint8Array<ArrayBuffer>,
|
||||
): Promise<Page> {
|
||||
return this.insertPage(user, pondId, title, state);
|
||||
}
|
||||
|
||||
private async insertPage(
|
||||
user: User,
|
||||
pondId: string,
|
||||
title: string,
|
||||
state: Uint8Array<ArrayBuffer>,
|
||||
): Promise<Page> {
|
||||
const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } });
|
||||
if (!pond) throw new NotFoundException();
|
||||
|
||||
const slug = await this.generateUniqueSlugInPond(pond.id, input.title);
|
||||
const slug = await this.generateUniqueSlugInPond(pond.id, title);
|
||||
const last = await this.prisma.page.findFirst({
|
||||
where: { pondId: pond.id },
|
||||
orderBy: { sortKey: 'desc' },
|
||||
select: { sortKey: true },
|
||||
});
|
||||
const sortKey = generateKeyBetween(last?.sortKey ?? null, null);
|
||||
const state = emptyPageState();
|
||||
const content = deriveContent(state);
|
||||
|
||||
const page = await this.prisma.page.create({
|
||||
data: {
|
||||
pondId: pond.id,
|
||||
title: input.title,
|
||||
title,
|
||||
slug,
|
||||
sortKey,
|
||||
ydocState: state,
|
||||
@ -156,10 +182,10 @@ export class PagesService {
|
||||
});
|
||||
// A new page may satisfy phantom wikilinks that referenced its slug (#47).
|
||||
await this.resolvePhantomLinks(pond.id, slug, page.id);
|
||||
// Index the (empty) page so a title-only match is findable immediately (#49).
|
||||
// Index the page so a title-only match is findable immediately (#49).
|
||||
await this.search.indexPage(page.id);
|
||||
this.logger.info({ pageId: page.id, pondId: pond.id, userId: user.id }, 'audit: page created');
|
||||
return this.viewOf(page);
|
||||
return page;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -32,13 +32,17 @@ function docFromState(state: Uint8Array): Node {
|
||||
}
|
||||
}
|
||||
|
||||
/** A fresh Yjs state containing a single empty paragraph. */
|
||||
export function emptyPageState(): Uint8Array<ArrayBuffer> {
|
||||
/**
|
||||
* Encode a ProseMirror document as the initial Yjs state a page is created
|
||||
* with. Used both for a fresh empty page and for importing a converted
|
||||
* document (#63) as a page's starting content — the editor binds to the same
|
||||
* {@link FRAGMENT_NAME}, so an opening client sees exactly this document.
|
||||
*/
|
||||
export function docToState(doc: Node): Uint8Array<ArrayBuffer> {
|
||||
const ydoc = new Y.Doc();
|
||||
try {
|
||||
const fragment = ydoc.getXmlFragment(FRAGMENT_NAME);
|
||||
const emptyDoc = editorSchema.node('doc', null, [editorSchema.node('paragraph')]);
|
||||
prosemirrorJSONToYXmlFragment(editorSchema, emptyDoc.toJSON(), fragment);
|
||||
prosemirrorJSONToYXmlFragment(editorSchema, doc.toJSON(), fragment);
|
||||
// Copy into a plain ArrayBuffer-backed view — yjs's own return type is
|
||||
// the wider `Uint8Array<ArrayBufferLike>`, which Prisma's Bytes input
|
||||
// (`Uint8Array<ArrayBuffer>`) does not accept directly.
|
||||
@ -48,6 +52,11 @@ export function emptyPageState(): Uint8Array<ArrayBuffer> {
|
||||
}
|
||||
}
|
||||
|
||||
/** A fresh Yjs state containing a single empty paragraph. */
|
||||
export function emptyPageState(): Uint8Array<ArrayBuffer> {
|
||||
return docToState(editorSchema.node('doc', null, [editorSchema.node('paragraph')]));
|
||||
}
|
||||
|
||||
export interface DerivedPageContent {
|
||||
plainText: string;
|
||||
markdown: string;
|
||||
|
||||
@ -22,7 +22,13 @@ export default tseslint.config(
|
||||
// Plain-Node maintenance scripts (no TypeScript, no bundler).
|
||||
files: ['scripts/**/*.mjs'],
|
||||
languageOptions: {
|
||||
globals: { console: 'readonly', process: 'readonly', URL: 'readonly' },
|
||||
globals: {
|
||||
console: 'readonly',
|
||||
process: 'readonly',
|
||||
URL: 'readonly',
|
||||
fetch: 'readonly',
|
||||
Buffer: 'readonly',
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
42
fixtures/import/README.md
Normal file
42
fixtures/import/README.md
Normal file
@ -0,0 +1,42 @@
|
||||
# Import fixture corpus (issue #63, ADR 0009)
|
||||
|
||||
Representative `.docx`/`.odt` documents and the Markdown our import pipeline is
|
||||
expected to produce from them. `import.fixtures.test.ts` runs the real two-pass
|
||||
pandoc conversion over each and asserts the result, so a change in behaviour
|
||||
(ours or pandoc's) surfaces as a snapshot diff to review — not a silent
|
||||
regression.
|
||||
|
||||
## Files
|
||||
|
||||
For each document `<name>` and format `<ext>` (`docx`, `odt`):
|
||||
|
||||
- `<name>.<ext>` — the source document.
|
||||
- `<name>.<ext>.expected.md` — the Markdown the pipeline produces, with every
|
||||
embedded image's `data:` URI normalised to the literal `data:embedded-image`
|
||||
(the base64 payload is volatile and not what the snapshot is pinning).
|
||||
|
||||
`article` covers headings, paragraphs, **bold**/_italic_, a link, a bullet list
|
||||
with nesting, an ordered list, an embedded image, and a table. `formatting`
|
||||
covers strikethrough, inline code, a blockquote, and three levels of nesting.
|
||||
|
||||
## Fidelity notes (structure, not layout — ADR 0009)
|
||||
|
||||
- **ODT images lose their alt text** and **ODT tables lose their header row**:
|
||||
pandoc's HTML→ODT _writer_ does not encode either, so the source documents
|
||||
genuinely lack them. The DOCX variants keep both. This is a pinned pandoc
|
||||
limitation, not an import bug — extend the corpus rather than chasing it.
|
||||
|
||||
## Limits (ADR 0009)
|
||||
|
||||
An uploaded document may be at most **25 MiB** (`MAX_CONVERSION_INPUT_BYTES`),
|
||||
and each of the two conversion passes may run for at most **60 s**
|
||||
(`CONVERSION_TIMEOUT_MS`). A ~50-page document converts well inside that ceiling
|
||||
(`import.fixtures.test.ts` asserts it); a document that exceeds the timeout fails
|
||||
the job with `converter_timeout`.
|
||||
|
||||
## Regenerating
|
||||
|
||||
The documents and snapshots are generated with the **pinned** `pandoc/core:3.6`
|
||||
(the production sidecar) so the snapshots match CI; a different pandoc version
|
||||
wraps lists and pads tables differently. `scripts/gen-import-fixtures.mjs`
|
||||
regenerates everything against a reachable pandoc sidecar (`PANDOC_URL`).
|
||||
BIN
fixtures/import/article.docx
Normal file
BIN
fixtures/import/article.docx
Normal file
Binary file not shown.
29
fixtures/import/article.docx.expected.md
Normal file
29
fixtures/import/article.docx.expected.md
Normal file
@ -0,0 +1,29 @@
|
||||
# Field Notes
|
||||
|
||||
An introduction with **bold**, *italic*, and a [link to the pond](https://example.com/pond).
|
||||
|
||||
## Observations
|
||||
|
||||
The pond hosts several species. Notable ones include an image below.
|
||||
|
||||

|
||||
|
||||
## Lists
|
||||
|
||||
- Amphibians
|
||||
- Frogs
|
||||
- Newts
|
||||
- Insects
|
||||
- Dragonflies
|
||||
- Water striders
|
||||
|
||||
1. Spring survey
|
||||
2. Summer survey
|
||||
3. Autumn survey
|
||||
|
||||
## Measurements
|
||||
|
||||
| Month | Depth (cm) |
|
||||
|-------|------------|
|
||||
| April | 120 |
|
||||
| July | 95 |
|
||||
BIN
fixtures/import/article.odt
Normal file
BIN
fixtures/import/article.odt
Normal file
Binary file not shown.
32
fixtures/import/article.odt.expected.md
Normal file
32
fixtures/import/article.odt.expected.md
Normal file
@ -0,0 +1,32 @@
|
||||
# Field Notes
|
||||
|
||||
An introduction with **bold**, *italic*, and a [link to the pond](https://example.com/pond).
|
||||
|
||||
## Observations
|
||||
|
||||
The pond hosts several species. Notable ones include an image below.
|
||||
|
||||

|
||||
|
||||
## Lists
|
||||
|
||||
- Amphibians
|
||||
|
||||
- Frogs
|
||||
- Newts
|
||||
|
||||
- Insects
|
||||
|
||||
- Dragonflies
|
||||
- Water striders
|
||||
|
||||
1. Spring survey
|
||||
2. Summer survey
|
||||
3. Autumn survey
|
||||
|
||||
## Measurements
|
||||
|
||||
| | |
|
||||
|-------|-----|
|
||||
| April | 120 |
|
||||
| July | 95 |
|
||||
21
fixtures/import/article.src.html
Normal file
21
fixtures/import/article.src.html
Normal file
@ -0,0 +1,21 @@
|
||||
<h1>Field Notes</h1>
|
||||
<p>An introduction with <strong>bold</strong>, <em>italic</em>, and a <a href="https://example.com/pond">link to the pond</a>.</p>
|
||||
<h2>Observations</h2>
|
||||
<p>The pond hosts several species. Notable ones include an image below.</p>
|
||||
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAAEUlEQVR4nGP8z8DAwMDAwAAADAABZuUq0AAAAABJRU5ErkJggg==" alt="A calm pond"></p>
|
||||
<h2>Lists</h2>
|
||||
<ul>
|
||||
<li>Amphibians
|
||||
<ul><li>Frogs</li><li>Newts</li></ul></li>
|
||||
<li>Insects
|
||||
<ul><li>Dragonflies</li><li>Water striders</li></ul></li>
|
||||
</ul>
|
||||
<ol><li>Spring survey</li><li>Summer survey</li><li>Autumn survey</li></ol>
|
||||
<h2>Measurements</h2>
|
||||
<table>
|
||||
<thead><tr><th>Month</th><th>Depth (cm)</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>April</td><td>120</td></tr>
|
||||
<tr><td>July</td><td>95</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
BIN
fixtures/import/formatting.docx
Normal file
BIN
fixtures/import/formatting.docx
Normal file
Binary file not shown.
13
fixtures/import/formatting.docx.expected.md
Normal file
13
fixtures/import/formatting.docx.expected.md
Normal file
@ -0,0 +1,13 @@
|
||||
# Formatting Sampler
|
||||
|
||||
This paragraph mixes **bold**, *italic*, ~~strikethrough~~, and `inline code`.
|
||||
|
||||
> A quoted remark about still water.
|
||||
|
||||
### Nested detail
|
||||
|
||||
- Top level
|
||||
- Second level
|
||||
- Third level
|
||||
|
||||
A closing line with another [external link](https://example.org).
|
||||
BIN
fixtures/import/formatting.odt
Normal file
BIN
fixtures/import/formatting.odt
Normal file
Binary file not shown.
15
fixtures/import/formatting.odt.expected.md
Normal file
15
fixtures/import/formatting.odt.expected.md
Normal file
@ -0,0 +1,15 @@
|
||||
# Formatting Sampler
|
||||
|
||||
This paragraph mixes **bold**, *italic*, ~~strikethrough~~, and `inline code`.
|
||||
|
||||
> A quoted remark about still water.
|
||||
|
||||
### Nested detail
|
||||
|
||||
- Top level
|
||||
|
||||
- Second level
|
||||
|
||||
- Third level
|
||||
|
||||
A closing line with another [external link](https://example.org).
|
||||
6
fixtures/import/formatting.src.html
Normal file
6
fixtures/import/formatting.src.html
Normal file
@ -0,0 +1,6 @@
|
||||
<h1>Formatting Sampler</h1>
|
||||
<p>This paragraph mixes <strong>bold</strong>, <em>italic</em>, <s>strikethrough</s>, and <code>inline code</code>.</p>
|
||||
<blockquote><p>A quoted remark about still water.</p></blockquote>
|
||||
<h3>Nested detail</h3>
|
||||
<ul><li>Top level<ul><li>Second level<ul><li>Third level</li></ul></li></ul></li></ul>
|
||||
<p>A closing line with another <a href="https://example.org">external link</a>.</p>
|
||||
@ -34,6 +34,7 @@
|
||||
"converter_unavailable": "Der Dokument-Konverter ist derzeit nicht verfügbar. Bitte versuche es später erneut.",
|
||||
"converter_timeout": "Die Konvertierung hat zu lange gedauert und wurde abgebrochen.",
|
||||
"conversion_failed": "Dieses Dokument konnte nicht konvertiert werden.",
|
||||
"import_unsupported_format": "Nur Word- (.docx) und OpenDocument-Dokumente (.odt) können importiert werden.",
|
||||
"network": "Der Server war nicht erreichbar.",
|
||||
"grant_exists": "Diese Berechtigung existiert bereits.",
|
||||
"grant_pond_admin_scope": "Eine Teich-Admin-Berechtigung muss für den ganzen Teich und eine bestimmte Person gelten.",
|
||||
|
||||
@ -34,6 +34,7 @@
|
||||
"converter_unavailable": "The document converter is currently unavailable. Please try again later.",
|
||||
"converter_timeout": "The conversion took too long and was cancelled.",
|
||||
"conversion_failed": "This document could not be converted.",
|
||||
"import_unsupported_format": "Only Word (.docx) and OpenDocument (.odt) documents can be imported.",
|
||||
"network": "The server could not be reached.",
|
||||
"grant_exists": "This grant already exists.",
|
||||
"grant_pond_admin_scope": "A Pond Admin grant must apply to the whole pond and a specific user.",
|
||||
|
||||
@ -12,8 +12,17 @@ export interface ConversionJobView {
|
||||
sourceFormat: string;
|
||||
targetFormat: string;
|
||||
/** Set only when `status` is `failed` — a code from the errors namespace
|
||||
* (`converter_unavailable` | `converter_timeout` | `conversion_failed`). */
|
||||
* (`converter_unavailable` | `converter_timeout` | `conversion_failed` |
|
||||
* `quota_exceeded`). */
|
||||
errorCode: string | null;
|
||||
/** Set once an import job (#63) succeeds: the id of the page it created, so
|
||||
* the client can navigate to it. `null` for a pending/failed import and for
|
||||
* plain byte→byte conversions (export). */
|
||||
resultPageId: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
/** Extensions the import endpoint accepts (ADR 0009, issue #63). */
|
||||
export const IMPORT_EXTENSIONS = ['docx', 'odt'] as const;
|
||||
export type ImportExtension = (typeof IMPORT_EXTENSIONS)[number];
|
||||
|
||||
69
scripts/gen-import-fixtures.mjs
Normal file
69
scripts/gen-import-fixtures.mjs
Normal file
@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env node
|
||||
// Regenerate the import fixture corpus (issue #63, ADR 0009) from the committed
|
||||
// `*.src.html` sources: for each source, write `<name>.docx`, `<name>.odt`, and
|
||||
// the expected Markdown snapshot for each format. Run against a reachable pandoc
|
||||
// sidecar (the pinned `pandoc/core:3.6`, so snapshots match CI):
|
||||
//
|
||||
// docker run --rm -p 3030:3030 pandoc/core:3.6 server
|
||||
// PANDOC_URL=http://localhost:3030 node scripts/gen-import-fixtures.mjs
|
||||
//
|
||||
// The pipeline mirrors ImportService: `html → <ext>` builds the document, then
|
||||
// `<ext> → html` (embed-resources) → `html → gfm` reproduces what an import
|
||||
// would parse, with image data URIs normalised to a stable token.
|
||||
|
||||
import { readFileSync, writeFileSync, readdirSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
const FIXTURES = join(dirname(fileURLToPath(import.meta.url)), '..', 'fixtures', 'import');
|
||||
const PANDOC_URL = process.env.PANDOC_URL ?? 'http://localhost:3030';
|
||||
const MARKDOWN_FORMAT = 'gfm-implicit_figures-raw_html';
|
||||
|
||||
/** POST a conversion to pandoc-server. Without an `Accept` header the server
|
||||
* returns text for text writers and raw bytes for binary ones (docx/odt). */
|
||||
async function pandoc(params, binaryOut = false) {
|
||||
const res = await fetch(`${PANDOC_URL}/`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(params),
|
||||
});
|
||||
if (!res.ok) throw new Error(`pandoc ${res.status}: ${await res.text()}`);
|
||||
return binaryOut ? Buffer.from(await res.arrayBuffer()) : res.text();
|
||||
}
|
||||
|
||||
async function buildDocument(html, ext) {
|
||||
return pandoc({ text: html, from: 'html', to: ext, standalone: true }, true);
|
||||
}
|
||||
|
||||
async function toExpectedMarkdown(documentBytes, ext) {
|
||||
const embedded = await pandoc({
|
||||
text: documentBytes.toString('base64'),
|
||||
from: ext,
|
||||
to: 'html',
|
||||
standalone: false,
|
||||
'embed-resources': true,
|
||||
});
|
||||
const md = await pandoc({
|
||||
text: embedded,
|
||||
from: 'html',
|
||||
to: MARKDOWN_FORMAT,
|
||||
standalone: false,
|
||||
wrap: 'none',
|
||||
});
|
||||
return md.replace(/data:image\/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g, 'data:embedded-image');
|
||||
}
|
||||
|
||||
const sources = readdirSync(FIXTURES).filter((f) => f.endsWith('.src.html'));
|
||||
for (const source of sources) {
|
||||
const name = source.replace(/\.src\.html$/, '');
|
||||
const html = readFileSync(join(FIXTURES, source), 'utf8');
|
||||
for (const ext of ['docx', 'odt']) {
|
||||
const bytes = await buildDocument(html, ext);
|
||||
writeFileSync(join(FIXTURES, `${name}.${ext}`), bytes);
|
||||
writeFileSync(
|
||||
join(FIXTURES, `${name}.${ext}.expected.md`),
|
||||
await toExpectedMarkdown(bytes, ext),
|
||||
);
|
||||
console.log(`wrote ${name}.${ext} (+ expected.md)`);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user