Import .docx and .odt documents as new pages (#63)
All checks were successful
CD / Build and push images (push) Successful in 3m19s
CI / Lint, typecheck, test (push) Successful in 2m55s
CI / Auth e2e pack (push) Successful in 3m45s
CI / Build container images (push) Has been skipped
CD / Deploy to Test (push) Successful in 9s
CD / Smoke tests against Test (push) Successful in 1m13s
CD / Promote to Int (push) Successful in 11s

Uploading a Word/OpenOffice document to POST /ponds/:id/import enqueues a
conversion job (the #62 queue) that produces a new page in the pond; the
client polls GET /jobs/:id for the created resultPageId.

Pipeline (ImportService, ADR 0009): pandoc-server is stateless and hands
back a document's media no other way, so we convert in two passes —
docx/odt → html with embed-resources inlines every image as a data: URI,
then html → gfm produces clean structural Markdown with those data URIs
still inline. Embedded images are stored as pond files (with quota
accounting) and their references rewritten to file ids on the Markdown
text before parsing (the editor parser only admits png/jpeg/gif/webp data
URIs); an image whose bytes the upload pipeline rejects is dropped, not
fatal. The title comes from a leading top-level heading (removed from the
body) else the file name. The page is created from the resulting Yjs state.

The shared conversion worker routes import-kind jobs to the pipeline via a
token (breaking a module cycle), so import inherits the queue's locking,
retry, and restart-survival. Media stored during a failed attempt is rolled
back; a pond that runs out of storage fails the job with quota_exceeded.

- schema: ConversionJob gains pond_id / source_name / result_page_id
  (migration 20260710041215_import_pages_conversion); ConversionJobView
  gains resultPageId.
- PagesService.createWithState / yjs-content docToState build a page from a
  prepared document; FilesService.linkAttachmentsToPage links import media.
- fixtures/import/: representative .docx/.odt corpus (headings, lists,
  nested lists, tables, images, links, bold/italic) with expected-Markdown
  snapshots; scripts/gen-import-fixtures.mjs regenerates them.
- tests: import.service.db.test.ts drives the full pipeline with a fake
  converter (CI); import.fixtures.test.ts runs the real two-pass conversion
  over the corpus and a 50-page timing check against a reachable sidecar.
- i18n: import_unsupported_format (de+en). Limits documented (25 MiB input,
  60 s per pass).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1
This commit is contained in:
Claude Opus 4.8 2026-07-10 07:34:43 +02:00
parent 4755c18ef5
commit 546e8279ac
32 changed files with 1116 additions and 18 deletions

View File

@ -6,3 +6,8 @@ pnpm-lock.yaml
# Must stay byte-identical to docToMarkdown's own output (issue #32) — # Must stay byte-identical to docToMarkdown's own output (issue #32) —
# Prettier's Markdown table/list opinions would break that fixed point. # Prettier's Markdown table/list opinions would break that fixed point.
apps/api/prisma/fixtures/content-page.md apps/api/prisma/fixtures/content-page.md
# Import fixture corpus (issue #63): the expected-Markdown snapshots and the
# HTML sources must stay exactly as pandoc produces/consumes them — Prettier's
# Markdown/HTML opinions would break the regression tests.
fixtures/import/*.expected.md
fixtures/import/*.src.html

View File

@ -0,0 +1,10 @@
-- AlterTable
ALTER TABLE "conversion_jobs" ADD COLUMN "pond_id" TEXT,
ADD COLUMN "result_page_id" TEXT,
ADD COLUMN "source_name" TEXT;
-- AddForeignKey
ALTER TABLE "conversion_jobs" ADD CONSTRAINT "conversion_jobs_pond_id_fkey" FOREIGN KEY ("pond_id") REFERENCES "ponds"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "conversion_jobs" ADD CONSTRAINT "conversion_jobs_result_page_id_fkey" FOREIGN KEY ("result_page_id") REFERENCES "pages"("id") ON DELETE SET NULL ON UPDATE CASCADE;

View File

@ -81,6 +81,7 @@ model Pond {
attachments Attachment[] attachments Attachment[]
labels Label[] labels Label[]
grants RoleGrant[] grants RoleGrant[]
conversionJobs ConversionJob[]
@@index([ownerId]) @@index([ownerId])
@@map("ponds") @@map("ponds")
@ -164,6 +165,7 @@ model Page {
labels PageLabel[] labels PageLabel[]
outgoingLinks PageLink[] @relation("outgoingLinks") outgoingLinks PageLink[] @relation("outgoingLinks")
incomingLinks PageLink[] @relation("incomingLinks") incomingLinks PageLink[] @relation("incomingLinks")
conversionJobs ConversionJob[]
@@unique([pondId, slug]) @@unique([pondId, slug])
@@index([pondId]) @@index([pondId])
@ -551,7 +553,16 @@ model ConversionJob {
createdAt DateTime @default(now()) @map("created_at") createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at") updatedAt DateTime @updatedAt @map("updated_at")
/// Import jobs (#63) carry the pond the document is imported into, the
/// original upload file name (title fallback), and the page they produced.
/// All null for a plain byte→byte conversion (export, #62/#65).
pondId String? @map("pond_id")
sourceName String? @map("source_name")
resultPageId String? @map("result_page_id")
owner User @relation(fields: [ownerId], references: [id], onDelete: Cascade) owner User @relation(fields: [ownerId], references: [id], onDelete: Cascade)
pond Pond? @relation(fields: [pondId], references: [id], onDelete: Cascade)
page Page? @relation(fields: [resultPageId], references: [id], onDelete: SetNull)
@@index([status, createdAt]) @@index([status, createdAt])
@@map("conversion_jobs") @@map("conversion_jobs")

View File

@ -11,6 +11,6 @@ import { FilesService } from './files.service';
imports: [PondsModule, QuotasModule], imports: [PondsModule, QuotasModule],
controllers: [FilesController], controllers: [FilesController],
providers: [FilesService, FileStorageService], providers: [FilesService, FileStorageService],
exports: [FileStorageService], exports: [FileStorageService, FilesService],
}) })
export class FilesModule {} export class FilesModule {}

View File

@ -169,6 +169,21 @@ export class FilesService {
} }
} }
/**
* Point a set of just-uploaded attachments at the page that now embeds them
* (issue #63 import). The import worker stores a document's media before the
* page exists (the page's Yjs state references their ids), then calls this so
* they list under the page and are purged with it (#31), the same invariant
* the collab persistence hook maintains for pasted images.
*/
async linkAttachmentsToPage(attachmentIds: string[], pageId: string): Promise<void> {
if (attachmentIds.length === 0) return;
await this.prisma.attachment.updateMany({
where: { id: { in: attachmentIds } },
data: { pageId },
});
}
/** Upload against the pond of `pageId`, linked to that page (#61). */ /** Upload against the pond of `pageId`, linked to that page (#61). */
async uploadToPage( async uploadToPage(
user: User, user: User,

View File

@ -15,6 +15,10 @@ export interface EnqueueConversion {
to: string; to: string;
input: Buffer; input: Buffer;
standalone?: boolean; standalone?: boolean;
/** Set for import jobs (#63): the pond the document is imported into and the
* original upload file name (a title fallback). */
pondId?: string;
sourceName?: string;
} }
export interface ConversionResultPayload { export interface ConversionResultPayload {
@ -55,6 +59,8 @@ export class ConversionJobService {
const job = await this.prisma.conversionJob.create({ const job = await this.prisma.conversionJob.create({
data: { data: {
ownerId: request.ownerId, ownerId: request.ownerId,
pondId: request.pondId ?? null,
sourceName: request.sourceName ?? null,
kind: request.kind, kind: request.kind,
sourceFormat: request.from, sourceFormat: request.from,
targetFormat: request.to, targetFormat: request.to,
@ -103,6 +109,8 @@ export class ConversionJobService {
sourceFormat: job.sourceFormat, sourceFormat: job.sourceFormat,
targetFormat: job.targetFormat, targetFormat: job.targetFormat,
errorCode: job.errorCode, errorCode: job.errorCode,
// Set once an import job succeeds (#63) so the client can open the page.
resultPageId: job.resultPageId,
createdAt: job.createdAt.toISOString(), createdAt: job.createdAt.toISOString(),
updatedAt: job.updatedAt.toISOString(), updatedAt: job.updatedAt.toISOString(),
}; };

View File

@ -1,10 +1,12 @@
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { ConversionJob } from '@prisma/client'; import { ConversionJob } from '@prisma/client';
import { PinoLogger } from 'nestjs-pino'; import { PinoLogger } from 'nestjs-pino';
import { AppConfig } from '../config/app-config.service'; import { AppConfig } from '../config/app-config.service';
import { PrismaService } from '../prisma/prisma.service'; import { PrismaService } from '../prisma/prisma.service';
import { IMPORT_PROCESSOR, ImportProcessor, isImportKind } from './import.constants';
import { ConversionError, PandocConverter } from './pandoc.converter'; import { ConversionError, PandocConverter } from './pandoc.converter';
/** How often the worker sweeps for pending jobs on its own the safety net /** How often the worker sweeps for pending jobs on its own the safety net
@ -38,6 +40,9 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy {
private readonly converter: PandocConverter, private readonly converter: PandocConverter,
private readonly config: AppConfig, private readonly config: AppConfig,
private readonly logger: PinoLogger, private readonly logger: PinoLogger,
// Resolved lazily to break the construction cycle (the import service
// enqueues via the job service, which wakes this worker).
private readonly moduleRef: ModuleRef,
) { ) {
this.logger.setContext(ConversionWorker.name); this.logger.setContext(ConversionWorker.name);
} }
@ -103,6 +108,13 @@ export class ConversionWorker implements OnModuleInit, OnModuleDestroy {
private async process(job: ConversionJob): Promise<void> { private async process(job: ConversionJob): Promise<void> {
try { try {
if (isImportKind(job.kind)) {
// Import runs a multi-step pipeline and records its own success (the
// created page id) on the job (#63). Resolved lazily via a token so the
// worker's file never imports the import service's (avoids a cycle).
await this.moduleRef.get<ImportProcessor>(IMPORT_PROCESSOR, { strict: false }).run(job);
return;
}
const result = await this.converter.convert({ const result = await this.converter.convert({
from: job.sourceFormat, from: job.sourceFormat,
to: job.targetFormat, to: job.targetFormat,

View File

@ -1,20 +1,32 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { FilesModule } from '../files/files.module';
import { PagesModule } from '../pages/pages.module';
import { ConversionJobService } from './conversion-job.service'; import { ConversionJobService } from './conversion-job.service';
import { ConversionWorker } from './conversion-worker.service'; import { ConversionWorker } from './conversion-worker.service';
import { IMPORT_PROCESSOR } from './import.constants';
import { ImportController } from './import.controller';
import { ImportService } from './import.service';
import { JobsController } from './jobs.controller'; import { JobsController } from './jobs.controller';
import { PandocConverter, PandocServerConverter } from './pandoc.converter'; import { PandocConverter, PandocServerConverter } from './pandoc.converter';
/** /**
* Import/export orchestration (ADR 0009, issue #62): the conversion job queue, * Import/export orchestration (ADR 0009): the conversion job queue, its worker,
* its worker, and the pandoc-server client. Later stories (#63 import, #65 * the pandoc-server client (#62), and the document import pipeline (#63, which
* export) add the feature endpoints that enqueue jobs here. * turns an uploaded `.docx`/`.odt` into a new page). Export (#65) will add its
* feature endpoints here too.
*/ */
@Module({ @Module({
controllers: [JobsController], imports: [FilesModule, PagesModule],
controllers: [JobsController, ImportController],
providers: [ providers: [
ConversionJobService, ConversionJobService,
ConversionWorker, ConversionWorker,
ImportService,
// The worker resolves the import pipeline through this token (never the
// class), so its file does not import the import service's (avoids a cycle).
{ provide: IMPORT_PROCESSOR, useExisting: ImportService },
// Bind the abstract converter to the HTTP implementation; tests override // Bind the abstract converter to the HTTP implementation; tests override
// this provider with a fake so the queue mechanics need no live sidecar. // this provider with a fake so the queue mechanics need no live sidecar.
{ provide: PandocConverter, useClass: PandocServerConverter }, { provide: PandocConverter, useClass: PandocServerConverter },

View File

@ -0,0 +1,21 @@
import { ConversionJob } from '@prisma/client';
/**
* Contract between the shared conversion worker and the import pipeline, kept in
* its own module so neither imports the other's file (the worker resolves the
* processor lazily via this token, breaking what would otherwise be a cycle:
* import.service conversion-job.service conversion-worker.service).
*/
export const IMPORT_PROCESSOR = Symbol('IMPORT_PROCESSOR');
export interface ImportProcessor {
/** Run one import job to completion, recording the created page on the job or
* throwing a ConversionError for the worker's retry/fail policy. */
run(job: ConversionJob): Promise<void>;
}
/** True for a job the import pipeline processes rather than the plain pandoc
* bytebyte path (export, #62/#65). */
export function isImportKind(kind: string): boolean {
return kind.startsWith('import_');
}

View File

@ -0,0 +1,39 @@
import {
BadRequestException,
Controller,
Param,
Post,
Req,
UploadedFile,
UseInterceptors,
} from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express';
import { ConversionJobView, MAX_UPLOAD_PARSE_BYTES } from '@dorfteich/shared';
import { AuthedRequest } from '../auth/auth.guard';
import { RequiresPondRole } from '../permissions/permission.decorators';
import { ImportService } from './import.service';
/**
* Document import (ADR 0009, issue #63). Uploading a `.docx`/`.odt` enqueues a
* conversion job (#62 queue) that produces a new page in the pond; the client
* polls `GET /jobs/:id` for `resultPageId`. Creating a page requires pond-wide
* editor access (permissions.md, same rule as `POST /ponds/:id/pages`).
*/
@Controller('ponds/:pondId')
export class ImportController {
constructor(private readonly imports: ImportService) {}
@Post('import')
@RequiresPondRole('editor', { idParam: 'pondId' })
@UseInterceptors(FileInterceptor('file', { limits: { fileSize: MAX_UPLOAD_PARSE_BYTES } }))
async import(
@Param('pondId') pondId: string,
@UploadedFile() file: Express.Multer.File | undefined,
@Req() request: AuthedRequest,
): Promise<ConversionJobView> {
if (!file) throw new BadRequestException({ code: 'bad_request' });
return this.imports.enqueue(request.user!, pondId, file);
}
}

View File

@ -0,0 +1,78 @@
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { markdownToDoc } from '@dorfteich/shared';
import { beforeAll, describe, expect, it, TestContext } from 'vitest';
import { AppConfig } from '../config/app-config.service';
import { convertImportedDocument } from './import.service';
import { CONVERSION_TIMEOUT_MS, PandocServerConverter } from './pandoc.converter';
/**
* Import fidelity regression (issue #63, ADR 0009): runs the real two-pass
* pandoc conversion over the committed `.docx`/`.odt` corpus and asserts each
* produces its expected Markdown. Needs a reachable pandoc sidecar (the pinned
* `pandoc/core:3.6`, so output matches the snapshots) each test skips itself
* when none is configured, and CI starts one and points `PANDOC_URL` at it.
*/
const PANDOC_URL = process.env.PANDOC_URL ?? 'http://localhost:3030';
// The test runner's cwd is `apps/api`; the corpus lives at the repo root.
const FIXTURES = join(process.cwd(), '../../fixtures/import');
const converter = new PandocServerConverter({
env: { PANDOC_URL },
} as unknown as AppConfig);
let reachable = false;
/** Normalise embedded image `data:` URIs to the stable token the snapshots use
* (the base64 payload is volatile and not what we are pinning). */
function normalize(markdown: string): string {
return markdown.replace(
/data:image\/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g,
'data:embedded-image',
);
}
const CORPUS = ['article.docx', 'article.odt', 'formatting.docx', 'formatting.odt'];
describe('import fixture corpus (real pandoc, issue #63)', () => {
beforeAll(async () => {
reachable = await converter.reachable().catch(() => false);
});
for (const fixture of CORPUS) {
it(`converts ${fixture} to its expected Markdown`, async (ctx: TestContext) => {
if (!reachable) ctx.skip();
const format = fixture.endsWith('.odt') ? 'odt' : 'docx';
const document = readFileSync(join(FIXTURES, fixture));
const expected = readFileSync(join(FIXTURES, `${fixture}.expected.md`), 'utf8');
const markdown = await convertImportedDocument(converter, format, document);
expect(normalize(markdown)).toBe(expected);
// The Markdown must also parse into a valid editor document (no schema
// surprises from real-world structure).
expect(() => markdownToDoc(markdown)).not.toThrow();
});
}
it('imports a 50-page document within the conversion timeout', async (ctx: TestContext) => {
if (!reachable) ctx.skip();
// Build a ~50-page document by repeating a page of structured content, then
// convert it to docx once and time the import conversion of that document.
const onePage =
'# Section\n\n' + 'A paragraph of survey notes about the pond. '.repeat(20) + '\n\n';
const large = Array.from({ length: 50 }, () => onePage).join('\n---\n\n');
const docx = await converter.convert({ from: 'gfm', to: 'docx', input: Buffer.from(large) });
const started = Date.now();
const markdown = await convertImportedDocument(converter, 'docx', docx.output);
const elapsed = Date.now() - started;
expect(markdown.length).toBeGreaterThan(1000);
// Comfortably inside the documented 60 s per-conversion ceiling (ADR 0009).
expect(elapsed).toBeLessThan(CONVERSION_TIMEOUT_MS);
});
});

View File

@ -0,0 +1,270 @@
import { INestApplication } from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import request from 'supertest';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { AuthTokensService } from '../auth/auth-tokens.service';
import { createTestApp, sessionCookieOf } from '../testing/test-app';
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
import { UsersService } from '../users/users.service';
import { ConversionWorker } from './conversion-worker.service';
import {
ConversionError,
ConversionRequest,
ConversionResult,
PandocConverter,
} from './pandoc.converter';
/**
* Document import pipeline (issue #63): upload conversion job media stored
* as pond files new page. Driven by an injected fake converter that returns a
* chosen Markdown for the structural pass, so the orchestration (media
* extraction, quota, title, rollback, failure codes) is tested deterministically
* without a live pandoc sidecar. The real conversion of the fixture corpus is
* covered against a running sidecar by import.fixtures.test.ts.
*/
/** A 1×1 PNG — passes the raster magic-byte sniff, so it is stored as an image. */
const PNG_BASE64 =
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';
/** A converter whose `markdown` each test sets. The import pipeline calls it
* twice (sourcehtml, then htmlgfm); the html pass is echoed, the gfm pass
* returns the chosen Markdown (its input is irrelevant to these tests). A test
* may instead make a pass throw to exercise a conversion failure. */
class FakeConverter extends PandocConverter {
markdown = '# Untitled\n\nBody.';
failHtmlWith: ConversionError | null = null;
convert(request: ConversionRequest): Promise<ConversionResult> {
if (request.to === 'html') {
if (this.failHtmlWith) return Promise.reject(this.failHtmlWith);
return Promise.resolve({ output: Buffer.from('<html/>'), mimeType: 'text/html' });
}
return Promise.resolve({ output: Buffer.from(this.markdown), mimeType: 'text/markdown' });
}
reachable(): Promise<boolean> {
return Promise.resolve(true);
}
}
describe.skipIf(!hasTestDb)('document import (e2e, issue #63)', () => {
let app: INestApplication;
let prisma: PrismaClient;
let worker: ConversionWorker;
let fake: FakeConverter;
const suffix = uniqueSuffix();
const password = 'importiere meine dokumente 1';
const owner = { username: `iris-import-${suffix}`, displayName: `Iris Import ${suffix}` };
const outsider = { username: `otis-out-${suffix}`, displayName: `Otis Out ${suffix}` };
let ownerId: string;
let pondId: string;
let ownerCookie: string;
let outsiderCookie: string;
const api = () => request(app.getHttpServer());
async function loginOf(username: string): Promise<string> {
const res = await api()
.post('/api/v1/auth/login')
.send({ usernameOrEmail: username, password })
.expect(200);
return sessionCookieOf(res);
}
/** Upload a document and drain the queue; returns the finished job view. */
async function importDoc(fileName: string, cookie = ownerCookie): Promise<request.Response> {
const enqueued = await api()
.post(`/api/v1/ponds/${pondId}/import`)
.set('Cookie', cookie)
.attach('file', Buffer.from('source-document-bytes'), fileName)
.expect(201);
await worker.drain();
return api().get(`/api/v1/jobs/${enqueued.body.id}`).set('Cookie', cookie).expect(200);
}
async function markdownOf(pageId: string): Promise<string> {
const cache = await prisma.pageContentCache.findUniqueOrThrow({ where: { pageId } });
return cache.markdown;
}
beforeAll(async () => {
prisma = createTestPrisma();
await prisma.rateLimit.deleteMany({});
fake = new FakeConverter();
app = await createTestApp((builder) =>
builder.overrideProvider(PandocConverter).useValue(fake),
);
worker = app.get(ConversionWorker);
const users = app.get(UsersService);
const tokens = app.get(AuthTokensService);
const ownerUser = await users.createUser({
username: owner.username,
email: `${owner.username}@example.org`,
displayName: owner.displayName,
password,
locale: 'en',
});
ownerId = ownerUser.id;
// Verifying the e-mail creates the owner's personal pond (+ owner-admin
// grant), which the owner may import into.
const verify = await tokens.issue(ownerUser.id, 'EMAIL_VERIFICATION', 600);
await api().post('/api/v1/auth/verify-email').send({ token: verify }).expect(204);
ownerCookie = await loginOf(owner.username);
pondId = (await prisma.pond.findFirstOrThrow({ where: { ownerId, type: 'PERSONAL' } })).id;
const outsiderUser = await users.createUser({
username: outsider.username,
email: `${outsider.username}@example.org`,
displayName: outsider.displayName,
password,
locale: 'en',
});
await users.markEmailVerified(outsiderUser.id);
outsiderCookie = await loginOf(outsider.username);
});
afterAll(async () => {
await prisma.conversionJob.deleteMany({ where: { owner: { username: { contains: suffix } } } });
await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } });
const where = { pond: { owner: { username: { contains: suffix } } } };
await prisma.attachment.deleteMany({ where });
// Imports created pages; remove them before their pond (pages restrict it).
await prisma.page.deleteMany({ where });
await prisma.roleGrant.deleteMany({ where });
await prisma.pond.deleteMany({ where: { owner: { username: { contains: suffix } } } });
await prisma.user.deleteMany({ where: { username: { contains: suffix } } });
await prisma.$disconnect();
await app.close();
});
it('imports a document as a new page, titled from the leading heading', async () => {
fake.markdown = '# Imported Report\n\nA paragraph with **bold** text.\n\n## Section\n\nMore.';
const job = await importDoc('report.docx');
expect(job.body.status).toBe('succeeded');
expect(job.body.errorCode).toBeNull();
expect(job.body.resultPageId).toBeTruthy();
const page = await prisma.page.findUniqueOrThrow({ where: { id: job.body.resultPageId } });
expect(page.title).toBe('Imported Report');
expect(page.pondId).toBe(pondId);
// The leading H1 became the title and was removed from the body; the rest
// of the structure survives.
const markdown = await markdownOf(page.id);
expect(markdown).not.toMatch(/^# Imported Report/);
expect(markdown).toContain('**bold**');
expect(markdown).toContain('## Section');
});
it('falls back to the file name (without extension) when there is no heading', async () => {
fake.markdown = 'Just a paragraph, no heading.';
const job = await importDoc('meeting-notes.odt');
const page = await prisma.page.findUniqueOrThrow({ where: { id: job.body.resultPageId } });
expect(page.title).toBe('meeting-notes');
});
it('stores an embedded image as a pond file with quota accounting', async () => {
fake.markdown = `# With Image\n\nBefore.\n\n![a dot](data:image/png;base64,${PNG_BASE64})\n\nAfter.`;
const before = await prisma.pondUsage.findUnique({ where: { pondId } });
const usedBefore = Number(before?.storageBytesUsed ?? 0n);
const job = await importDoc('illustrated.docx');
const pageId = job.body.resultPageId as string;
const files = await prisma.attachment.findMany({ where: { pageId } });
expect(files).toHaveLength(1);
expect(files[0]!.mimeType).toBe('image/png');
expect(files[0]!.sizeBytes).toBeGreaterThan(0);
// Usage grew by exactly the stored file's size (quota accounting).
const after = await prisma.pondUsage.findUniqueOrThrow({ where: { pondId } });
expect(Number(after.storageBytesUsed) - usedBefore).toBe(files[0]!.sizeBytes);
// The image node now references the stored file id, not the data URI.
const markdown = await markdownOf(pageId);
expect(markdown).toContain(`(${files[0]!.id})`);
expect(markdown).not.toContain('data:image');
});
it('drops an image whose type the upload pipeline rejects, keeping the rest', async () => {
// A vector image Word can embed — not a raster type, not on the allowlist.
fake.markdown = '# Vector\n\nText before.\n\n![v](data:image/x-emf;base64,AAAA)\n\nText after.';
const job = await importDoc('vector.docx');
const pageId = job.body.resultPageId as string;
expect(await prisma.attachment.count({ where: { pageId } })).toBe(0);
const markdown = await markdownOf(pageId);
expect(markdown).toContain('Text before.');
expect(markdown).toContain('Text after.');
expect(markdown).not.toContain('data:image');
});
it('fails the job with a meaningful error for a corrupt/unsupported document', async () => {
fake.failHtmlWith = new ConversionError('conversion_failed', false, 'not a real docx');
const pagesBefore = await prisma.page.count({ where: { pondId } });
const job = await importDoc('broken.docx');
fake.failHtmlWith = null;
expect(job.body.status).toBe('failed');
expect(job.body.errorCode).toBe('conversion_failed');
expect(job.body.resultPageId).toBeNull();
// No page is created for a failed import.
expect(await prisma.page.count({ where: { pondId } })).toBe(pagesBefore);
});
it('fails and rolls back media when the pond runs out of storage', async () => {
// Cap this pond's storage below one image so the media store fails.
await prisma.quotaOverride.upsert({
where: {
subjectType_subjectId_quotaKey: {
subjectType: 'POND',
subjectId: pondId,
quotaKey: 'storage_bytes',
},
},
create: { subjectType: 'POND', subjectId: pondId, quotaKey: 'storage_bytes', value: 10 },
update: { value: 10 },
});
const usedBefore = Number(
(await prisma.pondUsage.findUnique({ where: { pondId } }))?.storageBytesUsed ?? 0n,
);
const pagesBefore = await prisma.page.count({ where: { pondId } });
fake.markdown = `# Too big\n\n![x](data:image/png;base64,${PNG_BASE64})`;
const job = await importDoc('too-big.docx');
expect(job.body.status).toBe('failed');
expect(job.body.errorCode).toBe('quota_exceeded');
// Neither a page nor any stored bytes survive the failed attempt.
expect(await prisma.page.count({ where: { pondId } })).toBe(pagesBefore);
const usedAfter = Number(
(await prisma.pondUsage.findUnique({ where: { pondId } }))?.storageBytesUsed ?? 0n,
);
expect(usedAfter).toBe(usedBefore);
await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } });
});
it('rejects an unsupported upload extension (400) before enqueuing', async () => {
const res = await api()
.post(`/api/v1/ponds/${pondId}/import`)
.set('Cookie', ownerCookie)
.attach('file', Buffer.from('hello'), 'notes.txt')
.expect(400);
expect(res.body.code).toBe('import_unsupported_format');
});
it('hides the pond from a non-member (404, not an import)', async () => {
await api()
.post(`/api/v1/ponds/${pondId}/import`)
.set('Cookie', outsiderCookie)
.attach('file', Buffer.from('x'), 'doc.docx')
.expect(404);
});
});

View File

@ -0,0 +1,322 @@
import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/common';
import { ConversionJob, User } from '@prisma/client';
import { ConversionJobView, editorSchema, markdownToDoc } from '@dorfteich/shared';
import { Node } from 'prosemirror-model';
import { PinoLogger } from 'nestjs-pino';
import { FilesService } from '../files/files.service';
import { PagesService } from '../pages/pages.service';
import { docToState } from '../pages/yjs-content';
import { PrismaService } from '../prisma/prisma.service';
import { ImportProcessor } from './import.constants';
import { ConversionError, PandocConverter } from './pandoc.converter';
import { ConversionJobService } from './conversion-job.service';
/** pandoc source format per accepted upload extension (ADR 0009). */
const IMPORT_FORMATS: Readonly<Record<string, string>> = {
docx: 'docx',
odt: 'odt',
};
/** Job `kind` per source format, so the worker can route the job to the import
* pipeline (a plain bytebyte conversion has a different kind, #62/#65). */
function importKind(format: string): string {
return `import_${format}`;
}
/**
* Writer format for the structural pass. GFM matches our editor schema
* (tables, task lists, strikethrough); `-implicit_figures` keeps images as
* inline `![alt](src)` instead of wrapping them in a figure with a duplicated
* caption; `-raw_html` stops pandoc emitting raw HTML for constructs it can't
* represent (our Markdown parser runs with `html:false` and would drop it
* anyway per ADR 0009 we preserve structure, not layout).
*/
const IMPORT_MARKDOWN_FORMAT = 'gfm-implicit_figures-raw_html';
/**
* The two-pass conversion at the heart of import, extracted so tests can run it
* over the fixture corpus against a real sidecar. pandoc-server is stateless and
* hands back a document's media no other way, so we first inline every image as
* a `data:` URI (`source → html`, embed-resources), then produce clean
* structural Markdown that still carries those data URIs inline at the right
* positions (`html → gfm`, no wrapping).
*/
export async function convertImportedDocument(
converter: PandocConverter,
sourceFormat: string,
input: Buffer,
): Promise<string> {
const html = await converter.convert({
from: sourceFormat,
to: 'html',
input,
standalone: false,
embedResources: true,
});
const md = await converter.convert({
from: 'html',
to: IMPORT_MARKDOWN_FORMAT,
input: html.output,
standalone: false,
wrap: 'none',
});
return md.output.toString('utf8');
}
/** Minimal shape of a ProseMirror document as JSON enough to walk it for
* image nodes and the leading heading without pulling in prosemirror types. */
interface PmNode {
type: string;
attrs?: Record<string, unknown>;
content?: PmNode[];
text?: string;
marks?: unknown[];
}
interface DecodedImage {
buffer: Buffer;
extension: string;
}
/**
* Imports `.docx`/`.odt` documents as new pages (ADR 0009, issue #63). Enqueue
* is synchronous and cheap (validate + persist a job on the #62 queue); the
* heavy conversion runs out of band in {@link run}, invoked by the shared
* {@link ConversionWorker} for import-kind jobs so it inherits the queue's
* locking, retry, and restart-survival.
*
* Pipeline: the pandoc sidecar is stateless and will not hand back a
* document's embedded media any other way, so we convert in two passes
* `docx/odt → html` with `embed-resources` inlines every image as a `data:`
* URI, then `html → gfm` produces clean structural Markdown with those data
* URIs still inline at the right positions. We parse that to an editor
* document, store each embedded image as a pond file (with quota accounting)
* and rewrite its reference to the file id, derive the title from a leading
* top-level heading (else the file name), and create the page from the
* resulting Yjs state.
*/
@Injectable()
export class ImportService implements ImportProcessor {
constructor(
private readonly prisma: PrismaService,
private readonly jobs: ConversionJobService,
private readonly converter: PandocConverter,
private readonly files: FilesService,
private readonly pages: PagesService,
private readonly logger: PinoLogger,
) {
this.logger.setContext(ImportService.name);
}
/** Validate and enqueue an import; the returned job id is polled via
* `GET /jobs/:id` until it reports the created `resultPageId` (#62). */
async enqueue(
user: User,
pondId: string,
file: { buffer: Buffer; originalname: string },
): Promise<ConversionJobView> {
const extension = fileExtension(file.originalname);
const format = extension ? IMPORT_FORMATS[extension] : undefined;
if (!format) {
throw new BadRequestException({ code: 'import_unsupported_format' });
}
const job = await this.jobs.enqueue({
ownerId: user.id,
pondId,
kind: importKind(format),
from: format,
to: 'page',
input: file.buffer,
sourceName: file.originalname,
standalone: false,
});
return this.jobs.viewOf(job);
}
/**
* Run one import job to completion (called by the worker). Throws a
* {@link ConversionError} on failure so the worker applies the queue's
* retry/fail policy; on success it records the created page on the job.
* Media stored during a failed attempt is rolled back so a retry does not
* leak files or double-count quota.
*/
async run(job: ConversionJob): Promise<void> {
if (!job.pondId) {
throw new ConversionError('conversion_failed', false, 'import job has no pond');
}
const user = await this.prisma.user.findUnique({ where: { id: job.ownerId } });
if (!user) {
throw new ConversionError('conversion_failed', false, 'import job owner is gone');
}
const rawMarkdown = await convertImportedDocument(
this.converter,
job.sourceFormat,
Buffer.from(job.input),
);
// Media is stored before the page exists (the page's state references the
// file ids); track what we create so a later failure can be rolled back.
const storedFileIds: string[] = [];
try {
const markdown = await this.storeEmbeddedImages(rawMarkdown, user, job.pondId, storedFileIds);
const json = markdownToDoc(markdown).toJSON() as unknown as PmNode;
const { title, doc } = this.splitTitle(json, job.sourceName);
const state = docToState(Node.fromJSON(editorSchema, doc));
const page = await this.pages.createWithState(user, job.pondId, title, state);
await this.files.linkAttachmentsToPage(storedFileIds, page.id);
await this.prisma.conversionJob.update({
where: { id: job.id },
data: { status: 'SUCCEEDED', resultPageId: page.id, errorCode: null },
});
this.logger.info(
{ jobId: job.id, pageId: page.id, pondId: job.pondId, images: storedFileIds.length },
'audit: document imported',
);
} catch (error) {
await this.rollbackMedia(user, storedFileIds);
throw error;
}
}
/**
* Store each embedded image (an inline `data:` URI, from the embed-resources
* pass) as a pond file and rewrite the Markdown reference to the stored file
* id. Done on the Markdown text, before parsing, because the editor's parser
* only admits a whitelist of `data:` image types (png/jpeg/gif/webp) and
* would leave any other inline as a huge literal base64 string. base64 has no
* `)`, so the image regex is unambiguous. An image whose bytes the upload
* pipeline rejects (e.g. a vector `image/x-emf` Word can embed) is dropped
* rather than failing the whole import structure over layout (ADR 0009); a
* pond that runs out of storage fails the import.
*/
private async storeEmbeddedImages(
markdown: string,
user: User,
pondId: string,
storedFileIds: string[],
): Promise<string> {
const image = /!\[([^\]]*)\]\((data:[^)\s]+)\)/g;
const uris = new Set<string>();
for (const match of markdown.matchAll(image)) uris.add(match[2]!);
if (uris.size === 0) return markdown;
// Dedup identical images so a document that repeats one stores it once.
const resolved = new Map<string, string | null>();
let index = 0;
for (const uri of uris) {
resolved.set(uri, await this.storeImage(uri, user, pondId, ++index, storedFileIds));
}
return markdown.replace(image, (_whole, alt: string, uri: string) => {
const fileId = resolved.get(uri);
return fileId ? `![${alt}](${fileId})` : '';
});
}
private async storeImage(
uri: string,
user: User,
pondId: string,
index: number,
storedFileIds: string[],
): Promise<string | null> {
const decoded = decodeDataUri(uri);
if (!decoded) return null;
try {
const view = await this.files.upload(user, pondId, {
buffer: decoded.buffer,
size: decoded.buffer.length,
originalname: `import-${index}.${decoded.extension}`,
});
storedFileIds.push(view.id);
return view.id;
} catch (error) {
if (error instanceof ForbiddenException && errorCode(error) === 'quota_exceeded') {
throw new ConversionError('quota_exceeded', false, 'pond storage exhausted during import');
}
// Unsupported image type / rejected bytes — drop this one, keep going.
this.logger.warn({ pondId, code: errorCode(error) }, 'import: dropped an image');
return null;
}
}
/**
* Use a leading top-level heading as the page title (and remove it from the
* body so it is not duplicated), else fall back to the upload file name
* without its extension, else a generic title.
*/
private splitTitle(root: PmNode, sourceName: string | null): { title: string; doc: PmNode } {
const content = root.content ?? [];
const first = content[0];
if (first && first.type === 'heading') {
const headingText = textOf(first).trim();
if (headingText) {
return { title: headingText, doc: { ...root, content: content.slice(1) } };
}
}
const fallback = baseName(sourceName) || 'Imported document';
return { title: fallback, doc: root };
}
private async rollbackMedia(user: User, fileIds: string[]): Promise<void> {
for (const id of fileIds) {
try {
await this.files.remove(user, id);
} catch (error) {
this.logger.warn({ fileId: id, code: errorCode(error) }, 'import: media rollback failed');
}
}
}
}
/** Concatenated text of a node's inline content (headings have no nesting). */
function textOf(node: PmNode): string {
if (typeof node.text === 'string') return node.text;
return (node.content ?? []).map(textOf).join('');
}
const DATA_URI = /^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/;
const EXTENSION_BY_MIME: Readonly<Record<string, string>> = {
'image/png': 'png',
'image/jpeg': 'jpg',
'image/gif': 'gif',
'image/webp': 'webp',
'image/svg+xml': 'svg',
};
function decodeDataUri(value: string): DecodedImage | null {
const match = DATA_URI.exec(value);
if (!match) return null;
const mime = match[1]!;
const buffer = Buffer.from(match[2]!, 'base64');
if (buffer.length === 0) return null;
return { buffer, extension: EXTENSION_BY_MIME[mime] ?? 'bin' };
}
/** Lowercased extension without the dot, or '' when there is none. */
function fileExtension(fileName: string): string {
const dot = fileName.lastIndexOf('.');
return dot >= 0 ? fileName.slice(dot + 1).toLowerCase() : '';
}
/** File name without its extension. */
function baseName(fileName: string | null): string {
if (!fileName) return '';
const dot = fileName.lastIndexOf('.');
return (dot > 0 ? fileName.slice(0, dot) : fileName).trim();
}
/** The `code` from a Nest HttpException response body, if any. */
function errorCode(error: unknown): string | undefined {
if (error instanceof BadRequestException || error instanceof ForbiddenException) {
const body = error.getResponse();
if (body && typeof body === 'object' && 'code' in body) {
const code = (body as { code?: unknown }).code;
return typeof code === 'string' ? code : undefined;
}
}
return undefined;
}

View File

@ -10,6 +10,13 @@ export interface ConversionRequest {
to: string; to: string;
input: Buffer; input: Buffer;
standalone?: boolean; standalone?: boolean;
/** Inline referenced/embedded resources (images) as `data:` URIs in the
* output. Used by the import pipeline (#63): pandoc-server is stateless and
* will not hand back a document's media bytes any other way. */
embedResources?: boolean;
/** Line-wrapping of the writer's output. Import uses `none` so a paragraph
* stays on one line (no soft breaks inside image alt text or links). */
wrap?: 'none' | 'auto' | 'preserve';
} }
export interface ConversionResult { export interface ConversionResult {
@ -18,7 +25,12 @@ export interface ConversionResult {
} }
export type ConversionErrorCode = export type ConversionErrorCode =
'converter_unavailable' | 'converter_timeout' | 'conversion_failed'; | 'converter_unavailable'
| 'converter_timeout'
| 'conversion_failed'
// The pond ran out of storage while an import stored the document's media
// (#63) — final, and surfaced the same way through the worker.
| 'quota_exceeded';
/** A conversion failure with a stable, localizable code. `retryable` marks /** A conversion failure with a stable, localizable code. `retryable` marks
* the transient causes (sidecar down / timed out) the worker retries before * the transient causes (sidecar down / timed out) the worker retries before
@ -111,6 +123,10 @@ export class PandocServerConverter extends PandocConverter {
from: request.from, from: request.from,
to: request.to, to: request.to,
standalone: request.standalone ?? true, standalone: request.standalone ?? true,
// pandoc-server uses hyphenated option keys; unknown keys are ignored,
// so these are only present when the import pipeline sets them.
...(request.embedResources ? { 'embed-resources': true } : {}),
...(request.wrap ? { wrap: request.wrap } : {}),
}), }),
signal: controller.signal, signal: controller.signal,
}); });

View File

@ -130,23 +130,49 @@ export class PagesService {
} }
async create(user: User, pondId: string, input: CreatePageInput): Promise<PageView> { async create(user: User, pondId: string, input: CreatePageInput): Promise<PageView> {
const page = await this.insertPage(user, pondId, input.title, emptyPageState());
return this.viewOf(page);
}
/**
* Create a page from a prepared Yjs state (issue #63 import): the imported
* document is already a full Yjs state whose fragment the editor binds to, so
* an opening client sees the converted content immediately. Same invariants
* as {@link create} unique slug, appended sort key, derived content cache,
* phantom-link resolution, search indexing. Returns the persisted row so the
* caller (the import worker) can link the document's media to it.
*/
async createWithState(
user: User,
pondId: string,
title: string,
state: Uint8Array<ArrayBuffer>,
): Promise<Page> {
return this.insertPage(user, pondId, title, state);
}
private async insertPage(
user: User,
pondId: string,
title: string,
state: Uint8Array<ArrayBuffer>,
): Promise<Page> {
const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } }); const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } });
if (!pond) throw new NotFoundException(); if (!pond) throw new NotFoundException();
const slug = await this.generateUniqueSlugInPond(pond.id, input.title); const slug = await this.generateUniqueSlugInPond(pond.id, title);
const last = await this.prisma.page.findFirst({ const last = await this.prisma.page.findFirst({
where: { pondId: pond.id }, where: { pondId: pond.id },
orderBy: { sortKey: 'desc' }, orderBy: { sortKey: 'desc' },
select: { sortKey: true }, select: { sortKey: true },
}); });
const sortKey = generateKeyBetween(last?.sortKey ?? null, null); const sortKey = generateKeyBetween(last?.sortKey ?? null, null);
const state = emptyPageState();
const content = deriveContent(state); const content = deriveContent(state);
const page = await this.prisma.page.create({ const page = await this.prisma.page.create({
data: { data: {
pondId: pond.id, pondId: pond.id,
title: input.title, title,
slug, slug,
sortKey, sortKey,
ydocState: state, ydocState: state,
@ -156,10 +182,10 @@ export class PagesService {
}); });
// A new page may satisfy phantom wikilinks that referenced its slug (#47). // A new page may satisfy phantom wikilinks that referenced its slug (#47).
await this.resolvePhantomLinks(pond.id, slug, page.id); await this.resolvePhantomLinks(pond.id, slug, page.id);
// Index the (empty) page so a title-only match is findable immediately (#49). // Index the page so a title-only match is findable immediately (#49).
await this.search.indexPage(page.id); await this.search.indexPage(page.id);
this.logger.info({ pageId: page.id, pondId: pond.id, userId: user.id }, 'audit: page created'); this.logger.info({ pageId: page.id, pondId: pond.id, userId: user.id }, 'audit: page created');
return this.viewOf(page); return page;
} }
/** /**

View File

@ -32,13 +32,17 @@ function docFromState(state: Uint8Array): Node {
} }
} }
/** A fresh Yjs state containing a single empty paragraph. */ /**
export function emptyPageState(): Uint8Array<ArrayBuffer> { * Encode a ProseMirror document as the initial Yjs state a page is created
* with. Used both for a fresh empty page and for importing a converted
* document (#63) as a page's starting content the editor binds to the same
* {@link FRAGMENT_NAME}, so an opening client sees exactly this document.
*/
export function docToState(doc: Node): Uint8Array<ArrayBuffer> {
const ydoc = new Y.Doc(); const ydoc = new Y.Doc();
try { try {
const fragment = ydoc.getXmlFragment(FRAGMENT_NAME); const fragment = ydoc.getXmlFragment(FRAGMENT_NAME);
const emptyDoc = editorSchema.node('doc', null, [editorSchema.node('paragraph')]); prosemirrorJSONToYXmlFragment(editorSchema, doc.toJSON(), fragment);
prosemirrorJSONToYXmlFragment(editorSchema, emptyDoc.toJSON(), fragment);
// Copy into a plain ArrayBuffer-backed view — yjs's own return type is // Copy into a plain ArrayBuffer-backed view — yjs's own return type is
// the wider `Uint8Array<ArrayBufferLike>`, which Prisma's Bytes input // the wider `Uint8Array<ArrayBufferLike>`, which Prisma's Bytes input
// (`Uint8Array<ArrayBuffer>`) does not accept directly. // (`Uint8Array<ArrayBuffer>`) does not accept directly.
@ -48,6 +52,11 @@ export function emptyPageState(): Uint8Array<ArrayBuffer> {
} }
} }
/** A fresh Yjs state containing a single empty paragraph. */
export function emptyPageState(): Uint8Array<ArrayBuffer> {
return docToState(editorSchema.node('doc', null, [editorSchema.node('paragraph')]));
}
export interface DerivedPageContent { export interface DerivedPageContent {
plainText: string; plainText: string;
markdown: string; markdown: string;

View File

@ -22,7 +22,13 @@ export default tseslint.config(
// Plain-Node maintenance scripts (no TypeScript, no bundler). // Plain-Node maintenance scripts (no TypeScript, no bundler).
files: ['scripts/**/*.mjs'], files: ['scripts/**/*.mjs'],
languageOptions: { languageOptions: {
globals: { console: 'readonly', process: 'readonly', URL: 'readonly' }, globals: {
console: 'readonly',
process: 'readonly',
URL: 'readonly',
fetch: 'readonly',
Buffer: 'readonly',
},
}, },
}, },
{ {

42
fixtures/import/README.md Normal file
View File

@ -0,0 +1,42 @@
# Import fixture corpus (issue #63, ADR 0009)
Representative `.docx`/`.odt` documents and the Markdown our import pipeline is
expected to produce from them. `import.fixtures.test.ts` runs the real two-pass
pandoc conversion over each and asserts the result, so a change in behaviour
(ours or pandoc's) surfaces as a snapshot diff to review — not a silent
regression.
## Files
For each document `<name>` and format `<ext>` (`docx`, `odt`):
- `<name>.<ext>` — the source document.
- `<name>.<ext>.expected.md` — the Markdown the pipeline produces, with every
embedded image's `data:` URI normalised to the literal `data:embedded-image`
(the base64 payload is volatile and not what the snapshot is pinning).
`article` covers headings, paragraphs, **bold**/_italic_, a link, a bullet list
with nesting, an ordered list, an embedded image, and a table. `formatting`
covers strikethrough, inline code, a blockquote, and three levels of nesting.
## Fidelity notes (structure, not layout — ADR 0009)
- **ODT images lose their alt text** and **ODT tables lose their header row**:
pandoc's HTML→ODT _writer_ does not encode either, so the source documents
genuinely lack them. The DOCX variants keep both. This is a pinned pandoc
limitation, not an import bug — extend the corpus rather than chasing it.
## Limits (ADR 0009)
An uploaded document may be at most **25 MiB** (`MAX_CONVERSION_INPUT_BYTES`),
and each of the two conversion passes may run for at most **60 s**
(`CONVERSION_TIMEOUT_MS`). A ~50-page document converts well inside that ceiling
(`import.fixtures.test.ts` asserts it); a document that exceeds the timeout fails
the job with `converter_timeout`.
## Regenerating
The documents and snapshots are generated with the **pinned** `pandoc/core:3.6`
(the production sidecar) so the snapshots match CI; a different pandoc version
wraps lists and pads tables differently. `scripts/gen-import-fixtures.mjs`
regenerates everything against a reachable pandoc sidecar (`PANDOC_URL`).

Binary file not shown.

View File

@ -0,0 +1,29 @@
# Field Notes
An introduction with **bold**, *italic*, and a [link to the pond](https://example.com/pond).
## Observations
The pond hosts several species. Notable ones include an image below.
![A calm pond](data:embedded-image)
## Lists
- Amphibians
- Frogs
- Newts
- Insects
- Dragonflies
- Water striders
1. Spring survey
2. Summer survey
3. Autumn survey
## Measurements
| Month | Depth (cm) |
|-------|------------|
| April | 120 |
| July | 95 |

BIN
fixtures/import/article.odt Normal file

Binary file not shown.

View File

@ -0,0 +1,32 @@
# Field Notes
An introduction with **bold**, *italic*, and a [link to the pond](https://example.com/pond).
## Observations
The pond hosts several species. Notable ones include an image below.
![](data:embedded-image)
## Lists
- Amphibians
- Frogs
- Newts
- Insects
- Dragonflies
- Water striders
1. Spring survey
2. Summer survey
3. Autumn survey
## Measurements
| | |
|-------|-----|
| April | 120 |
| July | 95 |

View File

@ -0,0 +1,21 @@
<h1>Field Notes</h1>
<p>An introduction with <strong>bold</strong>, <em>italic</em>, and a <a href="https://example.com/pond">link to the pond</a>.</p>
<h2>Observations</h2>
<p>The pond hosts several species. Notable ones include an image below.</p>
<p><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAAEUlEQVR4nGP8z8DAwMDAwAAADAABZuUq0AAAAABJRU5ErkJggg==" alt="A calm pond"></p>
<h2>Lists</h2>
<ul>
<li>Amphibians
<ul><li>Frogs</li><li>Newts</li></ul></li>
<li>Insects
<ul><li>Dragonflies</li><li>Water striders</li></ul></li>
</ul>
<ol><li>Spring survey</li><li>Summer survey</li><li>Autumn survey</li></ol>
<h2>Measurements</h2>
<table>
<thead><tr><th>Month</th><th>Depth (cm)</th></tr></thead>
<tbody>
<tr><td>April</td><td>120</td></tr>
<tr><td>July</td><td>95</td></tr>
</tbody>
</table>

Binary file not shown.

View File

@ -0,0 +1,13 @@
# Formatting Sampler
This paragraph mixes **bold**, *italic*, ~~strikethrough~~, and `inline code`.
> A quoted remark about still water.
### Nested detail
- Top level
- Second level
- Third level
A closing line with another [external link](https://example.org).

Binary file not shown.

View File

@ -0,0 +1,15 @@
# Formatting Sampler
This paragraph mixes **bold**, *italic*, ~~strikethrough~~, and `inline code`.
> A quoted remark about still water.
### Nested detail
- Top level
- Second level
- Third level
A closing line with another [external link](https://example.org).

View File

@ -0,0 +1,6 @@
<h1>Formatting Sampler</h1>
<p>This paragraph mixes <strong>bold</strong>, <em>italic</em>, <s>strikethrough</s>, and <code>inline code</code>.</p>
<blockquote><p>A quoted remark about still water.</p></blockquote>
<h3>Nested detail</h3>
<ul><li>Top level<ul><li>Second level<ul><li>Third level</li></ul></li></ul></li></ul>
<p>A closing line with another <a href="https://example.org">external link</a>.</p>

View File

@ -34,6 +34,7 @@
"converter_unavailable": "Der Dokument-Konverter ist derzeit nicht verfügbar. Bitte versuche es später erneut.", "converter_unavailable": "Der Dokument-Konverter ist derzeit nicht verfügbar. Bitte versuche es später erneut.",
"converter_timeout": "Die Konvertierung hat zu lange gedauert und wurde abgebrochen.", "converter_timeout": "Die Konvertierung hat zu lange gedauert und wurde abgebrochen.",
"conversion_failed": "Dieses Dokument konnte nicht konvertiert werden.", "conversion_failed": "Dieses Dokument konnte nicht konvertiert werden.",
"import_unsupported_format": "Nur Word- (.docx) und OpenDocument-Dokumente (.odt) können importiert werden.",
"network": "Der Server war nicht erreichbar.", "network": "Der Server war nicht erreichbar.",
"grant_exists": "Diese Berechtigung existiert bereits.", "grant_exists": "Diese Berechtigung existiert bereits.",
"grant_pond_admin_scope": "Eine Teich-Admin-Berechtigung muss für den ganzen Teich und eine bestimmte Person gelten.", "grant_pond_admin_scope": "Eine Teich-Admin-Berechtigung muss für den ganzen Teich und eine bestimmte Person gelten.",

View File

@ -34,6 +34,7 @@
"converter_unavailable": "The document converter is currently unavailable. Please try again later.", "converter_unavailable": "The document converter is currently unavailable. Please try again later.",
"converter_timeout": "The conversion took too long and was cancelled.", "converter_timeout": "The conversion took too long and was cancelled.",
"conversion_failed": "This document could not be converted.", "conversion_failed": "This document could not be converted.",
"import_unsupported_format": "Only Word (.docx) and OpenDocument (.odt) documents can be imported.",
"network": "The server could not be reached.", "network": "The server could not be reached.",
"grant_exists": "This grant already exists.", "grant_exists": "This grant already exists.",
"grant_pond_admin_scope": "A Pond Admin grant must apply to the whole pond and a specific user.", "grant_pond_admin_scope": "A Pond Admin grant must apply to the whole pond and a specific user.",

View File

@ -12,8 +12,17 @@ export interface ConversionJobView {
sourceFormat: string; sourceFormat: string;
targetFormat: string; targetFormat: string;
/** Set only when `status` is `failed` a code from the errors namespace /** Set only when `status` is `failed` a code from the errors namespace
* (`converter_unavailable` | `converter_timeout` | `conversion_failed`). */ * (`converter_unavailable` | `converter_timeout` | `conversion_failed` |
* `quota_exceeded`). */
errorCode: string | null; errorCode: string | null;
/** Set once an import job (#63) succeeds: the id of the page it created, so
* the client can navigate to it. `null` for a pending/failed import and for
* plain bytebyte conversions (export). */
resultPageId: string | null;
createdAt: string; createdAt: string;
updatedAt: string; updatedAt: string;
} }
/** Extensions the import endpoint accepts (ADR 0009, issue #63). */
export const IMPORT_EXTENSIONS = ['docx', 'odt'] as const;
export type ImportExtension = (typeof IMPORT_EXTENSIONS)[number];

View File

@ -0,0 +1,69 @@
#!/usr/bin/env node
// Regenerate the import fixture corpus (issue #63, ADR 0009) from the committed
// `*.src.html` sources: for each source, write `<name>.docx`, `<name>.odt`, and
// the expected Markdown snapshot for each format. Run against a reachable pandoc
// sidecar (the pinned `pandoc/core:3.6`, so snapshots match CI):
//
// docker run --rm -p 3030:3030 pandoc/core:3.6 server
// PANDOC_URL=http://localhost:3030 node scripts/gen-import-fixtures.mjs
//
// The pipeline mirrors ImportService: `html → <ext>` builds the document, then
// `<ext> → html` (embed-resources) → `html → gfm` reproduces what an import
// would parse, with image data URIs normalised to a stable token.
import { readFileSync, writeFileSync, readdirSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
const FIXTURES = join(dirname(fileURLToPath(import.meta.url)), '..', 'fixtures', 'import');
const PANDOC_URL = process.env.PANDOC_URL ?? 'http://localhost:3030';
const MARKDOWN_FORMAT = 'gfm-implicit_figures-raw_html';
/** POST a conversion to pandoc-server. Without an `Accept` header the server
* returns text for text writers and raw bytes for binary ones (docx/odt). */
async function pandoc(params, binaryOut = false) {
const res = await fetch(`${PANDOC_URL}/`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!res.ok) throw new Error(`pandoc ${res.status}: ${await res.text()}`);
return binaryOut ? Buffer.from(await res.arrayBuffer()) : res.text();
}
async function buildDocument(html, ext) {
return pandoc({ text: html, from: 'html', to: ext, standalone: true }, true);
}
async function toExpectedMarkdown(documentBytes, ext) {
const embedded = await pandoc({
text: documentBytes.toString('base64'),
from: ext,
to: 'html',
standalone: false,
'embed-resources': true,
});
const md = await pandoc({
text: embedded,
from: 'html',
to: MARKDOWN_FORMAT,
standalone: false,
wrap: 'none',
});
return md.replace(/data:image\/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g, 'data:embedded-image');
}
const sources = readdirSync(FIXTURES).filter((f) => f.endsWith('.src.html'));
for (const source of sources) {
const name = source.replace(/\.src\.html$/, '');
const html = readFileSync(join(FIXTURES, source), 'utf8');
for (const ext of ['docx', 'odt']) {
const bytes = await buildDocument(html, ext);
writeFileSync(join(FIXTURES, `${name}.${ext}`), bytes);
writeFileSync(
join(FIXTURES, `${name}.${ext}.expected.md`),
await toExpectedMarkdown(bytes, ext),
);
console.log(`wrote ${name}.${ext} (+ expected.md)`);
}
}