From aaa9a253aed6b8694b0572c81bf1d1c57f3f2b66 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.8" Date: Fri, 10 Jul 2026 13:59:10 +0200 Subject: [PATCH] Add import/export fidelity gate to CI (#69) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the "structure-true best effort" fidelity contract (ADR 0009) an objective, pipeline-gated suite so "best effort" cannot erode silently. - New CI job "Import/export fidelity gate" (.gitea/workflows/ci.yml) runs the corpus suites against the pinned sidecar images the stages use (pandoc/core:3.6, gotenberg/gotenberg:8), started via docker run and reached over the host gateway. Small and separate so it stays well under five minutes; the suites self-skip in the main checks job (no sidecars). - Export fidelity: fixtures/export corpus + gen-export-fixtures.mjs + export.fidelity.test.ts — exports Markdown to docx/odt through the real pinned pandoc and reads it back, snapshotting the round trip so a writer drift (ours or a version bump) fails the gate. - PDF smoke: pdf.fidelity.test.ts renders a page through real Gotenberg and asserts the extracted text and a sane page count (pdf-parse, dev-only). - Fidelity contract doc: fixtures/README.md defines "corpus green = fidelity acceptable" and the fixture-first bug process; per-corpus READMEs updated. Because the snapshots are byte-exact and generated with the pinned tools, bumping a sidecar without regenerating shifts the output and fails the suite (AC3). The import corpus (#63) is folded into the same gate. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1 --- .gitea/workflows/ci.yml | 74 ++++++++++ .prettierignore | 3 + apps/api/package.json | 1 + .../src/import-export/export.fidelity.test.ts | 73 ++++++++++ .../src/import-export/pdf.fidelity.test.ts | 73 ++++++++++ fixtures/README.md | 70 +++++++++ fixtures/export/README.md | 41 ++++++ fixtures/export/article.docx.expected.md | 12 ++ fixtures/export/article.md | 12 ++ fixtures/export/article.odt.expected.md | 14 ++ fixtures/export/formatting.docx.expected.md | 7 + fixtures/export/formatting.md | 9 ++ fixtures/export/formatting.odt.expected.md | 7 + fixtures/import/README.md | 4 +- pnpm-lock.yaml | 133 ++++++++++++++++++ scripts/gen-export-fixtures.mjs | 54 +++++++ 16 files changed, 586 insertions(+), 1 deletion(-) create mode 100644 apps/api/src/import-export/export.fidelity.test.ts create mode 100644 apps/api/src/import-export/pdf.fidelity.test.ts create mode 100644 fixtures/README.md create mode 100644 fixtures/export/README.md create mode 100644 fixtures/export/article.docx.expected.md create mode 100644 fixtures/export/article.md create mode 100644 fixtures/export/article.odt.expected.md create mode 100644 fixtures/export/formatting.docx.expected.md create mode 100644 fixtures/export/formatting.md create mode 100644 fixtures/export/formatting.odt.expected.md create mode 100644 scripts/gen-export-fixtures.mjs diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 7b7e457..6f77196 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -336,6 +336,80 @@ jobs: if: failure() run: tail -50 /tmp/api.log /tmp/collab.log /tmp/web.log || true + # The import/export fidelity gate (issue #69, ADR 0009): runs the corpus + # snapshot suites and the PDF smoke check against the *pinned* sidecar images + # (the same versions the stages run), so a structural regression — ours or a + # pandoc/Gotenberg version bump that drifts the output — fails the pipeline + # instead of degrading "best effort" silently. Kept a small, separate job so + # it stays well under five minutes; the suites self-skip in the main `checks` + # job (no sidecars there). + fidelity: + name: Import/export fidelity gate + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up pnpm + uses: pnpm/action-setup@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # The fidelity tests import from @dorfteich/shared's built dist; they run + # TypeScript directly (vitest) so only shared needs building, not the api. + - name: Build shared package + run: pnpm --filter @dorfteich/shared build + + # The import corpus test reaches through import.service, which imports the + # generated Prisma client — generate it (no DB or migration needed; these + # suites never touch a database). + - name: Generate Prisma client + run: pnpm --filter @dorfteich/api exec prisma generate + + # Start the pinned sidecars with `docker run` (pandoc-server needs the + # `server` arg, which Actions `services:` cannot pass). Their published + # ports are reached from this job container via the Docker host gateway — + # robust regardless of the runner's per-job network name. Uncommon host + # ports avoid colliding with anything else on the runner. + - name: Start pinned pandoc + Gotenberg sidecars + run: | + docker run -d --name fidelity-pandoc -p 13030:3030 pandoc/core:3.6 server + docker run -d --name fidelity-gotenberg -p 13000:3000 gotenberg/gotenberg:8 + GW=$(ip -4 route show default | awk '{print $3; exit}') + echo "SIDECAR_HOST=$GW" >> "$GITHUB_ENV" + for i in $(seq 1 30); do + curl -sf "http://$GW:13030/version" >/dev/null && break + sleep 1 + done + for i in $(seq 1 30); do + curl -sf "http://$GW:13000/health" >/dev/null && break + sleep 1 + done + curl -sf "http://$GW:13030/version" + curl -sf "http://$GW:13000/health" + + - name: Run fidelity suite (import + export snapshots, PDF smoke) + run: | + PANDOC_URL="http://${SIDECAR_HOST}:13030" \ + GOTENBERG_URL="http://${SIDECAR_HOST}:13000" \ + pnpm --filter @dorfteich/api exec vitest run \ + src/import-export/import.fixtures.test.ts \ + src/import-export/export.fidelity.test.ts \ + src/import-export/pdf.fidelity.test.ts + + - name: Dump sidecar logs on failure + if: failure() + run: | + echo '--- pandoc ---'; docker logs fidelity-pandoc 2>&1 | tail -30 || true + echo '--- gotenberg ---'; docker logs fidelity-gotenberg 2>&1 | tail -30 || true + images: name: Build container images # PR-only: on main the CD workflow builds and pushes the same images — diff --git a/.prettierignore b/.prettierignore index 35e1aed..a19c045 100644 --- a/.prettierignore +++ b/.prettierignore @@ -11,3 +11,6 @@ apps/api/prisma/fixtures/content-page.md # Markdown/HTML opinions would break the regression tests. fixtures/import/*.expected.md fixtures/import/*.src.html +# Export fidelity corpus (issue #69): the round-trip snapshots must stay exactly +# as pandoc reads the exported document back — Prettier would break them. +fixtures/export/*.expected.md diff --git a/apps/api/package.json b/apps/api/package.json index 4db6c2e..ac00f7f 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -56,6 +56,7 @@ "@types/nodemailer": "^8.0.1", "@types/supertest": "^6.0.0", "fflate": "^0.8.3", + "pdf-parse": "^2.4.5", "pino-pretty": "^13.0.0", "supertest": "^7.0.0", "tsx": "^4.19.0", diff --git a/apps/api/src/import-export/export.fidelity.test.ts b/apps/api/src/import-export/export.fidelity.test.ts new file mode 100644 index 0000000..08858d7 --- /dev/null +++ b/apps/api/src/import-export/export.fidelity.test.ts @@ -0,0 +1,73 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { beforeAll, describe, expect, it, TestContext } from 'vitest'; + +import { AppConfig } from '../config/app-config.service'; + +import { markdownForDocument } from './export-markdown'; +import { PandocServerConverter } from './pandoc.converter'; + +/** + * Export fidelity regression (issue #69, ADR 0009): exports the committed + * Markdown corpus to `.docx`/`.odt` through the real pinned pandoc and reads + * each document back, asserting the round-trip Markdown matches its snapshot. + * This gates the export writer's structural fidelity — a pandoc version bump + * that shifts the office output surfaces as a snapshot diff to review, not a + * silent regression. Needs a reachable pandoc sidecar (`pandoc/core:3.6`, so + * output matches the snapshots); each test skips itself when none is + * configured, and CI starts one and points `PANDOC_URL` at it. + */ +const PANDOC_URL = process.env.PANDOC_URL ?? 'http://localhost:3030'; +// The runner's cwd is `apps/api`; the corpus lives at the repo root. +const FIXTURES = join(process.cwd(), '../../fixtures/export'); +// Same reader options as the import corpus, so both directions normalise alike. +const MARKDOWN_FORMAT = 'gfm-implicit_figures-raw_html'; + +const converter = new PandocServerConverter({ env: { PANDOC_URL } } as unknown as AppConfig); + +let reachable = false; + +const CORPUS = ['article', 'formatting']; +const FORMATS = ['docx', 'odt'] as const; + +/** Export `markdown` to an office document, then read it back to Markdown — + * exactly the ExportService write path (gfm → ext, standalone) followed by a + * re-read (ext → gfm). */ +async function roundTrip(markdown: string, ext: 'docx' | 'odt'): Promise { + const document = await converter.convert({ + from: 'gfm', + to: ext, + input: Buffer.from(markdown, 'utf8'), + standalone: true, + }); + const back = await converter.convert({ + from: ext, + to: MARKDOWN_FORMAT, + input: document.output, + standalone: false, + wrap: 'none', + }); + return back.output.toString('utf8'); +} + +describe('export fidelity corpus (real pandoc, issue #69)', () => { + beforeAll(async () => { + reachable = await converter.reachable().catch(() => false); + }); + + for (const name of CORPUS) { + for (const ext of FORMATS) { + it(`round-trips ${name}.md through ${ext} to its expected Markdown`, async (ctx: TestContext) => { + if (!reachable) ctx.skip(); + const source = readFileSync(join(FIXTURES, `${name}.md`), 'utf8'); + // The export transform (flatten wikilinks, inline images) runs first; for + // this text corpus it is an identity, so the snapshot isolates pandoc's + // office-writer fidelity — the part that drifts on a version bump. + const document = markdownForDocument(source, new Map()); + const expected = readFileSync(join(FIXTURES, `${name}.${ext}.expected.md`), 'utf8'); + expect(await roundTrip(document, ext)).toBe(expected); + }); + } + } +}); diff --git a/apps/api/src/import-export/pdf.fidelity.test.ts b/apps/api/src/import-export/pdf.fidelity.test.ts new file mode 100644 index 0000000..aec39ee --- /dev/null +++ b/apps/api/src/import-export/pdf.fidelity.test.ts @@ -0,0 +1,73 @@ +import { DEFAULT_FONTS, PondFonts } from '@dorfteich/shared'; +import { PDFParse } from 'pdf-parse'; +import { beforeAll, describe, expect, it, TestContext } from 'vitest'; + +import { AppConfig } from '../config/app-config.service'; + +import { GotenbergHttpRenderer } from './gotenberg.renderer'; +import { buildPdfHtml } from './pdf-html'; + +/** + * PDF export smoke check (issue #69, ADR 0009): renders a known page to PDF + * through the real pinned Gotenberg and asserts the output is a PDF whose + * extracted text carries the expected strings, with a sane page count. This + * catches a broken renderer or a template regression that unit tests (which use + * a fake renderer) cannot. Needs a reachable Gotenberg sidecar + * (`gotenberg/gotenberg:8`); the test skips itself when none is configured, and + * CI starts one and points `GOTENBERG_URL` at it. + */ +const GOTENBERG_URL = process.env.GOTENBERG_URL ?? 'http://localhost:3000'; + +const renderer = new GotenbergHttpRenderer({ env: { GOTENBERG_URL } } as unknown as AppConfig); + +let reachable = false; + +/** Extract the concatenated text and page count from PDF bytes. */ +async function readPdf(pdf: Buffer): Promise<{ text: string; pages: number }> { + const parser = new PDFParse({ data: new Uint8Array(pdf) }); + try { + const result = await parser.getText(); + return { text: result.text, pages: result.total }; + } finally { + await parser.destroy(); + } +} + +describe('PDF export smoke (real Gotenberg, issue #69)', () => { + beforeAll(async () => { + reachable = await renderer.reachable().catch(() => false); + }); + + it('renders a page to a PDF containing its text, with a sane page count', async (ctx: TestContext) => { + if (!reachable) ctx.skip(); + const html = buildPdfHtml({ + title: 'Pond Fidelity Report', + pondName: 'Fidelity Pond', + // A forced page break so we can assert multi-page sanity; the markers are + // distinctive strings we can look for in the extracted text. + bodyHtml: + '

The northern reeds have spread noticeably this season.

' + + '
' + + '

Recorded water temperature was fourteen degrees.

', + fonts: DEFAULT_FONTS as PondFonts, + // No inlined font faces — the render falls back to the system stack, which + // still produces selectable text (fonts are covered by the #66/#67 tests). + fontFaceCss: '', + }); + + const pdf = await renderer.renderHtmlToPdf(html); + expect(pdf.subarray(0, 5).toString('latin1')).toBe('%PDF-'); + + const { text, pages } = await readPdf(pdf); + // Title + pond name (from the header) and both body markers survive to text. + expect(text).toContain('Pond Fidelity Report'); + expect(text).toContain('Fidelity Pond'); + expect(text).toContain('northern reeds'); + expect(text).toContain('water temperature'); + // Page-count sanity: the forced break must produce a second page (≥ 2), and + // this trivial document must not balloon (≤ 3) — a runaway render from a CSS + // regression would blow well past that. + expect(pages).toBeGreaterThanOrEqual(2); + expect(pages).toBeLessThanOrEqual(3); + }); +}); diff --git a/fixtures/README.md b/fixtures/README.md new file mode 100644 index 0000000..c6126c8 --- /dev/null +++ b/fixtures/README.md @@ -0,0 +1,70 @@ +# Import/export fidelity corpus (issue #69, ADR 0009) + +The fidelity contract for document conversion is **structure-true best effort**: +headings, paragraphs, lists, tables, images, links, and inline emphasis are +preserved; layout (columns, text boxes, exact spacing) is explicitly out of +scope (ADR 0009). "Best effort" only stays meaningful if it is executable, so +this corpus **is** the contract: + +> **The corpus is green ⇒ current fidelity is acceptable.** + +The gate runs in CI (`.gitea/workflows/ci.yml`, job _Import/export fidelity +gate_) against the **pinned** sidecar images the stages run — `pandoc/core:3.6` +and `gotenberg/gotenberg:8`. It is deliberately a small, separate job that stays +well under five minutes; the same suites self-skip in the main `checks` job, +which has no sidecars. + +## What the gate covers + +| Suite | File | What it pins | +| ---------------- | ---------------------------------------------------- | ------------------------------------------------------------------------------------ | +| Import snapshots | `apps/api/src/import-export/import.fixtures.test.ts` | `.docx`/`.odt` → Markdown for the [`import/`](./import) corpus | +| Export snapshots | `apps/api/src/import-export/export.fidelity.test.ts` | Markdown → `.docx`/`.odt` → Markdown round trip for the [`export/`](./export) corpus | +| PDF smoke | `apps/api/src/import-export/pdf.fidelity.test.ts` | a rendered PDF contains the expected text, with a sane page count | + +Each corpus directory has its own README describing its files. + +## The fixture-first bug process (the contract in practice) + +A fidelity problem is not fixed by hand-patching output until one document looks +right — that lets "best effort" quietly erode. Instead, **every fidelity bug +enters the corpus first**: + +1. **Reproduce it as a fixture.** Add the smallest document that exhibits the + problem to the relevant corpus (`import/` a source document, `export/` a + source `.md`). If the shape is new, extend an existing source rather than + adding a near-duplicate. +2. **Regenerate the snapshot** with the pinned sidecar so it records _today's_ + real output: + - import: `PANDOC_URL=… node scripts/gen-import-fixtures.mjs` + - export: `PANDOC_URL=… node scripts/gen-export-fixtures.mjs` + (start the sidecar with `docker run --rm -p 3030:3030 pandoc/core:3.6 server`). +3. **Commit the fixture and its snapshot together**, and review the snapshot + diff — it is the human-readable statement of what the conversion does. If the + snapshot encodes the _bug_, fix the pipeline and regenerate until the diff + shows the intended structure; if it encodes a genuine pinned-tool limitation + (see below), the snapshot _is_ the accepted behaviour and the fix is + documentation, not code. +4. **The gate now defends it.** Any later regression — ours or a tool's — + changes that document's output and fails the suite. + +## Why pinning matters (and how a version bump fails the gate) + +The snapshots are byte-exact and generated with the pinned sidecar versions. +pandoc changes its writers between releases (3.10 wraps lists and pads tables +differently from 3.6), so **bumping `pandoc/core` — or `gotenberg/gotenberg` — +without regenerating shifts the output and fails the suite.** That is the point: +a version change becomes a reviewed snapshot diff, never a silent drift. To bump +a sidecar: change the pin in `deploy/compose/docker-compose.yml`, the CI job, and +these tests' defaults; regenerate the corpus; review the diff; commit together. + +## Known pinned-tool limitations (accepted, not bugs) + +- **ODT loses image alt text and table header rows**: pandoc's HTML→ODT _writer_ + does not encode either, so the ODT sources genuinely lack them (the DOCX + variants keep both). +- **ODT loosens tight lists and flattens fenced code blocks** to plain + paragraphs on the export round trip; DOCX preserves both. + +These are recorded in the corpus snapshots on purpose — extend the corpus rather +than chasing layout parity. diff --git a/fixtures/export/README.md b/fixtures/export/README.md new file mode 100644 index 0000000..46a2135 --- /dev/null +++ b/fixtures/export/README.md @@ -0,0 +1,41 @@ +# Export fixture corpus (issue #69, ADR 0009) + +Source Markdown documents and the Markdown our export produces after a full +**export → re-read round trip** through the pinned pandoc. `export.fidelity.test.ts` +exports each source to `.docx`/`.odt` exactly as `ExportService` does +(`gfm → `, standalone), reads the document back (` → gfm`), and asserts +the result matches its snapshot — so a change in pandoc's office _writer_ (ours +or a version bump) surfaces as a snapshot diff to review, not a silent +regression. See [`../README.md`](../README.md) for the fidelity contract and the +fixture-first bug process. + +## Files + +For each source `.md` and format `` (`docx`, `odt`): + +- `.md` — the source Markdown (the shape a page's cached Markdown has). +- `..expected.md` — the Markdown read back after exporting to ``. + +`article` covers headings, **bold**/_italic_, a link, a nested bullet list, and +an ordered list. `formatting` covers strikethrough, inline code, a blockquote, +and a fenced code block. + +## Fidelity notes (structure, not layout — ADR 0009) + +The round-trip snapshots record genuine pinned-pandoc writer behaviour, not +bugs: + +- **ODT loosens tight lists** (blank lines appear between items) where DOCX keeps + them tight. +- **ODT flattens a fenced code block** to a plain paragraph; DOCX preserves it as + code. Extend the corpus rather than chasing these. + +## Regenerating + +Generated with the **pinned** `pandoc/core:3.6` (the production sidecar) so the +snapshots match CI: + +```sh +docker run --rm -p 3030:3030 pandoc/core:3.6 server +PANDOC_URL=http://localhost:3030 node scripts/gen-export-fixtures.mjs +``` diff --git a/fixtures/export/article.docx.expected.md b/fixtures/export/article.docx.expected.md new file mode 100644 index 0000000..76f5f94 --- /dev/null +++ b/fixtures/export/article.docx.expected.md @@ -0,0 +1,12 @@ +# Field Notes + +A short **survey** of the *village pond*, with a [reference](https://example.org/ponds). + +## Observations + +- Frogs at the north edge + - Two clutches of spawn +- Reeds spreading west + +1. Measure depth +2. Log temperature diff --git a/fixtures/export/article.md b/fixtures/export/article.md new file mode 100644 index 0000000..5f54e5c --- /dev/null +++ b/fixtures/export/article.md @@ -0,0 +1,12 @@ +# Field Notes + +A short **survey** of the _village pond_, with a [reference](https://example.org/ponds). + +## Observations + +- Frogs at the north edge + - Two clutches of spawn +- Reeds spreading west + +1. Measure depth +2. Log temperature diff --git a/fixtures/export/article.odt.expected.md b/fixtures/export/article.odt.expected.md new file mode 100644 index 0000000..d74def4 --- /dev/null +++ b/fixtures/export/article.odt.expected.md @@ -0,0 +1,14 @@ +# Field Notes + +A short **survey** of the *village pond*, with a [reference](https://example.org/ponds). + +## Observations + +- Frogs at the north edge + + - Two clutches of spawn + +- Reeds spreading west + +1. Measure depth +2. Log temperature diff --git a/fixtures/export/formatting.docx.expected.md b/fixtures/export/formatting.docx.expected.md new file mode 100644 index 0000000..6c48bfa --- /dev/null +++ b/fixtures/export/formatting.docx.expected.md @@ -0,0 +1,7 @@ +# Formatting + +Some ~~struck~~ text and `inline code`. + +> A quoted remark about the water level. + +`sample = measure(pond)` diff --git a/fixtures/export/formatting.md b/fixtures/export/formatting.md new file mode 100644 index 0000000..91692d2 --- /dev/null +++ b/fixtures/export/formatting.md @@ -0,0 +1,9 @@ +# Formatting + +Some ~~struck~~ text and `inline code`. + +> A quoted remark about the water level. + +``` +sample = measure(pond) +``` diff --git a/fixtures/export/formatting.odt.expected.md b/fixtures/export/formatting.odt.expected.md new file mode 100644 index 0000000..50af24b --- /dev/null +++ b/fixtures/export/formatting.odt.expected.md @@ -0,0 +1,7 @@ +# Formatting + +Some ~~struck~~ text and `inline code`. + +> A quoted remark about the water level. + +sample = measure(pond) diff --git a/fixtures/import/README.md b/fixtures/import/README.md index 99b3e18..f1fb851 100644 --- a/fixtures/import/README.md +++ b/fixtures/import/README.md @@ -4,7 +4,9 @@ Representative `.docx`/`.odt` documents and the Markdown our import pipeline is expected to produce from them. `import.fixtures.test.ts` runs the real two-pass pandoc conversion over each and asserts the result, so a change in behaviour (ours or pandoc's) surfaces as a snapshot diff to review — not a silent -regression. +regression. This suite is part of the CI **fidelity gate** (issue #69); see +[`../README.md`](../README.md) for the fidelity contract and the fixture-first +bug process (how a fidelity bug becomes a new corpus fixture). ## Files diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 55d78c0..032569e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -144,6 +144,9 @@ importers: fflate: specifier: ^0.8.3 version: 0.8.3 + pdf-parse: + specifier: ^2.4.5 + version: 2.4.5 pino-pretty: specifier: ^13.0.0 version: 13.1.3 @@ -1663,6 +1666,75 @@ packages: resolution: {integrity: sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA==} engines: {node: '>=8'} + '@napi-rs/canvas-android-arm64@0.1.80': + resolution: {integrity: sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/canvas-darwin-arm64@0.1.80': + resolution: {integrity: sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/canvas-darwin-x64@0.1.80': + resolution: {integrity: sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.80': + resolution: {integrity: sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/canvas-linux-arm64-gnu@0.1.80': + resolution: {integrity: sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@napi-rs/canvas-linux-arm64-musl@0.1.80': + resolution: {integrity: sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@napi-rs/canvas-linux-riscv64-gnu@0.1.80': + resolution: {integrity: sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@napi-rs/canvas-linux-x64-gnu@0.1.80': + resolution: {integrity: sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@napi-rs/canvas-linux-x64-musl@0.1.80': + resolution: {integrity: sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@napi-rs/canvas-win32-x64-msvc@0.1.80': + resolution: {integrity: sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/canvas@0.1.80': + resolution: {integrity: sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==} + engines: {node: '>= 10'} + '@nestjs/cli@11.0.23': resolution: {integrity: sha512-2V0Bf5jz0KXhUZk3eJi9GljIyqH04otwsE/mYLbqJR+X0iiYx+6bkNJ2Qz28uHNFj1cpHgimf9xDzHkqarie0g==} engines: {node: '>= 20.11'} @@ -4111,6 +4183,15 @@ packages: resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} engines: {node: '>= 14.16'} + pdf-parse@2.4.5: + resolution: {integrity: sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==} + engines: {node: '>=20.16.0 <21 || >=22.3.0'} + hasBin: true + + pdfjs-dist@5.4.296: + resolution: {integrity: sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==} + engines: {node: '>=20.16.0 || >=22.3.0'} + perfect-debounce@1.0.0: resolution: {integrity: sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==} @@ -6726,6 +6807,49 @@ snapshots: '@lukeed/csprng@1.1.0': {} + '@napi-rs/canvas-android-arm64@0.1.80': + optional: true + + '@napi-rs/canvas-darwin-arm64@0.1.80': + optional: true + + '@napi-rs/canvas-darwin-x64@0.1.80': + optional: true + + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.80': + optional: true + + '@napi-rs/canvas-linux-arm64-gnu@0.1.80': + optional: true + + '@napi-rs/canvas-linux-arm64-musl@0.1.80': + optional: true + + '@napi-rs/canvas-linux-riscv64-gnu@0.1.80': + optional: true + + '@napi-rs/canvas-linux-x64-gnu@0.1.80': + optional: true + + '@napi-rs/canvas-linux-x64-musl@0.1.80': + optional: true + + '@napi-rs/canvas-win32-x64-msvc@0.1.80': + optional: true + + '@napi-rs/canvas@0.1.80': + optionalDependencies: + '@napi-rs/canvas-android-arm64': 0.1.80 + '@napi-rs/canvas-darwin-arm64': 0.1.80 + '@napi-rs/canvas-darwin-x64': 0.1.80 + '@napi-rs/canvas-linux-arm-gnueabihf': 0.1.80 + '@napi-rs/canvas-linux-arm64-gnu': 0.1.80 + '@napi-rs/canvas-linux-arm64-musl': 0.1.80 + '@napi-rs/canvas-linux-riscv64-gnu': 0.1.80 + '@napi-rs/canvas-linux-x64-gnu': 0.1.80 + '@napi-rs/canvas-linux-x64-musl': 0.1.80 + '@napi-rs/canvas-win32-x64-msvc': 0.1.80 + '@nestjs/cli@11.0.23(@swc/core@1.15.43)(@types/node@26.1.0)(prettier@3.9.4)': dependencies: '@angular-devkit/core': 19.2.27(chokidar@4.0.3) @@ -9334,6 +9458,15 @@ snapshots: pathval@2.0.1: {} + pdf-parse@2.4.5: + dependencies: + '@napi-rs/canvas': 0.1.80 + pdfjs-dist: 5.4.296 + + pdfjs-dist@5.4.296: + optionalDependencies: + '@napi-rs/canvas': 0.1.80 + perfect-debounce@1.0.0: {} pg-cloudflare@1.4.0: diff --git a/scripts/gen-export-fixtures.mjs b/scripts/gen-export-fixtures.mjs new file mode 100644 index 0000000..bfe67fd --- /dev/null +++ b/scripts/gen-export-fixtures.mjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node +// Regenerate the export fidelity corpus (issue #69, ADR 0009) from the committed +// `*.md` sources: for each source and each office format, write the Markdown you +// get back after a full export → re-read round trip through the pinned pandoc. +// That round-trip snapshot is what the fidelity gate pins — a change in pandoc's +// docx/odt writer (e.g. a version bump) shifts the output and fails the suite. +// +// docker run --rm -p 3030:3030 pandoc/core:3.6 server +// PANDOC_URL=http://localhost:3030 node scripts/gen-export-fixtures.mjs +// +// Mirrors the export path (ExportService: gfm → , standalone) and then +// reads the document back ( → gfm) exactly as export.fidelity.test.ts does. + +import { readFileSync, writeFileSync, readdirSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const FIXTURES = join(dirname(fileURLToPath(import.meta.url)), '..', 'fixtures', 'export'); +const PANDOC_URL = process.env.PANDOC_URL ?? 'http://localhost:3030'; +// Match the import corpus reader so both directions normalise the same way. +const MARKDOWN_FORMAT = 'gfm-implicit_figures-raw_html'; +const BINARY = new Set(['docx', 'odt']); + +async function pandoc(params) { + const res = await fetch(`${PANDOC_URL}/`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(params), + }); + if (!res.ok) throw new Error(`pandoc ${res.status}: ${await res.text()}`); + return BINARY.has(params.to) ? Buffer.from(await res.arrayBuffer()) : res.text(); +} + +/** Export markdown to an office document, then read it back to markdown. */ +async function roundTrip(markdown, ext) { + const document = await pandoc({ text: markdown, from: 'gfm', to: ext, standalone: true }); + return pandoc({ + text: document.toString('base64'), + from: ext, + to: MARKDOWN_FORMAT, + standalone: false, + wrap: 'none', + }); +} + +const sources = readdirSync(FIXTURES).filter((f) => f.endsWith('.md') && !f.includes('.expected.')); +for (const source of sources) { + const name = source.replace(/\.md$/, ''); + const markdown = readFileSync(join(FIXTURES, source), 'utf8'); + for (const ext of ['docx', 'odt']) { + writeFileSync(join(FIXTURES, `${name}.${ext}.expected.md`), await roundTrip(markdown, ext)); + console.log(`wrote ${name}.${ext}.expected.md`); + } +}