#210: Markdown ZIP with frontmatter, imprint and manifest #269

Merged
fable-5 merged 2 commits from issue-210-zip-marking into main 2026-07-31 07:36:22 +02:00
7 changed files with 227 additions and 10 deletions
Showing only changes of commit 68497046e9 - Show all commits

View File

@ -0,0 +1,39 @@
import { describe, expect, it } from 'vitest';
import { markClassifiedMarkdown, parseClassifiedMarkdown } from './classified-markdown';
const MARKING = 'VS NUR FÜR DEN DIENSTGEBRAUCH';
describe('classified markdown marking (issue #210)', () => {
it('wraps a classified page in frontmatter and top+bottom imprint', () => {
const marked = markClassifiedMarkdown('# Title\n\nBody.\n', 'vs_nfd');
expect(marked).toBe(
`---\nclassification: vs_nfd\n---\n\n${MARKING}\n\n# Title\n\nBody.\n\n${MARKING}\n`,
);
});
it('leaves unclassified markdown untouched', () => {
expect(markClassifiedMarkdown('# Title\n\nBody.\n', 'unclassified')).toBe('# Title\n\nBody.\n');
});
it('parse is the inverse of mark', () => {
const original = '# Title\n\nBody.\n';
const { markdown, classification } = parseClassifiedMarkdown(
markClassifiedMarkdown(original, 'vs_nfd'),
);
expect(classification).toBe('vs_nfd');
expect(markdown).toBe(original);
});
it('passes documents without our frontmatter through unchanged', () => {
for (const raw of [
'# Plain\n\nNo frontmatter.\n',
'---\ntitle: Foreign frontmatter\ntags: [a]\n---\n\n# Doc\n',
`${MARKING}\n\nJust an imprint line without frontmatter.\n`,
]) {
const { markdown, classification } = parseClassifiedMarkdown(raw);
expect(classification).toBeNull();
expect(markdown).toBe(raw);
}
});
});

View File

@ -0,0 +1,43 @@
import { PageClassification, classificationMarking } from '@dorfteich/shared';
/**
* VS-NfD marking of exported Markdown (issue #210, ADR 0022): a classified
* page's `.md` carries the level machine-readably in YAML frontmatter AND
* human-visibly as the marking line at the top and bottom of the file.
* Unclassified pages pass through untouched no marking, no frontmatter.
*/
export function markClassifiedMarkdown(
markdown: string,
classification: PageClassification,
): string {
const marking = classificationMarking(classification);
if (!marking) return markdown;
return `---\nclassification: ${classification}\n---\n\n${marking}\n\n${markdown.trimEnd()}\n\n${marking}\n`;
}
/**
* Inverse of {@link markClassifiedMarkdown} for the import side: recognizes
* exactly the frontmatter block we generate (a lone `classification:` key)
* and the marking lines around the body, so a round-trip re-import yields
* the original content and the page starts at the imported level (content
* must not escape its marking by traveling through a ZIP). Anything else
* foreign frontmatter, hand-written documents passes through unchanged.
*/
export function parseClassifiedMarkdown(raw: string): {
markdown: string;
classification: PageClassification | null;
} {
const match = raw.match(/^---\nclassification: (vs_nfd|unclassified)\n---\n\n/);
if (!match) return { markdown: raw, classification: null };
const classification = match[1] as PageClassification;
let body = raw.slice(match[0].length);
const marking = classificationMarking(classification);
if (marking) {
if (body.startsWith(`${marking}\n\n`)) body = body.slice(marking.length + 2);
const trimmed = body.trimEnd();
if (trimmed.endsWith(`\n\n${marking}`)) {
body = `${trimmed.slice(0, -(marking.length + 2)).trimEnd()}\n`;
}
}
return { markdown: body, classification };
}

View File

@ -193,6 +193,76 @@ describe.skipIf(!hasTestDb)('export (e2e, issue #65)', () => {
expect(target).toContain(`![dot](media/${image.id}.png)`);
});
it('marks classified pages in the pond ZIP with frontmatter+imprint, ships a manifest, and round-trips (#210)', async () => {
const marking = 'VS NUR FÜR DEN DIENSTGEBRAUCH';
const classifiedSlug = await seedPage(
personalPondId,
'Zip Classified',
'# Zip Classified\n\nclassified body text',
);
const openSlug = await seedPage(personalPondId, 'Zip Open', '# Zip Open\n\nopen body text');
await prisma.page.updateMany({
where: { pondId: personalPondId, slug: classifiedSlug },
data: { classification: 'VS_NFD' },
});
const res = await api()
.get(`/api/v1/ponds/${personalPondId}/export/markdown`)
.set('Cookie', ownerCookie)
.buffer(true)
.parse((r, cb) => {
const chunks: Buffer[] = [];
r.on('data', (c: Buffer) => chunks.push(c));
r.on('end', () => cb(null, Buffer.concat(chunks)));
})
.expect(200);
const entries = zipEntries(res.body as Buffer);
// Machine-readable frontmatter AND the visible imprint, top and bottom.
const marked = Buffer.from(entries[`${classifiedSlug}.md`]!).toString('utf8');
expect(marked.startsWith(`---\nclassification: vs_nfd\n---\n\n${marking}\n\n`)).toBe(true);
expect(marked.trimEnd().endsWith(marking)).toBe(true);
// Unclassified files are unchanged: no frontmatter, no imprint.
const open = Buffer.from(entries[`${openSlug}.md`]!).toString('utf8');
expect(open).not.toContain('classification:');
expect(open).not.toContain(marking);
// The manifest lists every file with its level and states the highest once.
const manifest = JSON.parse(Buffer.from(entries['manifest.json']!).toString('utf8')) as {
classification: string;
files: { path: string; classification: string }[];
};
expect(manifest.classification).toBe('vs_nfd');
expect(manifest.files).toContainEqual({
path: `${classifiedSlug}.md`,
classification: 'vs_nfd',
});
expect(manifest.files).toContainEqual({
path: `${openSlug}.md`,
classification: 'unclassified',
});
// Round-trip: re-importing the marked file must not confuse the importer —
// the page starts at the imported level, the body carries neither the
// frontmatter nor the imprint lines.
const imported = await api()
.post(`/api/v1/ponds/${personalPondId}/import`)
.set('Cookie', ownerCookie)
.attach('file', Buffer.from(marked, 'utf8'), 'reimported-classified.md')
.expect(201);
expect(imported.body.status).toBe('succeeded');
const reimported = await prisma.page.findUniqueOrThrow({
where: { id: imported.body.resultPageId as string },
});
expect(reimported.classification).toBe('VS_NFD');
const cache = await prisma.pageContentCache.findUniqueOrThrow({
where: { pageId: reimported.id },
});
expect(cache.markdown).toContain('classified body text');
expect(cache.markdown).not.toContain(marking);
expect(cache.markdown).not.toContain('classification:');
});
it('skips an attachment whose bytes are missing on disk instead of crashing', async () => {
// An attachment row with no file (data drift): upload then remove the bytes.
const image = await files.upload({ id: ownerId } as never, personalPondId, {

View File

@ -9,6 +9,8 @@ import {
fontSlug,
PageClassification,
classificationMarking,
classificationRank,
highestClassification,
pondSettingsSchema,
} from '@dorfteich/shared';
import { User } from '@prisma/client';
@ -23,6 +25,7 @@ import { PluginFallbackRenderer } from '../plugins/plugin-fallback-renderer';
import { PluginsService } from '../plugins/plugins.service';
import { PrismaService } from '../prisma/prisma.service';
import { markClassifiedMarkdown } from './classified-markdown';
import { ConversionJobService } from './conversion-job.service';
import {
imageExtension,
@ -133,15 +136,54 @@ export class ExportService {
attachments.map((a) => [a.id, `${a.id}.${imageExtension(a.mimeType)}`]),
);
// Media inherits the highest classification among the readable pages that
// reference it (fail-closed, ADR 0022 — a shared image is as classified
// as its most classified use).
const mediaClassification = new Map<string, PageClassification>();
for (const page of readablePages) {
const markdown = markdownForZip(
page.contentCache?.markdown ?? '',
readableSlugs,
mediaNameById,
const level = page.classification.toLowerCase() as PageClassification;
for (const id of imageFileIds(page.contentCache?.markdown ?? '')) {
const current = mediaClassification.get(id) ?? 'unclassified';
if (classificationRank(level) > classificationRank(current)) {
mediaClassification.set(id, level);
}
}
}
const manifestFiles: { path: string; classification: PageClassification }[] = [];
for (const page of readablePages) {
const level = page.classification.toLowerCase() as PageClassification;
// A classified page's file carries the level in YAML frontmatter and
// the marking line at top and bottom (#210); unclassified files are
// byte-identical to the pre-#210 export.
const markdown = markClassifiedMarkdown(
markdownForZip(page.contentCache?.markdown ?? '', readableSlugs, mediaNameById),
level,
);
// Page slugs are unique within a pond, so `<slug>.md` never collides.
archive.append(markdown, { name: `${prefix}${page.slug}.md` });
manifestFiles.push({ path: `${prefix}${page.slug}.md`, classification: level });
}
for (const attachment of attachments) {
manifestFiles.push({
path: `${prefix}media/${mediaNameById.get(attachment.id)!}`,
classification: mediaClassification.get(attachment.id) ?? 'unclassified',
});
}
// The archive-level manifest (#210): every file with its level, and the
// highest level contained stated once — the bulk-egress channel stays
// machine-checkable even after the ZIP is unpacked and copied onward.
archive.append(
JSON.stringify(
{
classification: highestClassification(manifestFiles.map((f) => f.classification)),
files: manifestFiles,
},
null,
2,
),
{ name: `${prefix}manifest.json` },
);
for (const attachment of attachments) {
const stream = this.storage.createReadStream(pond.id, attachment.id);
// Defence in depth: a file removed between the existence check and the

View File

@ -24,6 +24,7 @@ import { PagesService } from '../pages/pages.service';
import { docToState, emptyPageState } from '../pages/yjs-content';
import { PrismaService } from '../prisma/prisma.service';
import { parseClassifiedMarkdown } from './classified-markdown';
import { ImportProcessor } from './import.constants';
import {
ASSET_PLACEHOLDER_PREFIX,
@ -544,12 +545,24 @@ export class ImportService implements ImportProcessor {
// file ids); track what we create so a later failure can be rolled back.
const storedFileIds: string[] = [];
try {
const markdown = await this.storeEmbeddedImages(rawMarkdown, user, pondId, storedFileIds);
// Our own classified export wraps the content in frontmatter + marking
// lines (#210) — strip them and carry the level into the new page, so
// a round-trip neither duplicates the marking nor loses it.
const { markdown: unwrapped, classification } = parseClassifiedMarkdown(rawMarkdown);
const markdown = await this.storeEmbeddedImages(unwrapped, user, pondId, storedFileIds);
const json = markdownToDoc(markdown).toJSON() as unknown as PmNode;
const { title, doc } = this.splitTitle(json, sourceName);
const state = docToState(Node.fromJSON(editorSchema, doc));
const page = await this.pages.createWithState(user, pondId, title, state);
const page = await this.pages.createWithState(
user,
pondId,
title,
state,
null,
undefined,
classification,
);
await this.files.linkAttachmentsToPage(storedFileIds, page.id);
return page;
} catch (error) {

View File

@ -274,8 +274,9 @@ export class PagesService {
state: Uint8Array<ArrayBuffer>,
parentId: string | null = null,
presetSlug?: string,
atLeastClassification: PageClassification | null = null,
): Promise<Page> {
return this.insertPage(user, pondId, title, state, parentId, presetSlug);
return this.insertPage(user, pondId, title, state, parentId, presetSlug, atLeastClassification);
}
/**
@ -300,6 +301,7 @@ export class PagesService {
state: Uint8Array<ArrayBuffer>,
parentId: string | null = null,
presetSlug?: string,
atLeastClassification: PageClassification | null = null,
): Promise<Page> {
const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } });
if (!pond) throw new NotFoundException();
@ -316,11 +318,19 @@ export class PagesService {
const sortKey = generateKeyBetween(last?.sortKey ?? null, null);
const content = deriveContent(state);
// A new page starts at the instance-wide default level (ADR 0022, #204),
// raised to its parent's level when that is higher (#205): a subpage of
// classified content must never begin unmarked.
// raised to its parent's level when that is higher (#205) — and to an
// imported document's own level (#210): content must not escape its
// marking by traveling through an export/import. A subpage of classified
// content must never begin unmarked.
let classification: PageClassification = await this.settings.get(
'classification.newPageDefault',
);
if (
atLeastClassification &&
classificationRank(atLeastClassification) > classificationRank(classification)
) {
classification = atLeastClassification;
}
if (parentId) {
const parent = await this.prisma.page.findUniqueOrThrow({
where: { id: parentId },

View File

@ -56,7 +56,7 @@ _Meilenstein: `M26 — VS-NfD: classification metadata`_
- [x] **Print-CSS** (`@media print`, Kopf/Fuß je Seite) — fehlt komplett · 1 AT · #207
- [x] PDF via gotenberg (`pdf-html.ts` Header/Footer-Template) · 1 AT · #208
- [x] DOCX/ODT via pandoc (Reference-Doc mit Kopf-/Fußzeile) · 23 AT · #209
- [ ] Markdown-ZIP (Frontmatter + Aufdruck) · 1 AT · #210
- [x] Markdown-ZIP (Frontmatter + Aufdruck) · 1 AT · #210
- [ ] Atom-Feeds, Public-API, Suchergebnisse, No-JS-Shell · 23 AT · #211
- [ ] Attachment-Download (Dateiname-Präfix + Begleitdatei) · 12 AT · #212
- [ ] Warnung/Sperre beim Anhängen an eingestufte Seiten · 1 AT · #213