#210: mark the Markdown ZIP export with frontmatter, imprint and manifest
Some checks failed
CI / Lint, typecheck, test (pull_request) Successful in 6m1s
CI / Build container images (pull_request) Successful in 2m58s
CI / Auth e2e pack (pull_request) Successful in 9m6s
CI / Import/export fidelity gate (pull_request) Successful in 1m8s
CI / Import/export fidelity gate (push) Blocked by required conditions
CD / Build and push images (push) Successful in 24s
CI / Lint, typecheck, test (push) Successful in 6m37s
CD / Deploy to Test (push) Successful in 12s
CI / Build container images (push) Has been skipped
CD / Smoke tests against Test (push) Successful in 1m25s
CD / Promote to Int (push) Successful in 13s
CI / Auth e2e pack (push) Has been cancelled
Some checks failed
CI / Lint, typecheck, test (pull_request) Successful in 6m1s
CI / Build container images (pull_request) Successful in 2m58s
CI / Auth e2e pack (pull_request) Successful in 9m6s
CI / Import/export fidelity gate (pull_request) Successful in 1m8s
CI / Import/export fidelity gate (push) Blocked by required conditions
CD / Build and push images (push) Successful in 24s
CI / Lint, typecheck, test (push) Successful in 6m37s
CD / Deploy to Test (push) Successful in 12s
CI / Build container images (push) Has been skipped
CD / Smoke tests against Test (push) Successful in 1m25s
CD / Promote to Int (push) Successful in 13s
CI / Auth e2e pack (push) Has been cancelled
A classified page's .md carries the level in YAML frontmatter AND the marking line at top and bottom; unclassified files are byte-identical to before. Every pond archive (incl. the per-pond folders of the account data export) ships a manifest.json listing each file with its level and stating the highest level once at archive level — media inherits the highest classification among the readable pages referencing it (fail-closed). Round trip: the importer recognizes exactly our frontmatter block, strips it plus the imprint lines, and creates the page at least at the imported level (content must not escape its marking by traveling through a ZIP) — pinned by unit and e2e round-trip tests. Foreign frontmatter passes through unchanged; the Obsidian vault import keeps its own frontmatter modes. Co-Authored-By: Claude Fable 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
74a9e495e4
commit
68497046e9
39
apps/api/src/import-export/classified-markdown.test.ts
Normal file
39
apps/api/src/import-export/classified-markdown.test.ts
Normal file
@ -0,0 +1,39 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { markClassifiedMarkdown, parseClassifiedMarkdown } from './classified-markdown';
|
||||
|
||||
const MARKING = 'VS – NUR FÜR DEN DIENSTGEBRAUCH';
|
||||
|
||||
describe('classified markdown marking (issue #210)', () => {
|
||||
it('wraps a classified page in frontmatter and top+bottom imprint', () => {
|
||||
const marked = markClassifiedMarkdown('# Title\n\nBody.\n', 'vs_nfd');
|
||||
expect(marked).toBe(
|
||||
`---\nclassification: vs_nfd\n---\n\n${MARKING}\n\n# Title\n\nBody.\n\n${MARKING}\n`,
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves unclassified markdown untouched', () => {
|
||||
expect(markClassifiedMarkdown('# Title\n\nBody.\n', 'unclassified')).toBe('# Title\n\nBody.\n');
|
||||
});
|
||||
|
||||
it('parse is the inverse of mark', () => {
|
||||
const original = '# Title\n\nBody.\n';
|
||||
const { markdown, classification } = parseClassifiedMarkdown(
|
||||
markClassifiedMarkdown(original, 'vs_nfd'),
|
||||
);
|
||||
expect(classification).toBe('vs_nfd');
|
||||
expect(markdown).toBe(original);
|
||||
});
|
||||
|
||||
it('passes documents without our frontmatter through unchanged', () => {
|
||||
for (const raw of [
|
||||
'# Plain\n\nNo frontmatter.\n',
|
||||
'---\ntitle: Foreign frontmatter\ntags: [a]\n---\n\n# Doc\n',
|
||||
`${MARKING}\n\nJust an imprint line without frontmatter.\n`,
|
||||
]) {
|
||||
const { markdown, classification } = parseClassifiedMarkdown(raw);
|
||||
expect(classification).toBeNull();
|
||||
expect(markdown).toBe(raw);
|
||||
}
|
||||
});
|
||||
});
|
||||
43
apps/api/src/import-export/classified-markdown.ts
Normal file
43
apps/api/src/import-export/classified-markdown.ts
Normal file
@ -0,0 +1,43 @@
|
||||
import { PageClassification, classificationMarking } from '@dorfteich/shared';
|
||||
|
||||
/**
|
||||
* VS-NfD marking of exported Markdown (issue #210, ADR 0022): a classified
|
||||
* page's `.md` carries the level machine-readably in YAML frontmatter AND
|
||||
* human-visibly as the marking line at the top and bottom of the file.
|
||||
* Unclassified pages pass through untouched — no marking, no frontmatter.
|
||||
*/
|
||||
export function markClassifiedMarkdown(
|
||||
markdown: string,
|
||||
classification: PageClassification,
|
||||
): string {
|
||||
const marking = classificationMarking(classification);
|
||||
if (!marking) return markdown;
|
||||
return `---\nclassification: ${classification}\n---\n\n${marking}\n\n${markdown.trimEnd()}\n\n${marking}\n`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inverse of {@link markClassifiedMarkdown} for the import side: recognizes
|
||||
* exactly the frontmatter block we generate (a lone `classification:` key)
|
||||
* and the marking lines around the body, so a round-trip re-import yields
|
||||
* the original content — and the page starts at the imported level (content
|
||||
* must not escape its marking by traveling through a ZIP). Anything else —
|
||||
* foreign frontmatter, hand-written documents — passes through unchanged.
|
||||
*/
|
||||
export function parseClassifiedMarkdown(raw: string): {
|
||||
markdown: string;
|
||||
classification: PageClassification | null;
|
||||
} {
|
||||
const match = raw.match(/^---\nclassification: (vs_nfd|unclassified)\n---\n\n/);
|
||||
if (!match) return { markdown: raw, classification: null };
|
||||
const classification = match[1] as PageClassification;
|
||||
let body = raw.slice(match[0].length);
|
||||
const marking = classificationMarking(classification);
|
||||
if (marking) {
|
||||
if (body.startsWith(`${marking}\n\n`)) body = body.slice(marking.length + 2);
|
||||
const trimmed = body.trimEnd();
|
||||
if (trimmed.endsWith(`\n\n${marking}`)) {
|
||||
body = `${trimmed.slice(0, -(marking.length + 2)).trimEnd()}\n`;
|
||||
}
|
||||
}
|
||||
return { markdown: body, classification };
|
||||
}
|
||||
@ -193,6 +193,76 @@ describe.skipIf(!hasTestDb)('export (e2e, issue #65)', () => {
|
||||
expect(target).toContain(``);
|
||||
});
|
||||
|
||||
it('marks classified pages in the pond ZIP with frontmatter+imprint, ships a manifest, and round-trips (#210)', async () => {
|
||||
const marking = 'VS – NUR FÜR DEN DIENSTGEBRAUCH';
|
||||
const classifiedSlug = await seedPage(
|
||||
personalPondId,
|
||||
'Zip Classified',
|
||||
'# Zip Classified\n\nclassified body text',
|
||||
);
|
||||
const openSlug = await seedPage(personalPondId, 'Zip Open', '# Zip Open\n\nopen body text');
|
||||
await prisma.page.updateMany({
|
||||
where: { pondId: personalPondId, slug: classifiedSlug },
|
||||
data: { classification: 'VS_NFD' },
|
||||
});
|
||||
|
||||
const res = await api()
|
||||
.get(`/api/v1/ponds/${personalPondId}/export/markdown`)
|
||||
.set('Cookie', ownerCookie)
|
||||
.buffer(true)
|
||||
.parse((r, cb) => {
|
||||
const chunks: Buffer[] = [];
|
||||
r.on('data', (c: Buffer) => chunks.push(c));
|
||||
r.on('end', () => cb(null, Buffer.concat(chunks)));
|
||||
})
|
||||
.expect(200);
|
||||
const entries = zipEntries(res.body as Buffer);
|
||||
|
||||
// Machine-readable frontmatter AND the visible imprint, top and bottom.
|
||||
const marked = Buffer.from(entries[`${classifiedSlug}.md`]!).toString('utf8');
|
||||
expect(marked.startsWith(`---\nclassification: vs_nfd\n---\n\n${marking}\n\n`)).toBe(true);
|
||||
expect(marked.trimEnd().endsWith(marking)).toBe(true);
|
||||
// Unclassified files are unchanged: no frontmatter, no imprint.
|
||||
const open = Buffer.from(entries[`${openSlug}.md`]!).toString('utf8');
|
||||
expect(open).not.toContain('classification:');
|
||||
expect(open).not.toContain(marking);
|
||||
|
||||
// The manifest lists every file with its level and states the highest once.
|
||||
const manifest = JSON.parse(Buffer.from(entries['manifest.json']!).toString('utf8')) as {
|
||||
classification: string;
|
||||
files: { path: string; classification: string }[];
|
||||
};
|
||||
expect(manifest.classification).toBe('vs_nfd');
|
||||
expect(manifest.files).toContainEqual({
|
||||
path: `${classifiedSlug}.md`,
|
||||
classification: 'vs_nfd',
|
||||
});
|
||||
expect(manifest.files).toContainEqual({
|
||||
path: `${openSlug}.md`,
|
||||
classification: 'unclassified',
|
||||
});
|
||||
|
||||
// Round-trip: re-importing the marked file must not confuse the importer —
|
||||
// the page starts at the imported level, the body carries neither the
|
||||
// frontmatter nor the imprint lines.
|
||||
const imported = await api()
|
||||
.post(`/api/v1/ponds/${personalPondId}/import`)
|
||||
.set('Cookie', ownerCookie)
|
||||
.attach('file', Buffer.from(marked, 'utf8'), 'reimported-classified.md')
|
||||
.expect(201);
|
||||
expect(imported.body.status).toBe('succeeded');
|
||||
const reimported = await prisma.page.findUniqueOrThrow({
|
||||
where: { id: imported.body.resultPageId as string },
|
||||
});
|
||||
expect(reimported.classification).toBe('VS_NFD');
|
||||
const cache = await prisma.pageContentCache.findUniqueOrThrow({
|
||||
where: { pageId: reimported.id },
|
||||
});
|
||||
expect(cache.markdown).toContain('classified body text');
|
||||
expect(cache.markdown).not.toContain(marking);
|
||||
expect(cache.markdown).not.toContain('classification:');
|
||||
});
|
||||
|
||||
it('skips an attachment whose bytes are missing on disk instead of crashing', async () => {
|
||||
// An attachment row with no file (data drift): upload then remove the bytes.
|
||||
const image = await files.upload({ id: ownerId } as never, personalPondId, {
|
||||
|
||||
@ -9,6 +9,8 @@ import {
|
||||
fontSlug,
|
||||
PageClassification,
|
||||
classificationMarking,
|
||||
classificationRank,
|
||||
highestClassification,
|
||||
pondSettingsSchema,
|
||||
} from '@dorfteich/shared';
|
||||
import { User } from '@prisma/client';
|
||||
@ -23,6 +25,7 @@ import { PluginFallbackRenderer } from '../plugins/plugin-fallback-renderer';
|
||||
import { PluginsService } from '../plugins/plugins.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
|
||||
import { markClassifiedMarkdown } from './classified-markdown';
|
||||
import { ConversionJobService } from './conversion-job.service';
|
||||
import {
|
||||
imageExtension,
|
||||
@ -133,15 +136,54 @@ export class ExportService {
|
||||
attachments.map((a) => [a.id, `${a.id}.${imageExtension(a.mimeType)}`]),
|
||||
);
|
||||
|
||||
// Media inherits the highest classification among the readable pages that
|
||||
// reference it (fail-closed, ADR 0022 — a shared image is as classified
|
||||
// as its most classified use).
|
||||
const mediaClassification = new Map<string, PageClassification>();
|
||||
for (const page of readablePages) {
|
||||
const markdown = markdownForZip(
|
||||
page.contentCache?.markdown ?? '',
|
||||
readableSlugs,
|
||||
mediaNameById,
|
||||
const level = page.classification.toLowerCase() as PageClassification;
|
||||
for (const id of imageFileIds(page.contentCache?.markdown ?? '')) {
|
||||
const current = mediaClassification.get(id) ?? 'unclassified';
|
||||
if (classificationRank(level) > classificationRank(current)) {
|
||||
mediaClassification.set(id, level);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const manifestFiles: { path: string; classification: PageClassification }[] = [];
|
||||
for (const page of readablePages) {
|
||||
const level = page.classification.toLowerCase() as PageClassification;
|
||||
// A classified page's file carries the level in YAML frontmatter and
|
||||
// the marking line at top and bottom (#210); unclassified files are
|
||||
// byte-identical to the pre-#210 export.
|
||||
const markdown = markClassifiedMarkdown(
|
||||
markdownForZip(page.contentCache?.markdown ?? '', readableSlugs, mediaNameById),
|
||||
level,
|
||||
);
|
||||
// Page slugs are unique within a pond, so `<slug>.md` never collides.
|
||||
archive.append(markdown, { name: `${prefix}${page.slug}.md` });
|
||||
manifestFiles.push({ path: `${prefix}${page.slug}.md`, classification: level });
|
||||
}
|
||||
for (const attachment of attachments) {
|
||||
manifestFiles.push({
|
||||
path: `${prefix}media/${mediaNameById.get(attachment.id)!}`,
|
||||
classification: mediaClassification.get(attachment.id) ?? 'unclassified',
|
||||
});
|
||||
}
|
||||
// The archive-level manifest (#210): every file with its level, and the
|
||||
// highest level contained stated once — the bulk-egress channel stays
|
||||
// machine-checkable even after the ZIP is unpacked and copied onward.
|
||||
archive.append(
|
||||
JSON.stringify(
|
||||
{
|
||||
classification: highestClassification(manifestFiles.map((f) => f.classification)),
|
||||
files: manifestFiles,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
{ name: `${prefix}manifest.json` },
|
||||
);
|
||||
for (const attachment of attachments) {
|
||||
const stream = this.storage.createReadStream(pond.id, attachment.id);
|
||||
// Defence in depth: a file removed between the existence check and the
|
||||
|
||||
@ -24,6 +24,7 @@ import { PagesService } from '../pages/pages.service';
|
||||
import { docToState, emptyPageState } from '../pages/yjs-content';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
|
||||
import { parseClassifiedMarkdown } from './classified-markdown';
|
||||
import { ImportProcessor } from './import.constants';
|
||||
import {
|
||||
ASSET_PLACEHOLDER_PREFIX,
|
||||
@ -544,12 +545,24 @@ export class ImportService implements ImportProcessor {
|
||||
// file ids); track what we create so a later failure can be rolled back.
|
||||
const storedFileIds: string[] = [];
|
||||
try {
|
||||
const markdown = await this.storeEmbeddedImages(rawMarkdown, user, pondId, storedFileIds);
|
||||
// Our own classified export wraps the content in frontmatter + marking
|
||||
// lines (#210) — strip them and carry the level into the new page, so
|
||||
// a round-trip neither duplicates the marking nor loses it.
|
||||
const { markdown: unwrapped, classification } = parseClassifiedMarkdown(rawMarkdown);
|
||||
const markdown = await this.storeEmbeddedImages(unwrapped, user, pondId, storedFileIds);
|
||||
const json = markdownToDoc(markdown).toJSON() as unknown as PmNode;
|
||||
const { title, doc } = this.splitTitle(json, sourceName);
|
||||
const state = docToState(Node.fromJSON(editorSchema, doc));
|
||||
|
||||
const page = await this.pages.createWithState(user, pondId, title, state);
|
||||
const page = await this.pages.createWithState(
|
||||
user,
|
||||
pondId,
|
||||
title,
|
||||
state,
|
||||
null,
|
||||
undefined,
|
||||
classification,
|
||||
);
|
||||
await this.files.linkAttachmentsToPage(storedFileIds, page.id);
|
||||
return page;
|
||||
} catch (error) {
|
||||
|
||||
@ -274,8 +274,9 @@ export class PagesService {
|
||||
state: Uint8Array<ArrayBuffer>,
|
||||
parentId: string | null = null,
|
||||
presetSlug?: string,
|
||||
atLeastClassification: PageClassification | null = null,
|
||||
): Promise<Page> {
|
||||
return this.insertPage(user, pondId, title, state, parentId, presetSlug);
|
||||
return this.insertPage(user, pondId, title, state, parentId, presetSlug, atLeastClassification);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -300,6 +301,7 @@ export class PagesService {
|
||||
state: Uint8Array<ArrayBuffer>,
|
||||
parentId: string | null = null,
|
||||
presetSlug?: string,
|
||||
atLeastClassification: PageClassification | null = null,
|
||||
): Promise<Page> {
|
||||
const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } });
|
||||
if (!pond) throw new NotFoundException();
|
||||
@ -316,11 +318,19 @@ export class PagesService {
|
||||
const sortKey = generateKeyBetween(last?.sortKey ?? null, null);
|
||||
const content = deriveContent(state);
|
||||
// A new page starts at the instance-wide default level (ADR 0022, #204),
|
||||
// raised to its parent's level when that is higher (#205): a subpage of
|
||||
// classified content must never begin unmarked.
|
||||
// raised to its parent's level when that is higher (#205) — and to an
|
||||
// imported document's own level (#210): content must not escape its
|
||||
// marking by traveling through an export/import. A subpage of classified
|
||||
// content must never begin unmarked.
|
||||
let classification: PageClassification = await this.settings.get(
|
||||
'classification.newPageDefault',
|
||||
);
|
||||
if (
|
||||
atLeastClassification &&
|
||||
classificationRank(atLeastClassification) > classificationRank(classification)
|
||||
) {
|
||||
classification = atLeastClassification;
|
||||
}
|
||||
if (parentId) {
|
||||
const parent = await this.prisma.page.findUniqueOrThrow({
|
||||
where: { id: parentId },
|
||||
|
||||
@ -56,7 +56,7 @@ _Meilenstein: `M26 — VS-NfD: classification metadata`_
|
||||
- [x] **Print-CSS** (`@media print`, Kopf/Fuß je Seite) — fehlt komplett · 1 AT · #207
|
||||
- [x] PDF via gotenberg (`pdf-html.ts` Header/Footer-Template) · 1 AT · #208
|
||||
- [x] DOCX/ODT via pandoc (Reference-Doc mit Kopf-/Fußzeile) · 2–3 AT · #209
|
||||
- [ ] Markdown-ZIP (Frontmatter + Aufdruck) · 1 AT · #210
|
||||
- [x] Markdown-ZIP (Frontmatter + Aufdruck) · 1 AT · #210
|
||||
- [ ] Atom-Feeds, Public-API, Suchergebnisse, No-JS-Shell · 2–3 AT · #211
|
||||
- [ ] Attachment-Download (Dateiname-Präfix + Begleitdatei) · 1–2 AT · #212
|
||||
- [ ] Warnung/Sperre beim Anhängen an eingestufte Seiten · 1 AT · #213
|
||||
|
||||
Loading…
Reference in New Issue
Block a user