import { MAX_PAGE_DEPTH, slugify } from '@dorfteich/shared'; import { unzipSync } from 'fflate'; /** * Obsidian vault transform (issue #116): pure functions from "uploaded ZIP" * to an import plan — no DB, no DI, fully unit-testable. The orchestration * (issue #117) feeds the plan into the page/file/label services. * * The core impedance mismatch: Obsidian links reference note NAMES * (`[[Meine Notiz]]`, `[[folder/Note#Heading|shown]]`), Dorfteich wikilinks * resolve by SLUG within a pond — and the shared markdown parser reads * `[[target]]` verbatim. So every rewrite here is a text pre-pass over the * note Markdown, done before `markdownToDoc` ever sees it. */ export class VaultError extends Error { constructor( readonly code: 'import_vault_invalid_zip' | 'import_vault_too_large', message: string, ) { super(message); } } /** Zip-bomb ceiling for the unpacked vault (the ZIP itself is capped by the * upload limit). Matches the plugin-package model. */ export const MAX_VAULT_UNPACKED_BYTES = 256 * 1024 * 1024; /** Raster formats that become inline images. SVG deliberately stays an * attachment — it is never served inline (security.md §Uploads). */ const IMAGE_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp']); /** Placeholder scheme the orchestrator replaces with real file ids after * uploading: `![alt](vault-asset:)`. */ export const ASSET_PLACEHOLDER_PREFIX = 'vault-asset:'; export interface VaultNote { /** Vault-relative path without the `.md` extension. */ path: string; /** Basename without `.md` — what Obsidian links refer to. */ name: string; markdown: string; } export interface VaultAsset { path: string; name: string; data: Uint8Array; /** Lowercase extension without the dot. */ extension: string; } export interface Vault { notes: VaultNote[]; assets: VaultAsset[]; } export type FrontmatterMode = 'strip' | 'preserve'; export interface PlannedContainer { /** Unique key: the merged folder path. */ key: string; parentKey: string | null; /** Human title — a merged folder reads like `c/d`. */ title: string; slug: string; } export interface PlannedNote { key: string; parentKey: string | null; title: string; slug: string; /** Rewritten Markdown: links → final slugs, tags stripped, frontmatter * handled, asset embeds → `vault-asset:` placeholders. */ markdown: string; /** Tag paths (nested tags split at `/`), e.g. [['projekt'], ['a','b']]. */ tags: string[][]; /** Image assets referenced by this note (vault paths). */ imageAssets: string[]; /** Non-image assets referenced by this note (become page attachments). */ attachmentAssets: string[]; } export interface VaultImportPlan { /** Topologically ordered (parents before children). */ containers: PlannedContainer[]; notes: PlannedNote[]; /** Vault paths of every asset actually referenced by some note. */ referencedAssets: Set; } /** Rejects absolute paths and `..` traversal (zip-slip), plugin-package model. */ function safeRelativePath(rawPath: string): string { const path = rawPath.replace(/\\/g, '/').replace(/^\/+/, ''); if (path.split('/').some((segment) => segment === '..')) { throw new VaultError('import_vault_invalid_zip', `Illegal path in archive: ${rawPath}`); } return path; } /** * Repairs a ZIP entry name that fflate decoded as Latin-1 (#127). fflate * honors only the ZIP UTF-8 flag (bit 11), which common archivers omit — * the UTF-8 bytes of "Fußball" then arrive as one mojibake char per byte * and end up in page titles. Latin-1 decoding is byte-lossless (charCode == * byte), so when every char fits a byte we re-read them as UTF-8; names the * flag DID mark as UTF-8 contain either multi-byte chars (> 0xFF, left * alone) or precomposed Latin-1 chars whose bytes are invalid UTF-8 (the * decoder throws → keep the original). Finally NFC-normalize: macOS zips * store umlauts decomposed (NFD), which breaks slugify's ä→ae digraphs, * wikilink matching, and duplicate-basename detection. */ function decodeZipName(raw: string): string { let name = raw; // eslint-disable-next-line no-control-regex if (/^[\u0000-\u00ff]*$/.test(raw)) { const bytes = Uint8Array.from(raw, (char) => char.charCodeAt(0)); try { name = new TextDecoder('utf-8', { fatal: true }).decode(bytes); } catch { // Genuinely Latin-1 (or already plain ASCII) — keep as decoded. } } return name.normalize('NFC'); } /** * Unpacks the vault ZIP into notes (`.md`) and assets (everything else), * enforcing the unpacked-size ceiling incrementally. Hidden housekeeping * directories (`.obsidian/`, `.trash/`, any dot-directory) are skipped — * they hold app config, not content. */ export function parseVaultZip( zip: Uint8Array, maxUnpackedBytes: number = MAX_VAULT_UNPACKED_BYTES, ): Vault { let entries: Record; try { entries = unzipSync(zip); } catch { throw new VaultError('import_vault_invalid_zip', 'The file is not a valid ZIP archive'); } const notes: VaultNote[] = []; const assets: VaultAsset[] = []; let unpacked = 0; for (const [zipName, bytes] of Object.entries(entries)) { const rawPath = decodeZipName(zipName); if (rawPath.endsWith('/')) continue; // directory entries const path = safeRelativePath(rawPath); const segments = path.split('/'); if (segments.some((segment) => segment.startsWith('.'))) continue; unpacked += bytes.length; if (unpacked > maxUnpackedBytes) { throw new VaultError('import_vault_too_large', 'The unpacked vault is too large'); } const basename = segments[segments.length - 1]!; if (basename.toLowerCase().endsWith('.md')) { const noteName = basename.slice(0, -3); notes.push({ path: path.slice(0, -3), name: noteName, markdown: new TextDecoder().decode(bytes), }); } else { const dot = basename.lastIndexOf('.'); assets.push({ path, name: basename, data: bytes, extension: dot >= 0 ? basename.slice(dot + 1).toLowerCase() : '', }); } } if (notes.length === 0) { throw new VaultError('import_vault_invalid_zip', 'The archive contains no Markdown notes'); } // Deterministic plan regardless of zip entry order. notes.sort((a, b) => a.path.localeCompare(b.path)); assets.sort((a, b) => a.path.localeCompare(b.path)); return { notes, assets }; } export interface Frontmatter { /** The note body with the frontmatter handled per mode. */ markdown: string; /** Tag paths extracted from `tags:`/`tag:` (nested split at `/`). */ tags: string[][]; } function parseTagValue(value: string): string[][] { // `tags: a, b` · `tags: [a, b]` · quoted variants. const inner = value.trim().replace(/^\[/, '').replace(/\]$/, ''); return inner .split(',') .map((tag) => tag.trim().replace(/^["'#]+|["']+$/g, '')) .filter(Boolean) .map((tag) => tag.split('/').filter(Boolean)); } /** * Handles a leading YAML frontmatter block. Only the `tags:` forms Obsidian * uses are interpreted (scalar list, inline array, block list) — everything * else is opaque. `strip` drops the block; `preserve` re-emits it as a * fenced yaml code block so nothing is lost but nothing leaks as prose. */ export function extractFrontmatter(markdown: string, mode: FrontmatterMode): Frontmatter { const match = /^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/.exec(markdown); if (!match) return { markdown, tags: [] }; const block = match[1]!; const body = markdown.slice(match[0].length); const tags: string[][] = []; const lines = block.split(/\r?\n/); for (let i = 0; i < lines.length; i += 1) { const line = lines[i]!; const keyMatch = /^(tags?)[ \t]*:[ \t]*(.*)$/i.exec(line); if (!keyMatch) continue; const value = keyMatch[2]!.trim(); if (value) { tags.push(...parseTagValue(value)); continue; } // Block list form: consume the following `- item` lines. for (let j = i + 1; j < lines.length; j += 1) { const item = /^[ \t]+-[ \t]*(.+)$/.exec(lines[j]!); if (!item) break; tags.push(...parseTagValue(item[1]!)); } } const preserved = mode === 'preserve' ? `\`\`\`yaml\n${block}\n\`\`\`\n\n` : ''; return { markdown: preserved + body, tags }; } /** * Splits Markdown into alternating segments outside/inside code so the tag * and link passes never touch fenced blocks or inline code. Yields the * outside segments to `transform` and stitches everything back together. */ function transformOutsideCode(markdown: string, transform: (segment: string) => string): string { const lines = markdown.split('\n'); const out: string[] = []; let inFence = false; let fenceMarker = ''; for (const line of lines) { const fence = /^([ \t]*)(```+|~~~+)/.exec(line); if (fence && !inFence) { inFence = true; fenceMarker = fence[2]![0]!; out.push(line); continue; } if (fence && inFence && fence[2]![0] === fenceMarker) { inFence = false; out.push(line); continue; } if (inFence) { out.push(line); continue; } // Outside fences: protect inline code spans. const parts = line.split(/(`[^`]*`)/); out.push(parts.map((part) => (part.startsWith('`') ? part : transform(part))).join('')); } return out.join('\n'); } export interface InlineTags { markdown: string; tags: string[][]; } /** * Extracts inline `#tag` / `#nested/tag` occurrences and removes them from * the text (Stefan's decision: ALL Obsidian tags become labels). Headings are * safe — `#` followed by whitespace never matches; pure-number "tags" (like * anchors or issue refs) are ignored, matching Obsidian's own rule. */ export function extractInlineTags(markdown: string): InlineTags { const tags: string[][] = []; const result = transformOutsideCode(markdown, (segment) => segment.replace(/(^|[\s(])#([\p{L}\p{N}_/-]+)/gu, (whole, prefix: string, tag: string) => { if (!/\p{L}/u.test(tag)) return whole; // needs at least one letter tags.push(tag.split('/').filter(Boolean)); return prefix; }), ); // Collapse the whitespace runs left behind by removed tags. const cleaned = transformOutsideCode(result, (segment) => segment.replace(/[ \t]{2,}/g, ' ').replace(/[ \t]+$/g, ''), ); return { markdown: cleaned, tags }; } /** How a `[[target]]` resolves: to a note, an asset, or nothing. */ export interface LinkResolver { /** Final slug for a note target, or null when unresolvable. */ noteSlug(target: string): string | null; /** Vault path of an asset target, or null. */ assetPath(target: string): { path: string; extension: string } | null; } /** * Builds the vault-wide resolver: case-insensitive basename matching with * full-path precedence, duplicates resolved to the lexicographically first * vault path (documented rule; Obsidian's "closest" has no meaning without * a source-file anchor when the whole vault imports at once). */ export function buildResolver(vault: Vault, noteSlugByPath: Map): LinkResolver { const notesByName = new Map(); // lower name → path (first wins) const notesByPath = new Map(); // lower path → path for (const note of vault.notes) { const lowerName = note.name.toLowerCase(); if (!notesByName.has(lowerName)) notesByName.set(lowerName, note.path); notesByPath.set(note.path.toLowerCase(), note.path); } const assetsByName = new Map(); const assetsByPath = new Map(); for (const asset of vault.assets) { const lowerName = asset.name.toLowerCase(); if (!assetsByName.has(lowerName)) assetsByName.set(lowerName, asset); assetsByPath.set(asset.path.toLowerCase(), asset); } return { noteSlug(target: string): string | null { const lower = target.toLowerCase().replace(/\.md$/, ''); const path = lower.includes('/') ? (notesByPath.get(lower) ?? notesByName.get(lower.split('/').pop()!)) : notesByName.get(lower); return path ? (noteSlugByPath.get(path) ?? null) : null; }, assetPath(target: string): { path: string; extension: string } | null { const lower = target.toLowerCase(); const asset = lower.includes('/') ? (assetsByPath.get(lower) ?? assetsByName.get(lower.split('/').pop()!)) : assetsByName.get(lower); return asset ? { path: asset.path, extension: asset.extension } : null; }, }; } export interface RewrittenLinks { markdown: string; imageAssets: string[]; attachmentAssets: string[]; } /** * The link pass (fence-aware): rewrites every Obsidian link form to what the * shared markdown parser understands. * - `[[Name]]`, `[[Name|Display]]`, `[[Name#Heading]]`, `[[folder/Name]]` → * `[[final-slug|display or original name]]` (heading stripped). * - Unresolvable note targets → `[[slugify(target)|target]]` — a phantom * link, exactly like typing a dead wikilink by hand. * - `![[image.png]]` / `![](relative/path.png)` on a vault image → * `![name](vault-asset:path)` for the uploader to resolve. * - `![[doc.pdf]]` (non-image asset) → `*doc.pdf*` text; the file becomes a * page attachment (paperclip panel), which has no inline node. * - `![[Other Note]]` (note embed) → a plain wikilink (no transclusion). */ export function rewriteLinks(markdown: string, resolver: LinkResolver): RewrittenLinks { const imageAssets = new Set(); const attachmentAssets = new Set(); const rewriteTarget = (rawTarget: string, rawDisplay: string | undefined): string => { const target = rawTarget.split('#')[0]!.trim(); if (!target) return rawDisplay ?? rawTarget; // pure heading link — keep text const display = rawDisplay?.trim() || target.split('/').pop()!; const slug = resolver.noteSlug(target) ?? (slugify(target.split('/').pop()!) || 'page'); return `[[${slug}|${display}]]`; }; const result = transformOutsideCode(markdown, (segment) => { let out = segment; // Embeds first — `![[…]]` would otherwise be eaten as `!` + wikilink. out = out.replace(/!\[\[([^[\]\n]+)\]\]/g, (_whole, inner: string) => { const [target, display] = splitPipe(inner); const asset = resolver.assetPath(target.split('#')[0]!.trim()); if (asset) { if (IMAGE_EXTENSIONS.has(asset.extension)) { imageAssets.add(asset.path); const alt = display ?? asset.path.split('/').pop()!; return `![${alt}](${ASSET_PLACEHOLDER_PREFIX}${asset.path})`; } attachmentAssets.add(asset.path); return `*${display ?? asset.path.split('/').pop()!}*`; } // A note embed (transclusion) degrades to a normal link. return rewriteTarget(target, display); }); // Standard markdown images with a relative vault path. out = out.replace(/!\[([^\]]*)\]\(([^)\s]+)\)/g, (whole, alt: string, src: string) => { if (/^[a-z][a-z0-9+.-]*:/i.test(src)) return whole; // absolute URL/data: const asset = resolver.assetPath(decodeURIComponent(src)); if (!asset) return whole; if (IMAGE_EXTENSIONS.has(asset.extension)) { imageAssets.add(asset.path); return `![${alt}](${ASSET_PLACEHOLDER_PREFIX}${asset.path})`; } attachmentAssets.add(asset.path); return `*${alt || asset.path.split('/').pop()!}*`; }); // Plain wikilinks. out = out.replace(/\[\[([^[\]\n]+)\]\]/g, (_whole, inner: string) => { const [target, display] = splitPipe(inner); return rewriteTarget(target, display); }); return out; }); return { markdown: result, imageAssets: [...imageAssets], attachmentAssets: [...attachmentAssets], }; } function splitPipe(inner: string): [string, string | undefined] { const pipe = inner.indexOf('|'); if (pipe < 0) return [inner, undefined]; return [inner.slice(0, pipe), inner.slice(pipe + 1)]; } /** * Maps every note's folder path to a container chain of at most * `maxContainerLevels` (the deepest levels merge into one container whose * title keeps the joined path, e.g. `c/d`). Returns the ordered container * list (parents first) and the container key each note hangs under. */ export function planFolders( vault: Vault, maxContainerLevels: number, ): { containers: { key: string; parentKey: string | null; title: string }[]; parentKeyByNote: Map; } { const containers = new Map(); const parentKeyByNote = new Map(); for (const note of vault.notes) { const folders = note.path.split('/').slice(0, -1); if (folders.length === 0 || maxContainerLevels <= 0) { parentKeyByNote.set(note.path, null); continue; } const kept = folders.slice(0, Math.max(0, maxContainerLevels - 1)); const merged = folders.slice(Math.max(0, maxContainerLevels - 1)); const chain = merged.length > 0 ? [...kept, merged.join('/')] : kept; let parentKey: string | null = null; let keyPrefix = ''; for (const title of chain) { keyPrefix = keyPrefix ? `${keyPrefix}/${title}` : title; if (!containers.has(keyPrefix)) { containers.set(keyPrefix, { key: keyPrefix, parentKey, title }); } parentKey = keyPrefix; } parentKeyByNote.set(note.path, parentKey); } // Map insertion already guarantees parents-before-children (each chain is // walked top-down), and iteration order is insertion order. return { containers: [...containers.values()], parentKeyByNote }; } /** * Assigns final slugs to every container and note: `-2/-3` suffixing against * the existing pond slugs and everything planned so far. Deterministic: * containers (path order) before notes (path order). */ export function planSlugs( entries: { key: string; base: string }[], existingSlugs: ReadonlySet, ): Map { const taken = new Set(existingSlugs); const result = new Map(); for (const entry of entries) { const base = slugify(entry.base) || 'page'; let slug = base; for (let n = 2; taken.has(slug); n += 1) slug = `${base}-${n}`; taken.add(slug); result.set(entry.key, slug); } return result; } /** * The whole transform: ZIP bytes + options → an import plan the orchestrator * (issue #117) can feed into the page/file/label services. * * `mountDepth` is the 1-based depth of the chosen parent page (0 for the * pond root): containers + note level must fit inside MAX_PAGE_DEPTH. */ export function planVaultImport( zip: Uint8Array, options: { frontmatterMode: FrontmatterMode; existingSlugs: ReadonlySet; mountDepth: number; }, ): VaultImportPlan { const vault = parseVaultZip(zip); // Levels available below the mount page, minus one for the notes. const maxContainerLevels = Math.max(0, MAX_PAGE_DEPTH - options.mountDepth - 1); const { containers, parentKeyByNote } = planFolders(vault, maxContainerLevels); // Reserve every slug up front — containers first, then notes; the note // key is its vault path. const slugEntries = [ ...containers.map((c) => ({ key: `container:${c.key}`, base: c.title.split('/').pop()! })), ...vault.notes.map((note) => ({ key: `note:${note.path}`, base: note.name })), ]; const slugs = planSlugs(slugEntries, options.existingSlugs); const noteSlugByPath = new Map(); for (const note of vault.notes) { noteSlugByPath.set(note.path, slugs.get(`note:${note.path}`)!); } const resolver = buildResolver(vault, noteSlugByPath); const plannedNotes: PlannedNote[] = vault.notes.map((note) => { const frontmatter = extractFrontmatter(note.markdown, options.frontmatterMode); const inline = extractInlineTags(frontmatter.markdown); const links = rewriteLinks(inline.markdown, resolver); return { key: `note:${note.path}`, parentKey: parentKeyByNote.get(note.path) ? `container:${parentKeyByNote.get(note.path)!}` : null, title: note.name, slug: noteSlugByPath.get(note.path)!, markdown: links.markdown, tags: dedupeTags([...frontmatter.tags, ...inline.tags]), imageAssets: links.imageAssets, attachmentAssets: links.attachmentAssets, }; }); const referencedAssets = new Set(); for (const note of plannedNotes) { for (const path of note.imageAssets) referencedAssets.add(path); for (const path of note.attachmentAssets) referencedAssets.add(path); } return { containers: containers.map((c) => ({ key: `container:${c.key}`, parentKey: c.parentKey ? `container:${c.parentKey}` : null, title: c.title, slug: slugs.get(`container:${c.key}`)!, })), notes: plannedNotes, referencedAssets, }; } function dedupeTags(tags: string[][]): string[][] { const seen = new Set(); const result: string[][] = []; for (const tag of tags) { if (tag.length === 0) continue; const key = tag.join('/').toLowerCase(); if (seen.has(key)) continue; seen.add(key); result.push(tag); } return result; }