Obsidian vault transform module (#116)
Some checks failed
CI / Lint, typecheck, test (push) Failing after 52s
CI / Auth e2e pack (push) Has been skipped
CI / Import/export fidelity gate (push) Has been skipped
CI / Build container images (push) Has been skipped
CD / Build and push images (push) Failing after 1m36s
CD / Deploy to Test (push) Has been skipped
CD / Smoke tests against Test (push) Has been skipped
CD / Promote to Int (push) Has been skipped
Some checks failed
CI / Lint, typecheck, test (push) Failing after 52s
CI / Auth e2e pack (push) Has been skipped
CI / Import/export fidelity gate (push) Has been skipped
CI / Build container images (push) Has been skipped
CD / Build and push images (push) Failing after 1m36s
CD / Deploy to Test (push) Has been skipped
CD / Smoke tests against Test (push) Has been skipped
CD / Promote to Int (push) Has been skipped
Pure functions from vault ZIP to import plan — no DB, no DI: - parseVaultZip: fflate unzip with the plugin-package protections (zip-slip rejection, incremental unpacked ceiling 256 MiB, parameterized for tests); dot-directories like .obsidian/ skipped; deterministic ordering. - extractFrontmatter: leading --- block, tags:/tag: in scalar, inline- array, and block-list forms; strip mode drops the block, preserve re-emits it as a fenced yaml code block. - extractInlineTags: fence- and inline-code-aware #tag / #nested/tag extraction and removal (headings and pure numbers untouched). - rewriteLinks: [[Name]], [[Name|Display]], [[Name#Heading]] (fragment stripped), [[folder/Name]] (path match beats basename) → the FINAL slug with the human name as display; unresolvable → slugified phantom; ![[img]] and relative  → vault-asset: placeholders the uploader resolves (#117); non-image embeds → italic filename + page attachment; SVG deliberately stays an attachment (never inline, security.md); note embeds degrade to plain wikilinks. - planFolders: folder chains merged at the deepest levels to fit MAX_PAGE_DEPTH below the mount page (merged titles read c/d). - planSlugs: -n suffixing against existing ∪ batch; duplicate basenames resolve to the lexicographically first vault path. - planVaultImport ties it together into containers + notes + the referenced-asset set. Fixture vault under fixtures/import/obsidian-vault/ (umlauts, duplicate basenames, nested tags, deep folders, embeds, code traps); 13 unit tests colocated. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
64e21e9f94
commit
09abda3ada
221
apps/api/src/import-export/obsidian-vault.test.ts
Normal file
221
apps/api/src/import-export/obsidian-vault.test.ts
Normal file
@ -0,0 +1,221 @@
|
||||
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
||||
import { join, relative } from 'node:path';
|
||||
|
||||
import { zipSync } from 'fflate';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
ASSET_PLACEHOLDER_PREFIX,
|
||||
VaultError,
|
||||
extractFrontmatter,
|
||||
extractInlineTags,
|
||||
parseVaultZip,
|
||||
planFolders,
|
||||
planSlugs,
|
||||
planVaultImport,
|
||||
} from './obsidian-vault';
|
||||
|
||||
const FIXTURE_DIR = join(__dirname, '../../../../fixtures/import/obsidian-vault');
|
||||
|
||||
/** Zips the checked-in fixture vault (also the e2e helper's job later, #119). */
|
||||
function fixtureZip(extra: Record<string, Uint8Array> = {}): Uint8Array {
|
||||
const entries: Record<string, Uint8Array> = { ...extra };
|
||||
const walk = (dir: string): void => {
|
||||
for (const name of readdirSync(dir)) {
|
||||
const path = join(dir, name);
|
||||
if (statSync(path).isDirectory()) walk(path);
|
||||
else entries[relative(FIXTURE_DIR, path)] = new Uint8Array(readFileSync(path));
|
||||
}
|
||||
};
|
||||
walk(FIXTURE_DIR);
|
||||
return zipSync(entries);
|
||||
}
|
||||
|
||||
function planFixture(mountDepth = 0, frontmatterMode: 'strip' | 'preserve' = 'strip') {
|
||||
return planVaultImport(fixtureZip(), {
|
||||
frontmatterMode,
|
||||
existingSlugs: new Set(),
|
||||
mountDepth,
|
||||
});
|
||||
}
|
||||
|
||||
describe('parseVaultZip (issue #116)', () => {
|
||||
it('splits notes from assets and skips dot-directories', () => {
|
||||
const vault = parseVaultZip(fixtureZip());
|
||||
expect(vault.notes.map((n) => n.path)).toEqual([
|
||||
'Notizen/Über Uns',
|
||||
'Projekte/Projekt A',
|
||||
'Projekte/Projekt B',
|
||||
'Projekte/Tief/Ebene4/Ebene5/Tiefe Notiz',
|
||||
'Startseite',
|
||||
'Über Uns',
|
||||
]);
|
||||
expect(vault.assets.map((a) => a.path)).toEqual(['media/teich.png', 'media/unterlagen.pdf']);
|
||||
expect(vault.assets[0]!.extension).toBe('png');
|
||||
});
|
||||
|
||||
it('rejects zip-slip paths, garbage, and oversized vaults', () => {
|
||||
expect(() => parseVaultZip(new Uint8Array([1, 2, 3]))).toThrowError(VaultError);
|
||||
expect(() =>
|
||||
parseVaultZip(fixtureZip({ '../evil.md': new TextEncoder().encode('x') })),
|
||||
).toThrowError(/Illegal path/);
|
||||
expect(() => parseVaultZip(fixtureZip(), 64)).toThrowError(/too large/);
|
||||
try {
|
||||
parseVaultZip(fixtureZip(), 64);
|
||||
} catch (error) {
|
||||
expect((error as VaultError).code).toBe('import_vault_too_large');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractFrontmatter', () => {
|
||||
const note = '---\ntitle: X\ntags: [a, b/c]\n---\n\nBody.';
|
||||
|
||||
it('strips the block and reads inline-array tags', () => {
|
||||
const result = extractFrontmatter(note, 'strip');
|
||||
expect(result.markdown).toBe('\nBody.');
|
||||
expect(result.tags).toEqual([['a'], ['b', 'c']]);
|
||||
});
|
||||
|
||||
it('preserves the block as a yaml code fence', () => {
|
||||
const result = extractFrontmatter(note, 'preserve');
|
||||
expect(result.markdown).toContain('```yaml\ntitle: X\ntags: [a, b/c]\n```');
|
||||
expect(result.markdown.trimEnd().endsWith('Body.')).toBe(true);
|
||||
});
|
||||
|
||||
it('reads block-list and scalar tag forms; leaves notes without frontmatter alone', () => {
|
||||
const block = extractFrontmatter('---\ntags:\n - team\n - "x/y"\n---\nHi', 'strip');
|
||||
expect(block.tags).toEqual([['team'], ['x', 'y']]);
|
||||
const scalar = extractFrontmatter('---\ntag: solo\n---\nHi', 'strip');
|
||||
expect(scalar.tags).toEqual([['solo']]);
|
||||
const none = extractFrontmatter('# Just a heading\n---\nnot frontmatter', 'strip');
|
||||
expect(none.markdown).toContain('# Just a heading');
|
||||
expect(none.tags).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractInlineTags', () => {
|
||||
it('collects nested tags, removes them, and never touches code or headings', () => {
|
||||
const result = extractInlineTags(
|
||||
'# Heading stays\n\nText #eins mitten im Satz und #a/b am Ende.\n' +
|
||||
'Code: `#nichttag` bleibt.\n\n```\n#auchkeintag\n```\n\nNummern #123 bleiben.',
|
||||
);
|
||||
expect(result.tags).toEqual([['eins'], ['a', 'b']]);
|
||||
expect(result.markdown).toContain('# Heading stays');
|
||||
expect(result.markdown).toContain('Text mitten im Satz und am Ende.');
|
||||
expect(result.markdown).toContain('`#nichttag`');
|
||||
expect(result.markdown).toContain('#auchkeintag');
|
||||
expect(result.markdown).toContain('#123');
|
||||
});
|
||||
});
|
||||
|
||||
describe('planFolders / planSlugs', () => {
|
||||
it('merges the deepest folder levels beyond the budget', () => {
|
||||
const vault = parseVaultZip(fixtureZip());
|
||||
const { containers, parentKeyByNote } = planFolders(vault, 3);
|
||||
// Assets never create containers (no `media` here).
|
||||
expect(containers.map((c) => c.title)).toEqual([
|
||||
'Notizen',
|
||||
'Projekte',
|
||||
'Tief',
|
||||
'Ebene4/Ebene5',
|
||||
]);
|
||||
expect(parentKeyByNote.get('Projekte/Tief/Ebene4/Ebene5/Tiefe Notiz')).toBe(
|
||||
'Projekte/Tief/Ebene4/Ebene5',
|
||||
);
|
||||
expect(parentKeyByNote.get('Startseite')).toBeNull();
|
||||
});
|
||||
|
||||
it('suffixes collisions against existing and batch slugs deterministically', () => {
|
||||
const slugs = planSlugs(
|
||||
[
|
||||
{ key: 'a', base: 'Über Uns' },
|
||||
{ key: 'b', base: 'über uns' },
|
||||
{ key: 'c', base: 'Taken' },
|
||||
],
|
||||
new Set(['taken']),
|
||||
);
|
||||
expect(slugs.get('a')).toBe('ueber-uns');
|
||||
expect(slugs.get('b')).toBe('ueber-uns-2');
|
||||
expect(slugs.get('c')).toBe('taken-2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('planVaultImport (the whole transform)', () => {
|
||||
it('rewrites every link form to final slugs', () => {
|
||||
const plan = planFixture();
|
||||
const note = (path: string) => plan.notes.find((n) => n.key === `note:${path}`)!;
|
||||
|
||||
const start = note('Startseite');
|
||||
// Basename link → resolved slug with the original name as display.
|
||||
expect(start.markdown).toContain('[[projekt-a|Projekt A]]');
|
||||
// Full-path link keeps its display text.
|
||||
expect(start.markdown).toContain('[[projekt-b|Das zweite Projekt]]');
|
||||
// Duplicate basename: lexicographically first path (Notizen/Über Uns) wins.
|
||||
expect(start.markdown).toContain('[[ueber-uns|Über Uns]]');
|
||||
expect(note('Notizen/Über Uns').slug).toBe('ueber-uns');
|
||||
expect(note('Über Uns').slug).toBe('ueber-uns-2');
|
||||
// Unresolvable → phantom out of the slugified name.
|
||||
expect(start.markdown).toContain('[[fehlt-noch|Fehlt Noch]]');
|
||||
// Heading link strips the fragment.
|
||||
expect(note('Projekte/Projekt B').markdown).toContain('[[projekt-a|Projekt A]]');
|
||||
});
|
||||
|
||||
it('turns image embeds into placeholders and other files into attachments', () => {
|
||||
const plan = planFixture();
|
||||
const a = plan.notes.find((n) => n.key === 'note:Projekte/Projekt A')!;
|
||||
expect(a.markdown).toContain(``);
|
||||
expect(a.imageAssets).toEqual(['media/teich.png']);
|
||||
|
||||
const b = plan.notes.find((n) => n.key === 'note:Projekte/Projekt B')!;
|
||||
expect(b.markdown).toContain('*unterlagen.pdf*');
|
||||
expect(b.attachmentAssets).toEqual(['media/unterlagen.pdf']);
|
||||
// Code spans and fences survive untouched.
|
||||
expect(b.markdown).toContain('`[[NichtLink]] #nichttag`');
|
||||
expect(b.markdown).toContain('[[AuchKeinLink]] #auchkeintag');
|
||||
|
||||
// A relative standard-markdown image resolves too.
|
||||
const about = plan.notes.find((n) => n.key === 'note:Über Uns')!;
|
||||
expect(about.markdown).toContain(``);
|
||||
|
||||
expect([...plan.referencedAssets].sort()).toEqual(['media/teich.png', 'media/unterlagen.pdf']);
|
||||
});
|
||||
|
||||
it('collects frontmatter and inline tags per note', () => {
|
||||
const plan = planFixture();
|
||||
const a = plan.notes.find((n) => n.key === 'note:Projekte/Projekt A')!;
|
||||
expect(a.tags).toEqual([['projekt'], ['status', 'aktiv']]);
|
||||
expect(a.markdown).not.toContain('#status/aktiv');
|
||||
expect(a.markdown).not.toContain('tags: [projekt');
|
||||
|
||||
const start = plan.notes.find((n) => n.key === 'note:Startseite')!;
|
||||
expect(start.tags).toEqual([['willkommen']]);
|
||||
});
|
||||
|
||||
it('honors the preserve frontmatter mode', () => {
|
||||
const plan = planFixture(0, 'preserve');
|
||||
const a = plan.notes.find((n) => n.key === 'note:Projekte/Projekt A')!;
|
||||
expect(a.markdown).toContain('```yaml');
|
||||
expect(a.markdown).toContain('title: Projekt A');
|
||||
});
|
||||
|
||||
it('budgets container levels from the mount depth', () => {
|
||||
// Mount at depth 0 (pond root): 5 container levels available — the deep
|
||||
// path (4 folders) fits unmerged.
|
||||
const roomy = planFixture(0);
|
||||
expect(roomy.containers.map((c) => c.title)).toContain('Ebene5');
|
||||
|
||||
// Mount at depth 2: 3 container levels — the deepest two merge.
|
||||
const tight = planFixture(2);
|
||||
expect(tight.containers.map((c) => c.title)).toContain('Ebene4/Ebene5');
|
||||
const deep = tight.notes.find((n) => n.key.includes('Tiefe Notiz'))!;
|
||||
expect(deep.parentKey).toBe('container:Projekte/Tief/Ebene4/Ebene5');
|
||||
// Containers stay parents-before-children.
|
||||
const keys = tight.containers.map((c) => c.key);
|
||||
for (const container of tight.containers) {
|
||||
if (container.parentKey) {
|
||||
expect(keys.indexOf(container.parentKey)).toBeLessThan(keys.indexOf(container.key));
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
547
apps/api/src/import-export/obsidian-vault.ts
Normal file
547
apps/api/src/import-export/obsidian-vault.ts
Normal file
@ -0,0 +1,547 @@
|
||||
import { MAX_PAGE_DEPTH, slugify } from '@dorfteich/shared';
|
||||
import { unzipSync } from 'fflate';
|
||||
|
||||
/**
|
||||
* Obsidian vault transform (issue #116): pure functions from "uploaded ZIP"
|
||||
* to an import plan — no DB, no DI, fully unit-testable. The orchestration
|
||||
* (issue #117) feeds the plan into the page/file/label services.
|
||||
*
|
||||
* The core impedance mismatch: Obsidian links reference note NAMES
|
||||
* (`[[Meine Notiz]]`, `[[folder/Note#Heading|shown]]`), Dorfteich wikilinks
|
||||
* resolve by SLUG within a pond — and the shared markdown parser reads
|
||||
* `[[target]]` verbatim. So every rewrite here is a text pre-pass over the
|
||||
* note Markdown, done before `markdownToDoc` ever sees it.
|
||||
*/
|
||||
|
||||
export class VaultError extends Error {
|
||||
constructor(
|
||||
readonly code: 'import_vault_invalid_zip' | 'import_vault_too_large',
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
/** Zip-bomb ceiling for the unpacked vault (the ZIP itself is capped by the
|
||||
* upload limit). Matches the plugin-package model. */
|
||||
export const MAX_VAULT_UNPACKED_BYTES = 256 * 1024 * 1024;
|
||||
|
||||
/** Raster formats that become inline images. SVG deliberately stays an
|
||||
* attachment — it is never served inline (security.md §Uploads). */
|
||||
const IMAGE_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp']);
|
||||
|
||||
/** Placeholder scheme the orchestrator replaces with real file ids after
|
||||
* uploading: ``. */
|
||||
export const ASSET_PLACEHOLDER_PREFIX = 'vault-asset:';
|
||||
|
||||
export interface VaultNote {
|
||||
/** Vault-relative path without the `.md` extension. */
|
||||
path: string;
|
||||
/** Basename without `.md` — what Obsidian links refer to. */
|
||||
name: string;
|
||||
markdown: string;
|
||||
}
|
||||
|
||||
export interface VaultAsset {
|
||||
path: string;
|
||||
name: string;
|
||||
data: Uint8Array;
|
||||
/** Lowercase extension without the dot. */
|
||||
extension: string;
|
||||
}
|
||||
|
||||
export interface Vault {
|
||||
notes: VaultNote[];
|
||||
assets: VaultAsset[];
|
||||
}
|
||||
|
||||
export type FrontmatterMode = 'strip' | 'preserve';
|
||||
|
||||
export interface PlannedContainer {
|
||||
/** Unique key: the merged folder path. */
|
||||
key: string;
|
||||
parentKey: string | null;
|
||||
/** Human title — a merged folder reads like `c/d`. */
|
||||
title: string;
|
||||
slug: string;
|
||||
}
|
||||
|
||||
export interface PlannedNote {
|
||||
key: string;
|
||||
parentKey: string | null;
|
||||
title: string;
|
||||
slug: string;
|
||||
/** Rewritten Markdown: links → final slugs, tags stripped, frontmatter
|
||||
* handled, asset embeds → `vault-asset:` placeholders. */
|
||||
markdown: string;
|
||||
/** Tag paths (nested tags split at `/`), e.g. [['projekt'], ['a','b']]. */
|
||||
tags: string[][];
|
||||
/** Image assets referenced by this note (vault paths). */
|
||||
imageAssets: string[];
|
||||
/** Non-image assets referenced by this note (become page attachments). */
|
||||
attachmentAssets: string[];
|
||||
}
|
||||
|
||||
export interface VaultImportPlan {
|
||||
/** Topologically ordered (parents before children). */
|
||||
containers: PlannedContainer[];
|
||||
notes: PlannedNote[];
|
||||
/** Vault paths of every asset actually referenced by some note. */
|
||||
referencedAssets: Set<string>;
|
||||
}
|
||||
|
||||
/** Rejects absolute paths and `..` traversal (zip-slip), plugin-package model. */
|
||||
function safeRelativePath(rawPath: string): string {
|
||||
const path = rawPath.replace(/\\/g, '/').replace(/^\/+/, '');
|
||||
if (path.split('/').some((segment) => segment === '..')) {
|
||||
throw new VaultError('import_vault_invalid_zip', `Illegal path in archive: ${rawPath}`);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unpacks the vault ZIP into notes (`.md`) and assets (everything else),
|
||||
* enforcing the unpacked-size ceiling incrementally. Hidden housekeeping
|
||||
* directories (`.obsidian/`, `.trash/`, any dot-directory) are skipped —
|
||||
* they hold app config, not content.
|
||||
*/
|
||||
export function parseVaultZip(
|
||||
zip: Uint8Array,
|
||||
maxUnpackedBytes: number = MAX_VAULT_UNPACKED_BYTES,
|
||||
): Vault {
|
||||
let entries: Record<string, Uint8Array>;
|
||||
try {
|
||||
entries = unzipSync(zip);
|
||||
} catch {
|
||||
throw new VaultError('import_vault_invalid_zip', 'The file is not a valid ZIP archive');
|
||||
}
|
||||
|
||||
const notes: VaultNote[] = [];
|
||||
const assets: VaultAsset[] = [];
|
||||
let unpacked = 0;
|
||||
for (const [rawPath, bytes] of Object.entries(entries)) {
|
||||
if (rawPath.endsWith('/')) continue; // directory entries
|
||||
const path = safeRelativePath(rawPath);
|
||||
const segments = path.split('/');
|
||||
if (segments.some((segment) => segment.startsWith('.'))) continue;
|
||||
unpacked += bytes.length;
|
||||
if (unpacked > maxUnpackedBytes) {
|
||||
throw new VaultError('import_vault_too_large', 'The unpacked vault is too large');
|
||||
}
|
||||
const basename = segments[segments.length - 1]!;
|
||||
if (basename.toLowerCase().endsWith('.md')) {
|
||||
const noteName = basename.slice(0, -3);
|
||||
notes.push({
|
||||
path: path.slice(0, -3),
|
||||
name: noteName,
|
||||
markdown: new TextDecoder().decode(bytes),
|
||||
});
|
||||
} else {
|
||||
const dot = basename.lastIndexOf('.');
|
||||
assets.push({
|
||||
path,
|
||||
name: basename,
|
||||
data: bytes,
|
||||
extension: dot >= 0 ? basename.slice(dot + 1).toLowerCase() : '',
|
||||
});
|
||||
}
|
||||
}
|
||||
if (notes.length === 0) {
|
||||
throw new VaultError('import_vault_invalid_zip', 'The archive contains no Markdown notes');
|
||||
}
|
||||
// Deterministic plan regardless of zip entry order.
|
||||
notes.sort((a, b) => a.path.localeCompare(b.path));
|
||||
assets.sort((a, b) => a.path.localeCompare(b.path));
|
||||
return { notes, assets };
|
||||
}
|
||||
|
||||
export interface Frontmatter {
|
||||
/** The note body with the frontmatter handled per mode. */
|
||||
markdown: string;
|
||||
/** Tag paths extracted from `tags:`/`tag:` (nested split at `/`). */
|
||||
tags: string[][];
|
||||
}
|
||||
|
||||
function parseTagValue(value: string): string[][] {
|
||||
// `tags: a, b` · `tags: [a, b]` · quoted variants.
|
||||
const inner = value.trim().replace(/^\[/, '').replace(/\]$/, '');
|
||||
return inner
|
||||
.split(',')
|
||||
.map((tag) => tag.trim().replace(/^["'#]+|["']+$/g, ''))
|
||||
.filter(Boolean)
|
||||
.map((tag) => tag.split('/').filter(Boolean));
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles a leading YAML frontmatter block. Only the `tags:` forms Obsidian
|
||||
* uses are interpreted (scalar list, inline array, block list) — everything
|
||||
* else is opaque. `strip` drops the block; `preserve` re-emits it as a
|
||||
* fenced yaml code block so nothing is lost but nothing leaks as prose.
|
||||
*/
|
||||
export function extractFrontmatter(markdown: string, mode: FrontmatterMode): Frontmatter {
|
||||
const match = /^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/.exec(markdown);
|
||||
if (!match) return { markdown, tags: [] };
|
||||
|
||||
const block = match[1]!;
|
||||
const body = markdown.slice(match[0].length);
|
||||
const tags: string[][] = [];
|
||||
const lines = block.split(/\r?\n/);
|
||||
for (let i = 0; i < lines.length; i += 1) {
|
||||
const line = lines[i]!;
|
||||
const keyMatch = /^(tags?)[ \t]*:[ \t]*(.*)$/i.exec(line);
|
||||
if (!keyMatch) continue;
|
||||
const value = keyMatch[2]!.trim();
|
||||
if (value) {
|
||||
tags.push(...parseTagValue(value));
|
||||
continue;
|
||||
}
|
||||
// Block list form: consume the following `- item` lines.
|
||||
for (let j = i + 1; j < lines.length; j += 1) {
|
||||
const item = /^[ \t]+-[ \t]*(.+)$/.exec(lines[j]!);
|
||||
if (!item) break;
|
||||
tags.push(...parseTagValue(item[1]!));
|
||||
}
|
||||
}
|
||||
|
||||
const preserved = mode === 'preserve' ? `\`\`\`yaml\n${block}\n\`\`\`\n\n` : '';
|
||||
return { markdown: preserved + body, tags };
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits Markdown into alternating segments outside/inside code so the tag
|
||||
* and link passes never touch fenced blocks or inline code. Yields the
|
||||
* outside segments to `transform` and stitches everything back together.
|
||||
*/
|
||||
function transformOutsideCode(markdown: string, transform: (segment: string) => string): string {
|
||||
const lines = markdown.split('\n');
|
||||
const out: string[] = [];
|
||||
let inFence = false;
|
||||
let fenceMarker = '';
|
||||
for (const line of lines) {
|
||||
const fence = /^([ \t]*)(```+|~~~+)/.exec(line);
|
||||
if (fence && !inFence) {
|
||||
inFence = true;
|
||||
fenceMarker = fence[2]![0]!;
|
||||
out.push(line);
|
||||
continue;
|
||||
}
|
||||
if (fence && inFence && fence[2]![0] === fenceMarker) {
|
||||
inFence = false;
|
||||
out.push(line);
|
||||
continue;
|
||||
}
|
||||
if (inFence) {
|
||||
out.push(line);
|
||||
continue;
|
||||
}
|
||||
// Outside fences: protect inline code spans.
|
||||
const parts = line.split(/(`[^`]*`)/);
|
||||
out.push(parts.map((part) => (part.startsWith('`') ? part : transform(part))).join(''));
|
||||
}
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
export interface InlineTags {
|
||||
markdown: string;
|
||||
tags: string[][];
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts inline `#tag` / `#nested/tag` occurrences and removes them from
|
||||
* the text (Stefan's decision: ALL Obsidian tags become labels). Headings are
|
||||
* safe — `#` followed by whitespace never matches; pure-number "tags" (like
|
||||
* anchors or issue refs) are ignored, matching Obsidian's own rule.
|
||||
*/
|
||||
export function extractInlineTags(markdown: string): InlineTags {
|
||||
const tags: string[][] = [];
|
||||
const result = transformOutsideCode(markdown, (segment) =>
|
||||
segment.replace(/(^|[\s(])#([\p{L}\p{N}_/-]+)/gu, (whole, prefix: string, tag: string) => {
|
||||
if (!/\p{L}/u.test(tag)) return whole; // needs at least one letter
|
||||
tags.push(tag.split('/').filter(Boolean));
|
||||
return prefix;
|
||||
}),
|
||||
);
|
||||
// Collapse the whitespace runs left behind by removed tags.
|
||||
const cleaned = transformOutsideCode(result, (segment) =>
|
||||
segment.replace(/[ \t]{2,}/g, ' ').replace(/[ \t]+$/g, ''),
|
||||
);
|
||||
return { markdown: cleaned, tags };
|
||||
}
|
||||
|
||||
/** How a `[[target]]` resolves: to a note, an asset, or nothing. */
|
||||
export interface LinkResolver {
|
||||
/** Final slug for a note target, or null when unresolvable. */
|
||||
noteSlug(target: string): string | null;
|
||||
/** Vault path of an asset target, or null. */
|
||||
assetPath(target: string): { path: string; extension: string } | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the vault-wide resolver: case-insensitive basename matching with
|
||||
* full-path precedence, duplicates resolved to the lexicographically first
|
||||
* vault path (documented rule; Obsidian's "closest" has no meaning without
|
||||
* a source-file anchor when the whole vault imports at once).
|
||||
*/
|
||||
export function buildResolver(vault: Vault, noteSlugByPath: Map<string, string>): LinkResolver {
|
||||
const notesByName = new Map<string, string>(); // lower name → path (first wins)
|
||||
const notesByPath = new Map<string, string>(); // lower path → path
|
||||
for (const note of vault.notes) {
|
||||
const lowerName = note.name.toLowerCase();
|
||||
if (!notesByName.has(lowerName)) notesByName.set(lowerName, note.path);
|
||||
notesByPath.set(note.path.toLowerCase(), note.path);
|
||||
}
|
||||
const assetsByName = new Map<string, VaultAsset>();
|
||||
const assetsByPath = new Map<string, VaultAsset>();
|
||||
for (const asset of vault.assets) {
|
||||
const lowerName = asset.name.toLowerCase();
|
||||
if (!assetsByName.has(lowerName)) assetsByName.set(lowerName, asset);
|
||||
assetsByPath.set(asset.path.toLowerCase(), asset);
|
||||
}
|
||||
|
||||
return {
|
||||
noteSlug(target: string): string | null {
|
||||
const lower = target.toLowerCase().replace(/\.md$/, '');
|
||||
const path = lower.includes('/')
|
||||
? (notesByPath.get(lower) ?? notesByName.get(lower.split('/').pop()!))
|
||||
: notesByName.get(lower);
|
||||
return path ? (noteSlugByPath.get(path) ?? null) : null;
|
||||
},
|
||||
assetPath(target: string): { path: string; extension: string } | null {
|
||||
const lower = target.toLowerCase();
|
||||
const asset = lower.includes('/')
|
||||
? (assetsByPath.get(lower) ?? assetsByName.get(lower.split('/').pop()!))
|
||||
: assetsByName.get(lower);
|
||||
return asset ? { path: asset.path, extension: asset.extension } : null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export interface RewrittenLinks {
|
||||
markdown: string;
|
||||
imageAssets: string[];
|
||||
attachmentAssets: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The link pass (fence-aware): rewrites every Obsidian link form to what the
|
||||
* shared markdown parser understands.
|
||||
* - `[[Name]]`, `[[Name|Display]]`, `[[Name#Heading]]`, `[[folder/Name]]` →
|
||||
* `[[final-slug|display or original name]]` (heading stripped).
|
||||
* - Unresolvable note targets → `[[slugify(target)|target]]` — a phantom
|
||||
* link, exactly like typing a dead wikilink by hand.
|
||||
* - `![[image.png]]` / `` on a vault image →
|
||||
* `` for the uploader to resolve.
|
||||
* - `![[doc.pdf]]` (non-image asset) → `*doc.pdf*` text; the file becomes a
|
||||
* page attachment (paperclip panel), which has no inline node.
|
||||
* - `![[Other Note]]` (note embed) → a plain wikilink (no transclusion).
|
||||
*/
|
||||
export function rewriteLinks(markdown: string, resolver: LinkResolver): RewrittenLinks {
|
||||
const imageAssets = new Set<string>();
|
||||
const attachmentAssets = new Set<string>();
|
||||
|
||||
const rewriteTarget = (rawTarget: string, rawDisplay: string | undefined): string => {
|
||||
const target = rawTarget.split('#')[0]!.trim();
|
||||
if (!target) return rawDisplay ?? rawTarget; // pure heading link — keep text
|
||||
const display = rawDisplay?.trim() || target.split('/').pop()!;
|
||||
const slug = resolver.noteSlug(target) ?? (slugify(target.split('/').pop()!) || 'page');
|
||||
return `[[${slug}|${display}]]`;
|
||||
};
|
||||
|
||||
const result = transformOutsideCode(markdown, (segment) => {
|
||||
let out = segment;
|
||||
// Embeds first — `![[…]]` would otherwise be eaten as `!` + wikilink.
|
||||
out = out.replace(/!\[\[([^[\]\n]+)\]\]/g, (_whole, inner: string) => {
|
||||
const [target, display] = splitPipe(inner);
|
||||
const asset = resolver.assetPath(target.split('#')[0]!.trim());
|
||||
if (asset) {
|
||||
if (IMAGE_EXTENSIONS.has(asset.extension)) {
|
||||
imageAssets.add(asset.path);
|
||||
const alt = display ?? asset.path.split('/').pop()!;
|
||||
return ``;
|
||||
}
|
||||
attachmentAssets.add(asset.path);
|
||||
return `*${display ?? asset.path.split('/').pop()!}*`;
|
||||
}
|
||||
// A note embed (transclusion) degrades to a normal link.
|
||||
return rewriteTarget(target, display);
|
||||
});
|
||||
// Standard markdown images with a relative vault path.
|
||||
out = out.replace(/!\[([^\]]*)\]\(([^)\s]+)\)/g, (whole, alt: string, src: string) => {
|
||||
if (/^[a-z][a-z0-9+.-]*:/i.test(src)) return whole; // absolute URL/data:
|
||||
const asset = resolver.assetPath(decodeURIComponent(src));
|
||||
if (!asset) return whole;
|
||||
if (IMAGE_EXTENSIONS.has(asset.extension)) {
|
||||
imageAssets.add(asset.path);
|
||||
return ``;
|
||||
}
|
||||
attachmentAssets.add(asset.path);
|
||||
return `*${alt || asset.name}*`;
|
||||
});
|
||||
// Plain wikilinks.
|
||||
out = out.replace(/\[\[([^[\]\n]+)\]\]/g, (_whole, inner: string) => {
|
||||
const [target, display] = splitPipe(inner);
|
||||
return rewriteTarget(target, display);
|
||||
});
|
||||
return out;
|
||||
});
|
||||
|
||||
return {
|
||||
markdown: result,
|
||||
imageAssets: [...imageAssets],
|
||||
attachmentAssets: [...attachmentAssets],
|
||||
};
|
||||
}
|
||||
|
||||
function splitPipe(inner: string): [string, string | undefined] {
|
||||
const pipe = inner.indexOf('|');
|
||||
if (pipe < 0) return [inner, undefined];
|
||||
return [inner.slice(0, pipe), inner.slice(pipe + 1)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps every note's folder path to a container chain of at most
|
||||
* `maxContainerLevels` (the deepest levels merge into one container whose
|
||||
* title keeps the joined path, e.g. `c/d`). Returns the ordered container
|
||||
* list (parents first) and the container key each note hangs under.
|
||||
*/
|
||||
export function planFolders(
|
||||
vault: Vault,
|
||||
maxContainerLevels: number,
|
||||
): {
|
||||
containers: { key: string; parentKey: string | null; title: string }[];
|
||||
parentKeyByNote: Map<string, string | null>;
|
||||
} {
|
||||
const containers = new Map<string, { key: string; parentKey: string | null; title: string }>();
|
||||
const parentKeyByNote = new Map<string, string | null>();
|
||||
|
||||
for (const note of vault.notes) {
|
||||
const folders = note.path.split('/').slice(0, -1);
|
||||
if (folders.length === 0 || maxContainerLevels <= 0) {
|
||||
parentKeyByNote.set(note.path, null);
|
||||
continue;
|
||||
}
|
||||
const kept = folders.slice(0, Math.max(0, maxContainerLevels - 1));
|
||||
const merged = folders.slice(Math.max(0, maxContainerLevels - 1));
|
||||
const chain = merged.length > 0 ? [...kept, merged.join('/')] : kept;
|
||||
|
||||
let parentKey: string | null = null;
|
||||
let keyPrefix = '';
|
||||
for (const title of chain) {
|
||||
keyPrefix = keyPrefix ? `${keyPrefix}/${title}` : title;
|
||||
if (!containers.has(keyPrefix)) {
|
||||
containers.set(keyPrefix, { key: keyPrefix, parentKey, title });
|
||||
}
|
||||
parentKey = keyPrefix;
|
||||
}
|
||||
parentKeyByNote.set(note.path, parentKey);
|
||||
}
|
||||
|
||||
// Map insertion already guarantees parents-before-children (each chain is
|
||||
// walked top-down), and iteration order is insertion order.
|
||||
return { containers: [...containers.values()], parentKeyByNote };
|
||||
}
|
||||
|
||||
/**
|
||||
* Assigns final slugs to every container and note: `-2/-3` suffixing against
|
||||
* the existing pond slugs and everything planned so far. Deterministic:
|
||||
* containers (path order) before notes (path order).
|
||||
*/
|
||||
export function planSlugs(
|
||||
entries: { key: string; base: string }[],
|
||||
existingSlugs: ReadonlySet<string>,
|
||||
): Map<string, string> {
|
||||
const taken = new Set(existingSlugs);
|
||||
const result = new Map<string, string>();
|
||||
for (const entry of entries) {
|
||||
const base = slugify(entry.base) || 'page';
|
||||
let slug = base;
|
||||
for (let n = 2; taken.has(slug); n += 1) slug = `${base}-${n}`;
|
||||
taken.add(slug);
|
||||
result.set(entry.key, slug);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* The whole transform: ZIP bytes + options → an import plan the orchestrator
|
||||
* (issue #117) can feed into the page/file/label services.
|
||||
*
|
||||
* `mountDepth` is the 1-based depth of the chosen parent page (0 for the
|
||||
* pond root): containers + note level must fit inside MAX_PAGE_DEPTH.
|
||||
*/
|
||||
export function planVaultImport(
|
||||
zip: Uint8Array,
|
||||
options: {
|
||||
frontmatterMode: FrontmatterMode;
|
||||
existingSlugs: ReadonlySet<string>;
|
||||
mountDepth: number;
|
||||
},
|
||||
): VaultImportPlan {
|
||||
const vault = parseVaultZip(zip);
|
||||
|
||||
// Levels available below the mount page, minus one for the notes.
|
||||
const maxContainerLevels = Math.max(0, MAX_PAGE_DEPTH - options.mountDepth - 1);
|
||||
const { containers, parentKeyByNote } = planFolders(vault, maxContainerLevels);
|
||||
|
||||
// Reserve every slug up front — containers first, then notes; the note
|
||||
// key is its vault path.
|
||||
const slugEntries = [
|
||||
...containers.map((c) => ({ key: `container:${c.key}`, base: c.title.split('/').pop()! })),
|
||||
...vault.notes.map((note) => ({ key: `note:${note.path}`, base: note.name })),
|
||||
];
|
||||
const slugs = planSlugs(slugEntries, options.existingSlugs);
|
||||
|
||||
const noteSlugByPath = new Map<string, string>();
|
||||
for (const note of vault.notes) {
|
||||
noteSlugByPath.set(note.path, slugs.get(`note:${note.path}`)!);
|
||||
}
|
||||
const resolver = buildResolver(vault, noteSlugByPath);
|
||||
|
||||
const plannedNotes: PlannedNote[] = vault.notes.map((note) => {
|
||||
const frontmatter = extractFrontmatter(note.markdown, options.frontmatterMode);
|
||||
const inline = extractInlineTags(frontmatter.markdown);
|
||||
const links = rewriteLinks(inline.markdown, resolver);
|
||||
return {
|
||||
key: `note:${note.path}`,
|
||||
parentKey: parentKeyByNote.get(note.path)
|
||||
? `container:${parentKeyByNote.get(note.path)!}`
|
||||
: null,
|
||||
title: note.name,
|
||||
slug: noteSlugByPath.get(note.path)!,
|
||||
markdown: links.markdown,
|
||||
tags: dedupeTags([...frontmatter.tags, ...inline.tags]),
|
||||
imageAssets: links.imageAssets,
|
||||
attachmentAssets: links.attachmentAssets,
|
||||
};
|
||||
});
|
||||
|
||||
const referencedAssets = new Set<string>();
|
||||
for (const note of plannedNotes) {
|
||||
for (const path of note.imageAssets) referencedAssets.add(path);
|
||||
for (const path of note.attachmentAssets) referencedAssets.add(path);
|
||||
}
|
||||
|
||||
return {
|
||||
containers: containers.map((c) => ({
|
||||
key: `container:${c.key}`,
|
||||
parentKey: c.parentKey ? `container:${c.parentKey}` : null,
|
||||
title: c.title,
|
||||
slug: slugs.get(`container:${c.key}`)!,
|
||||
})),
|
||||
notes: plannedNotes,
|
||||
referencedAssets,
|
||||
};
|
||||
}
|
||||
|
||||
function dedupeTags(tags: string[][]): string[][] {
|
||||
const seen = new Set<string>();
|
||||
const result: string[][] = [];
|
||||
for (const tag of tags) {
|
||||
if (tag.length === 0) continue;
|
||||
const key = tag.join('/').toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
result.push(tag);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
1
fixtures/import/obsidian-vault/.obsidian/app.json
vendored
Normal file
1
fixtures/import/obsidian-vault/.obsidian/app.json
vendored
Normal file
@ -0,0 +1 @@
|
||||
{"theme":"moonstone"}
|
||||
3
fixtures/import/obsidian-vault/Notizen/Über Uns.md
Normal file
3
fixtures/import/obsidian-vault/Notizen/Über Uns.md
Normal file
@ -0,0 +1,3 @@
|
||||
# Über Uns (Notizen)
|
||||
|
||||
Der Namensvetter im Notizen-Ordner.
|
||||
14
fixtures/import/obsidian-vault/Projekte/Projekt A.md
Normal file
14
fixtures/import/obsidian-vault/Projekte/Projekt A.md
Normal file
@ -0,0 +1,14 @@
|
||||
---
|
||||
title: Projekt A
|
||||
tags: [projekt, status/aktiv]
|
||||
---
|
||||
|
||||
# Projekt A
|
||||
|
||||
Zustaendig: [[Über Uns|das Team]] — Status #status/aktiv
|
||||
|
||||
![[teich.png]]
|
||||
|
||||
## Ziele
|
||||
|
||||
Mehr Seerosen.
|
||||
9
fixtures/import/obsidian-vault/Projekte/Projekt B.md
Normal file
9
fixtures/import/obsidian-vault/Projekte/Projekt B.md
Normal file
@ -0,0 +1,9 @@
|
||||
# Projekt B
|
||||
|
||||
Siehe [[Projekt A#Ziele]] und die Unterlagen: ![[unterlagen.pdf]]
|
||||
|
||||
Im Code bleibt alles stehen: `[[NichtLink]] #nichttag`
|
||||
|
||||
```text
|
||||
[[AuchKeinLink]] #auchkeintag
|
||||
```
|
||||
@ -0,0 +1,3 @@
|
||||
# Tiefe Notiz
|
||||
|
||||
Ganz unten. Zurueck zur [[Startseite]].
|
||||
8
fixtures/import/obsidian-vault/Startseite.md
Normal file
8
fixtures/import/obsidian-vault/Startseite.md
Normal file
@ -0,0 +1,8 @@
|
||||
# Startseite
|
||||
|
||||
Willkommen im Vault. #willkommen
|
||||
|
||||
- Projekt: [[Projekt A]]
|
||||
- Direkt per Pfad: [[Projekte/Projekt B|Das zweite Projekt]]
|
||||
- Team: [[Über Uns]]
|
||||
- Noch offen: [[Fehlt Noch]]
|
||||
BIN
fixtures/import/obsidian-vault/media/teich.png
Normal file
BIN
fixtures/import/obsidian-vault/media/teich.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 70 B |
4
fixtures/import/obsidian-vault/media/unterlagen.pdf
Normal file
4
fixtures/import/obsidian-vault/media/unterlagen.pdf
Normal file
@ -0,0 +1,4 @@
|
||||
%PDF-1.4
|
||||
1 0 obj<<>>endobj
|
||||
trailer<<>>
|
||||
%%EOF
|
||||
8
fixtures/import/obsidian-vault/Über Uns.md
Normal file
8
fixtures/import/obsidian-vault/Über Uns.md
Normal file
@ -0,0 +1,8 @@
|
||||
---
|
||||
tags:
|
||||
- team
|
||||
---
|
||||
|
||||
# Über Uns
|
||||
|
||||
Wir sind der Teich. 
|
||||
Loading…
Reference in New Issue
Block a user