diff --git a/apps/api/prisma/migrations/20260714120000_conversion_job_options/migration.sql b/apps/api/prisma/migrations/20260714120000_conversion_job_options/migration.sql new file mode 100644 index 0000000..1a467f6 --- /dev/null +++ b/apps/api/prisma/migrations/20260714120000_conversion_job_options/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "conversion_jobs" ADD COLUMN "options" JSONB; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index fbfc8ef..2c43fb5 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -705,6 +705,9 @@ model ConversionJob { /// downloadable and is purged (GDPR data minimization). Null for every /// other job kind, whose result never expires. expiresAt DateTime? @map("expires_at") + /// Kind-specific job options (issue #117): a vault import carries + /// `{parentPageId, labelIds, frontmatterMode}`. Null for other kinds. + options Json? createdAt DateTime @default(now()) @map("created_at") updatedAt DateTime @updatedAt @map("updated_at") diff --git a/apps/api/src/import-export/conversion-job.service.ts b/apps/api/src/import-export/conversion-job.service.ts index 514bfe4..ba6b9c8 100644 --- a/apps/api/src/import-export/conversion-job.service.ts +++ b/apps/api/src/import-export/conversion-job.service.ts @@ -19,6 +19,12 @@ export interface EnqueueConversion { * original upload file name (a title fallback). */ pondId?: string; sourceName?: string; + /** Kind-specific options persisted with the job (issue #117). */ + options?: unknown; + /** Input ceiling override (issue #117): the 25 MiB default protects the + * pandoc sidecar; a vault import never touches it and may use the full + * upload limit instead. */ + maxInputBytes?: number; } export interface ConversionResultPayload { @@ -50,10 +56,11 @@ export class ConversionJobService { } async enqueue(request: EnqueueConversion): Promise { - if (request.input.byteLength > MAX_CONVERSION_INPUT_BYTES) { + const maxInputBytes = request.maxInputBytes ?? MAX_CONVERSION_INPUT_BYTES; + if (request.input.byteLength > maxInputBytes) { throw new PayloadTooLargeException({ code: 'file_too_large', - details: { limitBytes: MAX_CONVERSION_INPUT_BYTES }, + details: { limitBytes: maxInputBytes }, }); } const job = await this.prisma.conversionJob.create({ @@ -65,6 +72,7 @@ export class ConversionJobService { sourceFormat: request.from, targetFormat: request.to, standalone: request.standalone ?? true, + options: request.options === undefined ? undefined : (request.options as object), // Prisma's Bytes maps to Uint8Array; a Node Buffer's // backing store is ArrayBufferLike, so copy into a plain Uint8Array. input: new Uint8Array(request.input), diff --git a/apps/api/src/import-export/import-export.module.ts b/apps/api/src/import-export/import-export.module.ts index e74ec26..2af2b98 100644 --- a/apps/api/src/import-export/import-export.module.ts +++ b/apps/api/src/import-export/import-export.module.ts @@ -1,6 +1,7 @@ import { Module, OnModuleInit } from '@nestjs/common'; import { FilesModule } from '../files/files.module'; +import { LabelsModule } from '../labels/labels.module'; import { PagesModule } from '../pages/pages.module'; import { PluginsModule } from '../plugins/plugins.module'; import { SchedulerModule } from '../scheduler/scheduler.module'; @@ -30,7 +31,7 @@ const EXPORT_PURGE_CADENCE_SECONDS = 60 * 60; * feature exports (#65/#67), and the GDPR account data export (#68). */ @Module({ - imports: [FilesModule, PagesModule, PluginsModule, SchedulerModule], + imports: [FilesModule, LabelsModule, PagesModule, PluginsModule, SchedulerModule], controllers: [JobsController, ImportController, ExportController, DataExportController], providers: [ ConversionJobService, diff --git a/apps/api/src/import-export/import-vault.e2e.db.test.ts b/apps/api/src/import-export/import-vault.e2e.db.test.ts new file mode 100644 index 0000000..79bb507 --- /dev/null +++ b/apps/api/src/import-export/import-vault.e2e.db.test.ts @@ -0,0 +1,284 @@ +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join, relative } from 'node:path'; + +import { INestApplication } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; +import { zipSync } from 'fflate'; +import request from 'supertest'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +import { AuthTokensService } from '../auth/auth-tokens.service'; +import { createTestApp, sessionCookieOf } from '../testing/test-app'; +import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db'; +import { UsersService } from '../users/users.service'; + +import { ConversionWorker } from './conversion-worker.service'; + +/** + * Obsidian vault import (issue #117): endpoint gating, the whole fixture + * import (tree, slugs, links, labels, assets), collision handling, quota + * rollback, and the size ceilings. Runs the real worker (drained per test) + * against the checked-in fixture vault. + */ + +const FIXTURE_DIR = join(__dirname, '../../../../fixtures/import/obsidian-vault'); + +function fixtureZip(): Buffer { + const entries: Record = {}; + const walk = (dir: string): void => { + for (const name of readdirSync(dir)) { + const path = join(dir, name); + if (statSync(path).isDirectory()) walk(path); + else entries[relative(FIXTURE_DIR, path)] = new Uint8Array(readFileSync(path)); + } + }; + walk(FIXTURE_DIR); + return Buffer.from(zipSync(entries)); +} + +describe.skipIf(!hasTestDb)('vault import (e2e, issue #117)', () => { + let app: INestApplication; + let prisma: PrismaClient; + let worker: ConversionWorker; + const suffix = uniqueSuffix(); + const password = 'vault voller notizen 12'; + + let ownerId: string; + let pondId: string; + let ownerCookie: string; + let editorCookie: string; + + const api = () => request(app.getHttpServer()); + + async function loginOf(username: string): Promise { + const res = await api() + .post('/api/v1/auth/login') + .send({ usernameOrEmail: username, password }) + .expect(200); + return sessionCookieOf(res); + } + + async function importVault( + options: object, + cookie = ownerCookie, + zip: Buffer = fixtureZip(), + ): Promise { + const enqueued = await api() + .post(`/api/v1/ponds/${pondId}/import/vault`) + .set('Cookie', cookie) + .field('options', JSON.stringify(options)) + .attach('file', zip, 'vault.zip') + .expect(201); + await worker.drain(); + return api().get(`/api/v1/jobs/${enqueued.body.id}`).set('Cookie', cookie).expect(200); + } + + async function pageBySlug(slug: string) { + return prisma.page.findFirst({ where: { pondId, slug } }); + } + + beforeAll(async () => { + prisma = createTestPrisma(); + await prisma.rateLimit.deleteMany({}); + app = await createTestApp(); + worker = app.get(ConversionWorker); + + const users = app.get(UsersService); + const tokens = app.get(AuthTokensService); + const ownerUser = await users.createUser({ + username: `vera-vault-${suffix}`, + email: `vera-vault-${suffix}@example.org`, + displayName: 'Vera Vault', + password, + locale: 'en', + }); + ownerId = ownerUser.id; + const verify = await tokens.issue(ownerUser.id, 'EMAIL_VERIFICATION', 600); + await api().post('/api/v1/auth/verify-email').send({ token: verify }).expect(204); + pondId = (await prisma.pond.findFirstOrThrow({ where: { ownerId, type: 'PERSONAL' } })).id; + + // A pond-wide editor (NOT pond admin) — vault import must refuse them. + // Grant created before any permission resolution warms the pond's cache. + const editorUser = await users.createUser({ + username: `egon-editor-${suffix}`, + email: `egon-editor-${suffix}@example.org`, + displayName: 'Egon Editor', + password, + locale: 'en', + }); + await users.markEmailVerified(editorUser.id); + await prisma.roleGrant.create({ + data: { + pondId, + subjectType: 'USER', + subjectId: editorUser.id, + role: 'EDITOR', + scopeType: 'POND', + effect: 'ALLOW', + createdBy: ownerId, + }, + }); + + ownerCookie = await loginOf(`vera-vault-${suffix}`); + editorCookie = await loginOf(`egon-editor-${suffix}`); + }); + + afterAll(async () => { + await prisma.conversionJob.deleteMany({ where: { ownerId } }); + await prisma.attachment.deleteMany({ where: { pondId } }); + await prisma.roleGrant.deleteMany({ where: { pondId } }); + await prisma.label.deleteMany({ where: { pondId } }); + await prisma.page.deleteMany({ where: { pondId } }); + await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } }); + await prisma.pond.deleteMany({ where: { ownerId } }); + await prisma.user.deleteMany({ where: { username: { contains: suffix } } }); + await prisma.$disconnect(); + await app.close(); + }); + + it('rejects non-admins, garbage archives, and unknown options up front', async () => { + await api() + .post(`/api/v1/ponds/${pondId}/import/vault`) + .set('Cookie', editorCookie) + .attach('file', fixtureZip(), 'vault.zip') + .expect(403); + + const garbage = await api() + .post(`/api/v1/ponds/${pondId}/import/vault`) + .set('Cookie', ownerCookie) + .attach('file', Buffer.from('not a zip'), 'vault.zip') + .expect(400); + expect(garbage.body.code).toBe('import_vault_invalid_zip'); + + await api() + .post(`/api/v1/ponds/${pondId}/import/vault`) + .set('Cookie', ownerCookie) + .field('options', JSON.stringify({ parentPageId: 'missing' })) + .attach('file', fixtureZip(), 'vault.zip') + .expect(404); + + await api() + .post(`/api/v1/ponds/${pondId}/import/vault`) + .set('Cookie', ownerCookie) + .attach('file', Buffer.from('x'), 'vault.rar') + .expect(400); + }); + + it('imports the fixture vault: tree, links, tags, assets, extra label', async () => { + // Pre-existing page that collides with a vault note's slug. + await api() + .post(`/api/v1/ponds/${pondId}/pages`) + .set('Cookie', ownerCookie) + .send({ title: 'Startseite' }) + .expect(201); + const extra = await api() + .post(`/api/v1/ponds/${pondId}/labels`) + .set('Cookie', ownerCookie) + .send({ name: `importiert-${suffix}` }) + .expect(201); + + const mount = await api() + .post(`/api/v1/ponds/${pondId}/pages`) + .set('Cookie', ownerCookie) + .send({ title: `Vault Mount ${suffix}` }) + .expect(201); + + const job = await importVault({ + parentPageId: mount.body.id, + labelIds: [extra.body.id], + frontmatterMode: 'strip', + }); + expect(job.body.status).toBe('succeeded'); + expect(job.body.resultPageId).toBe(mount.body.id); + + // Tree: containers under the mount page. Mount depth 1 leaves exactly 4 + // container levels — the 4-folder chain fits unmerged (the merge itself + // is pinned by the transform's unit tests). + const projekte = await pageBySlug('projekte'); + expect(projekte?.parentId).toBe(mount.body.id); + const ebene4 = await pageBySlug('ebene4'); + const ebene5 = await pageBySlug('ebene5'); + expect(ebene5?.parentId).toBe(ebene4?.id); + const deepNote = await pageBySlug('tiefe-notiz'); + expect(deepNote?.parentId).toBe(ebene5?.id); + + // Slug collision with the pre-existing page → suffix; the vault-internal + // link follows the FINAL slug. + const start = await pageBySlug('startseite-2'); + expect(start).toBeTruthy(); + const startCache = await prisma.pageContentCache.findUniqueOrThrow({ + where: { pageId: start!.id }, + }); + expect(startCache.markdown).toContain('[[projekt-a|Projekt A]]'); + + // Outgoing links are indexed at creation: resolved edge + phantom. + const links = await prisma.pageLink.findMany({ where: { fromPageId: start!.id } }); + const projektA = await pageBySlug('projekt-a'); + expect(links.find((l) => l.targetSlug === 'projekt-a')?.toPageId).toBe(projektA!.id); + expect(links.find((l) => l.targetSlug === 'fehlt-noch')?.toPageId).toBeNull(); + + // Tags → labels: nested #status/aktiv becomes a hierarchy; the extra + // dialog label is on every imported page. + const status = await prisma.label.findFirst({ where: { pondId, name: 'status' } }); + const aktiv = await prisma.label.findFirst({ where: { pondId, name: 'aktiv' } }); + expect(aktiv?.parentId).toBe(status?.id); + const projektALabels = await prisma.pageLabel.findMany({ + where: { pageId: projektA!.id }, + include: { label: true }, + }); + const names = projektALabels.map((row) => row.label.name).sort(); + expect(names).toEqual(expect.arrayContaining(['aktiv', 'projekt', `importiert-${suffix}`])); + const startLabels = await prisma.pageLabel.findMany({ + where: { pageId: start!.id }, + include: { label: true }, + }); + expect(startLabels.map((row) => row.label.name)).toEqual( + expect.arrayContaining(['willkommen', `importiert-${suffix}`]), + ); + + // Frontmatter stripped; inline tag gone from the text. + const aCache = await prisma.pageContentCache.findUniqueOrThrow({ + where: { pageId: projektA!.id }, + }); + expect(aCache.markdown).not.toContain('tags:'); + expect(aCache.markdown).not.toContain('#status'); + + // The image became a pond file embedded in the page; the PDF an attachment. + const attachments = await prisma.attachment.findMany({ where: { pondId } }); + expect(attachments.some((a) => a.fileName === 'teich.png')).toBe(true); + const pdf = attachments.find((a) => a.fileName === 'unterlagen.pdf'); + expect(pdf?.pageId).toBe((await pageBySlug('projekt-b'))!.id); + expect(aCache.markdown).toMatch(/!\[[^\]]*\]\([0-9a-f-]{36}\)/); + }); + + it('rolls back everything when the pond runs out of storage', async () => { + const before = { + pages: await prisma.page.count({ where: { pondId } }), + files: await prisma.attachment.count({ where: { pondId } }), + labels: await prisma.label.count({ where: { pondId } }), + }; + // Storage quota 1 byte → the first asset upload fails mid-import. + await prisma.quotaOverride.create({ + data: { subjectType: 'POND', subjectId: pondId, quotaKey: 'storage_bytes', value: 1 }, + }); + try { + const job = await importVault({ frontmatterMode: 'strip' }); + expect(job.body.status).toBe('failed'); + expect(job.body.errorCode).toBe('quota_exceeded'); + + expect(await prisma.page.count({ where: { pondId } })).toBe(before.pages); + expect(await prisma.attachment.count({ where: { pondId } })).toBe(before.files); + } finally { + await prisma.quotaOverride.deleteMany({ + where: { subjectId: pondId, quotaKey: 'storage_bytes' }, + }); + } + }); + + it('re-imports cleanly after a failure (no duplicates, fresh suffixes)', async () => { + const job = await importVault({ frontmatterMode: 'strip' }); + expect(job.body.status).toBe('succeeded'); + // The second import suffixes against the first one's slugs. + expect(await pageBySlug('projekt-a-2')).toBeTruthy(); + }); +}); diff --git a/apps/api/src/import-export/import.controller.ts b/apps/api/src/import-export/import.controller.ts index 185cb26..d7d4160 100644 --- a/apps/api/src/import-export/import.controller.ts +++ b/apps/api/src/import-export/import.controller.ts @@ -1,5 +1,6 @@ import { BadRequestException, + Body, Controller, Param, Post, @@ -8,9 +9,14 @@ import { UseInterceptors, } from '@nestjs/common'; import { FileInterceptor } from '@nestjs/platform-express'; -import { ConversionJobView, MAX_UPLOAD_PARSE_BYTES } from '@dorfteich/shared'; +import { + ConversionJobView, + MAX_UPLOAD_PARSE_BYTES, + importVaultOptionsSchema, +} from '@dorfteich/shared'; import { AuthedRequest } from '../auth/auth.guard'; +import { ZodValidationPipe } from '../common/zod-validation.pipe'; import { RequiresPondRole } from '../permissions/permission.decorators'; import { ImportService } from './import.service'; @@ -36,4 +42,33 @@ export class ImportController { if (!file) throw new BadRequestException({ code: 'bad_request' }); return this.imports.enqueue(request.user!, pondId, file); } + + /** + * Obsidian vault import (issue #117): a `.zip` of the whole vault plus a + * JSON `options` field ({parentPageId?, labelIds?, frontmatterMode}) in the + * same multipart form. Pond-admin-gated — a vault import creates a whole + * subtree, uploads files, and creates labels, which is pond administration + * rather than everyday editing. + */ + @Post('import/vault') + @RequiresPondRole('pond_admin', { idParam: 'pondId' }) + @UseInterceptors(FileInterceptor('file', { limits: { fileSize: MAX_UPLOAD_PARSE_BYTES } })) + async importVault( + @Param('pondId') pondId: string, + @UploadedFile() file: Express.Multer.File | undefined, + @Body() body: { options?: string }, + @Req() request: AuthedRequest, + ): Promise { + if (!file) throw new BadRequestException({ code: 'bad_request' }); + let raw: unknown = {}; + if (body.options) { + try { + raw = JSON.parse(body.options); + } catch { + throw new BadRequestException({ code: 'bad_request' }); + } + } + const options = new ZodValidationPipe(importVaultOptionsSchema).transform(raw); + return this.imports.enqueueVault(request.user!, pondId, file, options); + } } diff --git a/apps/api/src/import-export/import.service.ts b/apps/api/src/import-export/import.service.ts index c1d2555..068fd5d 100644 --- a/apps/api/src/import-export/import.service.ts +++ b/apps/api/src/import-export/import.service.ts @@ -1,15 +1,37 @@ -import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/common'; +import { + BadRequestException, + ForbiddenException, + Injectable, + NotFoundException, +} from '@nestjs/common'; import { ConversionJob, Page, User } from '@prisma/client'; -import { ConversionJobView, editorSchema, markdownToDoc } from '@dorfteich/shared'; +import { + ConversionJobView, + ImportVaultOptions, + MAX_LABEL_DEPTH, + MAX_UPLOAD_PARSE_BYTES, + editorSchema, + importVaultOptionsSchema, + markdownToDoc, + nodeDepth, +} from '@dorfteich/shared'; import { Node } from 'prosemirror-model'; import { PinoLogger } from 'nestjs-pino'; import { FilesService } from '../files/files.service'; +import { LabelsService } from '../labels/labels.service'; import { PagesService } from '../pages/pages.service'; -import { docToState } from '../pages/yjs-content'; +import { docToState, emptyPageState } from '../pages/yjs-content'; import { PrismaService } from '../prisma/prisma.service'; import { ImportProcessor } from './import.constants'; +import { + ASSET_PLACEHOLDER_PREFIX, + VaultError, + VaultImportPlan, + parseVaultZip, + planVaultImport, +} from './obsidian-vault'; import { ConversionError, PandocConverter } from './pandoc.converter'; import { ConversionJobService } from './conversion-job.service'; @@ -108,6 +130,7 @@ export class ImportService implements ImportProcessor { private readonly converter: PandocConverter, private readonly files: FilesService, private readonly pages: PagesService, + private readonly labels: LabelsService, private readonly logger: PinoLogger, ) { this.logger.setContext(ImportService.name); @@ -180,6 +203,51 @@ export class ImportService implements ImportProcessor { } } + /** + * Validate and enqueue an Obsidian vault import (issue #117). The ZIP is + * parsed once here for fast feedback (an invalid archive 400s instead of + * failing a job later); the worker re-parses when the job runs. The mount + * parent is validated up front too, and again at run time. + */ + async enqueueVault( + user: User, + pondId: string, + file: { buffer: Buffer; originalname: string }, + options: ImportVaultOptions, + ): Promise { + if (fileExtension(file.originalname) !== 'zip') { + throw new BadRequestException({ code: 'import_unsupported_format' }); + } + try { + parseVaultZip(new Uint8Array(file.buffer)); + } catch (error) { + if (error instanceof VaultError) { + throw new BadRequestException({ code: error.code }); + } + throw error; + } + if (options.parentPageId) await this.requireLivePageInPond(pondId, options.parentPageId); + for (const labelId of options.labelIds) { + const label = await this.prisma.label.findFirst({ where: { id: labelId, pondId } }); + if (!label) throw new NotFoundException(); + } + + const job = await this.jobs.enqueue({ + ownerId: user.id, + pondId, + kind: 'import_vault', + from: 'zip', + to: 'pages', + input: file.buffer, + sourceName: file.originalname, + standalone: false, + options, + // The 25 MiB default guards the pandoc sidecar; a vault never goes there. + maxInputBytes: MAX_UPLOAD_PARSE_BYTES, + }); + return this.jobs.viewOf(job); + } + /** * Run one import job to completion (called by the worker). Throws a * {@link ConversionError} on failure so the worker applies the queue's @@ -195,6 +263,9 @@ export class ImportService implements ImportProcessor { if (!user) { throw new ConversionError('conversion_failed', false, 'import job owner is gone'); } + if (job.kind === 'import_vault') { + return this.runVault(job, user, job.pondId); + } const rawMarkdown = await convertImportedDocument( this.converter, @@ -212,6 +283,251 @@ export class ImportService implements ImportProcessor { ); } + /** + * Run a vault import job (issue #117): transform (issue #116) → containers + * top-down → notes → assets → labels, ALL-OR-NOTHING. On any failure every + * created page is hard-deleted (children before parents) and every stored + * file removed (quota restored), so the worker's retry policy is safe and a + * failed import leaves no trace. + */ + private async runVault(job: ConversionJob, user: User, pondId: string): Promise { + const options = importVaultOptionsSchema.parse(job.options ?? {}); + + // Re-validate the mount parent (it may have been trashed since enqueue) + // and compute its depth for the folder budget. + let mountDepth = 0; + if (options.parentPageId) { + const live = await this.prisma.page.findFirst({ + where: { id: options.parentPageId, pondId, deletedAt: null }, + select: { id: true }, + }); + if (!live) { + throw new ConversionError('conversion_failed', false, 'vault mount parent is gone'); + } + const tree = await this.prisma.page.findMany({ + where: { pondId, deletedAt: null }, + select: { id: true, parentId: true }, + }); + mountDepth = nodeDepth(tree, options.parentPageId); + } + + // Slugs are unique across live AND trashed pages (the unique index does + // not care about deletedAt), so the reservation set includes everything. + const existing = await this.prisma.page.findMany({ + where: { pondId }, + select: { slug: true }, + }); + + let plan: VaultImportPlan; + let assetBytes: Map; + try { + const zip = new Uint8Array(job.input); + plan = planVaultImport(zip, { + frontmatterMode: options.frontmatterMode, + existingSlugs: new Set(existing.map((row) => row.slug)), + mountDepth, + }); + const vault = parseVaultZip(zip); + assetBytes = new Map( + vault.assets.map((asset) => [asset.path, { data: asset.data, name: asset.name }]), + ); + } catch (error) { + if (error instanceof VaultError) { + throw new ConversionError(error.code, false, error.message); + } + throw error; + } + + const createdPageIds: string[] = []; + const storedFileIds: string[] = []; + try { + // Container pages top-down (parents exist before their children). + const pageIdByKey = new Map(); + const parentOf = (key: string | null): string | null => + key ? pageIdByKey.get(key)! : (options.parentPageId ?? null); + for (const container of plan.containers) { + const page = await this.pages.createWithState( + user, + pondId, + container.title, + emptyPageState(), + parentOf(container.parentKey), + container.slug, + ); + createdPageIds.push(page.id); + pageIdByKey.set(container.key, page.id); + } + + // Referenced assets, uploaded once; the attachment row links to the + // first page that references the asset (done per note below). + const fileIdByAsset = new Map(); + for (const path of plan.referencedAssets) { + const asset = assetBytes.get(path); + if (!asset) continue; + const view = await this.uploadVaultAsset(user, pondId, asset, storedFileIds); + if (view) fileIdByAsset.set(path, view); + } + + // Notes: replace asset placeholders, parse, create, link, label. + const labelIdByPath = new Map(); + const linkedAssets = new Set(); + for (const note of plan.notes) { + const placeholder = new RegExp( + `!\\[([^\\]]*)\\]\\(${ASSET_PLACEHOLDER_PREFIX}([^)\\s]+)\\)`, + 'g', + ); + const markdown = note.markdown.replaceAll( + placeholder, + (_whole, alt: string, path: string) => { + const fileId = fileIdByAsset.get(path); + return fileId ? `![${alt}](${fileId})` : ''; + }, + ); + const json = markdownToDoc(markdown).toJSON() as unknown as PmNode; + const state = docToState(Node.fromJSON(editorSchema, json)); + const page = await this.pages.createWithState( + user, + pondId, + note.title, + state, + parentOf(note.parentKey), + note.slug, + ); + createdPageIds.push(page.id); + + const noteAssets = [...note.imageAssets, ...note.attachmentAssets] + .filter((path) => !linkedAssets.has(path)) + .map((path) => fileIdByAsset.get(path)) + .filter((id): id is string => Boolean(id)); + for (const path of [...note.imageAssets, ...note.attachmentAssets]) { + linkedAssets.add(path); + } + await this.files.linkAttachmentsToPage(noteAssets, page.id); + + for (const tag of note.tags) { + const labelId = await this.ensureLabelPath(user, pondId, tag, labelIdByPath); + if (labelId) await this.labels.assign(user, page.id, labelId); + } + for (const labelId of options.labelIds) { + await this.labels.assign(user, page.id, labelId); + } + } + + await this.prisma.conversionJob.update({ + where: { id: job.id }, + data: { + status: 'SUCCEEDED', + resultPageId: options.parentPageId ?? null, + errorCode: null, + }, + }); + this.logger.info( + { jobId: job.id, pondId, pages: createdPageIds.length, files: storedFileIds.length }, + 'audit: vault imported', + ); + } catch (error) { + await this.rollbackPages(createdPageIds); + await this.rollbackMedia(user, storedFileIds); + if (error instanceof ForbiddenException && errorCode(error) === 'quota_exceeded') { + throw new ConversionError('quota_exceeded', false, 'pond storage exhausted during import'); + } + if (error instanceof ConversionError || error instanceof VaultError) { + throw error instanceof VaultError + ? new ConversionError(error.code, false, error.message) + : error; + } + throw new ConversionError( + 'conversion_failed', + true, + error instanceof Error ? error.message : 'vault import failed', + ); + } + } + + /** Uploads one vault asset as a pond file; a type the instance rejects is + * dropped (structure over layout), an exhausted pond fails the import. */ + private async uploadVaultAsset( + user: User, + pondId: string, + asset: { data: Uint8Array; name: string }, + storedFileIds: string[], + ): Promise { + try { + const view = await this.files.upload(user, pondId, { + buffer: Buffer.from(asset.data), + size: asset.data.length, + originalname: asset.name, + }); + storedFileIds.push(view.id); + return view.id; + } catch (error) { + if (error instanceof ForbiddenException && errorCode(error) === 'quota_exceeded') { + throw error; + } + this.logger.warn({ pondId, code: errorCode(error) }, 'vault import: dropped an asset'); + return null; + } + } + + /** + * Find-or-create the label chain for a nested tag (`a/b` → label `a` with + * child `b`), returning the deepest label's id. Creation goes through + * LabelsService so locking and permission-cache invalidation apply; chains + * deeper than the label limit are clamped to it. + */ + private async ensureLabelPath( + user: User, + pondId: string, + tag: string[], + cache: Map, + ): Promise { + const path = tag.slice(0, MAX_LABEL_DEPTH); + let parentId: string | null = null; + let key = ''; + for (const name of path) { + key = key ? `${key}/${name}` : name; + const cached = cache.get(key); + if (cached) { + parentId = cached; + continue; + } + const found: { id: string } | null = await this.prisma.label.findFirst({ + where: { pondId, parentId, name }, + select: { id: true }, + }); + const id: string = found + ? found.id + : (await this.labels.create(user, pondId, { name, parentId })).id; + cache.set(key, id); + parentId = id; + } + return parentId; + } + + /** Hard-deletes the pages a failed vault import created — children before + * parents (creation order reversed), attachments already gone via the media + * rollback, everything else cascades. */ + private async rollbackPages(pageIds: string[]): Promise { + for (const id of [...pageIds].reverse()) { + try { + await this.prisma.page.delete({ where: { id } }); + } catch (error) { + this.logger.warn( + { pageId: id, error: error instanceof Error ? error.message : String(error) }, + 'vault import: page rollback failed', + ); + } + } + } + + private async requireLivePageInPond(pondId: string, pageId: string): Promise { + const page = await this.prisma.page.findFirst({ + where: { id: pageId, pondId, deletedAt: null }, + select: { id: true }, + }); + if (!page) throw new NotFoundException(); + } + /** * The shared tail of every import: store embedded images, parse the Markdown * into an editor document, and create a page from it. Media stored during a diff --git a/apps/api/src/import-export/pandoc.converter.ts b/apps/api/src/import-export/pandoc.converter.ts index 8993436..83a2dc4 100644 --- a/apps/api/src/import-export/pandoc.converter.ts +++ b/apps/api/src/import-export/pandoc.converter.ts @@ -30,7 +30,10 @@ export type ConversionErrorCode = | 'conversion_failed' // The pond ran out of storage while an import stored the document's media // (#63) — final, and surfaced the same way through the worker. - | 'quota_exceeded'; + | 'quota_exceeded' + // Vault import (#117): rejected archive / unpacked ceiling exceeded. + | 'import_vault_invalid_zip' + | 'import_vault_too_large'; /** A conversion failure with a stable, localizable code. `retryable` marks * the transient causes (sidecar down / timed out) the worker retries before diff --git a/apps/api/src/pages/pages.service.ts b/apps/api/src/pages/pages.service.ts index 295f7d8..7b80284 100644 --- a/apps/api/src/pages/pages.service.ts +++ b/apps/api/src/pages/pages.service.ts @@ -225,8 +225,9 @@ export class PagesService { title: string, state: Uint8Array, parentId: string | null = null, + presetSlug?: string, ): Promise { - return this.insertPage(user, pondId, title, state, parentId); + return this.insertPage(user, pondId, title, state, parentId, presetSlug); } /** @@ -250,12 +251,15 @@ export class PagesService { title: string, state: Uint8Array, parentId: string | null = null, + presetSlug?: string, ): Promise { const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } }); if (!pond) throw new NotFoundException(); if (parentId) this.assertValidParent(await this.livePageTree(pond.id), parentId, null); - const slug = await this.generateUniqueSlugInPond(pond.id, title); + // A batch import (#117) reserves its slugs up front against the pond + // snapshot ∪ batch — the unique index stays the final arbiter. + const slug = presetSlug ?? (await this.generateUniqueSlugInPond(pond.id, title)); const last = await this.prisma.page.findFirst({ where: { pondId: pond.id }, orderBy: { sortKey: 'desc' }, @@ -278,12 +282,35 @@ export class PagesService { }); // A new page may satisfy phantom wikilinks that referenced its slug (#47). await this.resolvePhantomLinks(pond.id, slug, page.id); + // Seed the page's own outgoing links (issue #117): pages born with + // content (imports) would otherwise stay invisible to backlinks and the + // graph until their first collab save — collab rewrites these rows on + // every later save, exactly as it does for hand-typed links. + if (content.wikilinkSlugs.length > 0) { + await this.indexOutgoingLinks(pond.id, page.id, content.wikilinkSlugs); + } // Index the page so a title-only match is findable immediately (#49). await this.search.indexPage(page.id); this.logger.info({ pageId: page.id, pondId: pond.id, userId: user.id }, 'audit: page created'); return page; } + /** + * Insert the outgoing `page_links` rows of a freshly created page (issue + * #117), resolving each target slug against the pond's live pages — the + * same shape the collab persistence writes on every save (#47). Insert-only: + * a brand-new page has no rows to replace. + */ + private async indexOutgoingLinks(pondId: string, pageId: string, slugs: string[]): Promise { + await this.prisma.$executeRaw` + INSERT INTO page_links (id, from_page_id, to_page_id, target_slug) + SELECT gen_random_uuid(), ${pageId}, target.id, s.link_slug + FROM unnest(${slugs}::text[]) AS s(link_slug) + LEFT JOIN pages AS target + ON target.pond_id = ${pondId} AND target.slug = s.link_slug + AND target.deleted_at IS NULL`; + } + /** * Point phantom `page_links` (unresolved, `to_page_id` null) whose * `target_slug` matches `slug` within the pond at `pageId` (issue #47). diff --git a/apps/api/src/pages/yjs-content.ts b/apps/api/src/pages/yjs-content.ts index cc8dd4e..bc8d2b2 100644 --- a/apps/api/src/pages/yjs-content.ts +++ b/apps/api/src/pages/yjs-content.ts @@ -5,6 +5,7 @@ import { editorSchema, extractOutline, OutlineEntry, + extractWikilinkSlugs, } from '@dorfteich/shared'; import { Node } from 'prosemirror-model'; import { prosemirrorJSONToYXmlFragment, yXmlFragmentToProseMirrorRootNode } from 'y-prosemirror'; @@ -68,6 +69,10 @@ export interface DerivedPageContent { * embeds the file, which is what the trash-purge job uses to find a * purged page's files. */ imageFileIds: string[]; + /** Outgoing wikilink target slugs (issue #117): pages created through the + * api (imports, phantom-create) seed their `page_links` rows from this — + * collab, the content writer, rewrites them on every later save. */ + wikilinkSlugs: string[]; } function imageFileIdsOf(doc: Node): string[] { @@ -93,5 +98,6 @@ export function deriveContent(state: Uint8Array): DerivedPageContent { html: docToHtml(doc), outline: extractOutline(doc), imageFileIds: imageFileIdsOf(doc), + wikilinkSlugs: extractWikilinkSlugs(doc), }; } diff --git a/packages/shared/i18n/de/errors.json b/packages/shared/i18n/de/errors.json index 152d142..834a6fd 100644 --- a/packages/shared/i18n/de/errors.json +++ b/packages/shared/i18n/de/errors.json @@ -39,6 +39,8 @@ "renderer_unavailable": "Der PDF-Renderer ist derzeit nicht verfügbar. Bitte versuche es später erneut.", "render_failed": "Diese Seite konnte nicht als PDF gerendert werden.", "import_unsupported_format": "Nur Word- (.docx) und OpenDocument-Dokumente (.odt) können importiert werden.", + "import_vault_invalid_zip": "Das Archiv ist kein gültiger Obsidian-Vault (ZIP mit Markdown-Notizen).", + "import_vault_too_large": "Der entpackte Vault ist zu groß für einen Import.", "network": "Der Server war nicht erreichbar.", "grant_exists": "Diese Berechtigung existiert bereits.", "plugin_invalid_zip": "Die hochgeladene Datei ist kein gültiges ZIP-Archiv.", diff --git a/packages/shared/i18n/en/errors.json b/packages/shared/i18n/en/errors.json index c8b9b2b..0e84f57 100644 --- a/packages/shared/i18n/en/errors.json +++ b/packages/shared/i18n/en/errors.json @@ -39,6 +39,8 @@ "renderer_unavailable": "The PDF renderer is currently unavailable. Please try again later.", "render_failed": "This page could not be rendered as a PDF.", "import_unsupported_format": "Only Word (.docx) and OpenDocument (.odt) documents can be imported.", + "import_vault_invalid_zip": "The archive is not a valid Obsidian vault (ZIP with Markdown notes).", + "import_vault_too_large": "The unpacked vault is too large for an import.", "network": "The server could not be reached.", "grant_exists": "This grant already exists.", "plugin_invalid_zip": "The uploaded file is not a valid ZIP archive.", diff --git a/packages/shared/src/conversion.ts b/packages/shared/src/conversion.ts index 6bc1d9d..d829a43 100644 --- a/packages/shared/src/conversion.ts +++ b/packages/shared/src/conversion.ts @@ -35,6 +35,24 @@ export interface ConversionJobView { export const IMPORT_EXTENSIONS = ['docx', 'odt', 'md', 'markdown'] as const; export type ImportExtension = (typeof IMPORT_EXTENSIONS)[number]; +/** What happens to a note's YAML frontmatter on vault import (issue #117): + * dropped, or preserved as a yaml code block at the top of the page. */ +export const VAULT_FRONTMATTER_MODES = ['strip', 'preserve'] as const; +export type VaultFrontmatterMode = (typeof VAULT_FRONTMATTER_MODES)[number]; + +/** + * Options of an Obsidian vault import (issue #117), sent as a JSON string in + * the multipart `options` field and stored on the job row. `parentPageId` + * mounts the vault under an existing page (null/absent = the pond root); + * `labelIds` are assigned to every imported page on top of the tag labels. + */ +export const importVaultOptionsSchema = z.object({ + parentPageId: z.string().min(1).nullish(), + labelIds: z.array(z.string().min(1)).max(20).default([]), + frontmatterMode: z.enum(VAULT_FRONTMATTER_MODES).default('strip'), +}); +export type ImportVaultOptions = z.infer; + /** Formats a single page exports to via a conversion job whose result is * downloaded from `GET /jobs/:id/result` (issues #65/#67). `docx`/`odt` run * `markdown → pandoc`; `pdf` renders HTML through Gotenberg. Markdown export is