Obsidian vault import: endpoint, job orchestration, rollback (#117)
Some checks failed
CD / Build and push images (push) Successful in 4m5s
CD / Deploy to Test (push) Successful in 9s
CI / Lint, typecheck, test (push) Failing after 4m21s
CI / Auth e2e pack (push) Has been skipped
CI / Import/export fidelity gate (push) Has been skipped
CI / Build container images (push) Has been skipped
CD / Smoke tests against Test (push) Successful in 1m22s
CD / Promote to Int (push) Successful in 11s
Some checks failed
CD / Build and push images (push) Successful in 4m5s
CD / Deploy to Test (push) Successful in 9s
CI / Lint, typecheck, test (push) Failing after 4m21s
CI / Auth e2e pack (push) Has been skipped
CI / Import/export fidelity gate (push) Has been skipped
CI / Build container images (push) Has been skipped
CD / Smoke tests against Test (push) Successful in 1m22s
CD / Promote to Int (push) Successful in 11s
POST /ponds/:pondId/import/vault (pond-admin-gated; a vault import
creates a subtree, uploads files, and creates labels — administration,
not everyday editing) takes the ZIP plus a JSON options field
{parentPageId?, labelIds?, frontmatterMode}. The archive is parsed at
enqueue for fast 400s; the job (new kind import_vault, riding the
existing isImportKind worker routing) re-parses and runs the #116
transform, then: containers top-down → notes (asset placeholders →
uploaded pond files; non-images become page attachments) → tags to
labels (nested tags build a label hierarchy via LabelsService, so
locking and cache invalidation apply) plus the dialog labels.
All-or-nothing: any failure hard-deletes the created pages (children
first) and removes the stored files (quota restored), then surfaces as
import_vault_invalid_zip / import_vault_too_large / quota_exceeded /
conversion_failed — and makes the worker's retry policy safe.
Supporting changes:
- conversion_jobs gains a nullable options jsonb column; enqueue takes
kind-specific options and a maxInputBytes override (the 25 MiB
default protects the pandoc sidecar, which a vault never touches —
vaults use the 64 MiB upload limit).
- insertPage accepts a pre-reserved slug (the batch reserves all slugs
up front against pond ∪ batch).
- NEW: pages born with content seed their outgoing page_links rows
(deriveContent now returns wikilinkSlugs) — imported pages would
otherwise stay invisible to backlinks and the graph until their
first collab save. Collab still rewrites the rows on every save, and
the existing phantom resolution heals batch creation order.
import-vault.e2e.db.test.ts (4 tests, real worker drained): gating +
input rejection, the full fixture import (tree under a mount page,
collision suffixes, link rows incl. phantom, nested tag labels, extra
label everywhere, frontmatter stripped, image embedded + PDF attached),
complete quota rollback, and a clean re-import with fresh suffixes.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
269b36c761
commit
8ae010218e
@ -0,0 +1,2 @@
|
|||||||
|
-- AlterTable
|
||||||
|
ALTER TABLE "conversion_jobs" ADD COLUMN "options" JSONB;
|
||||||
@ -705,6 +705,9 @@ model ConversionJob {
|
|||||||
/// downloadable and is purged (GDPR data minimization). Null for every
|
/// downloadable and is purged (GDPR data minimization). Null for every
|
||||||
/// other job kind, whose result never expires.
|
/// other job kind, whose result never expires.
|
||||||
expiresAt DateTime? @map("expires_at")
|
expiresAt DateTime? @map("expires_at")
|
||||||
|
/// Kind-specific job options (issue #117): a vault import carries
|
||||||
|
/// `{parentPageId, labelIds, frontmatterMode}`. Null for other kinds.
|
||||||
|
options Json?
|
||||||
createdAt DateTime @default(now()) @map("created_at")
|
createdAt DateTime @default(now()) @map("created_at")
|
||||||
updatedAt DateTime @updatedAt @map("updated_at")
|
updatedAt DateTime @updatedAt @map("updated_at")
|
||||||
|
|
||||||
|
|||||||
@ -19,6 +19,12 @@ export interface EnqueueConversion {
|
|||||||
* original upload file name (a title fallback). */
|
* original upload file name (a title fallback). */
|
||||||
pondId?: string;
|
pondId?: string;
|
||||||
sourceName?: string;
|
sourceName?: string;
|
||||||
|
/** Kind-specific options persisted with the job (issue #117). */
|
||||||
|
options?: unknown;
|
||||||
|
/** Input ceiling override (issue #117): the 25 MiB default protects the
|
||||||
|
* pandoc sidecar; a vault import never touches it and may use the full
|
||||||
|
* upload limit instead. */
|
||||||
|
maxInputBytes?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ConversionResultPayload {
|
export interface ConversionResultPayload {
|
||||||
@ -50,10 +56,11 @@ export class ConversionJobService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async enqueue(request: EnqueueConversion): Promise<ConversionJob> {
|
async enqueue(request: EnqueueConversion): Promise<ConversionJob> {
|
||||||
if (request.input.byteLength > MAX_CONVERSION_INPUT_BYTES) {
|
const maxInputBytes = request.maxInputBytes ?? MAX_CONVERSION_INPUT_BYTES;
|
||||||
|
if (request.input.byteLength > maxInputBytes) {
|
||||||
throw new PayloadTooLargeException({
|
throw new PayloadTooLargeException({
|
||||||
code: 'file_too_large',
|
code: 'file_too_large',
|
||||||
details: { limitBytes: MAX_CONVERSION_INPUT_BYTES },
|
details: { limitBytes: maxInputBytes },
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
const job = await this.prisma.conversionJob.create({
|
const job = await this.prisma.conversionJob.create({
|
||||||
@ -65,6 +72,7 @@ export class ConversionJobService {
|
|||||||
sourceFormat: request.from,
|
sourceFormat: request.from,
|
||||||
targetFormat: request.to,
|
targetFormat: request.to,
|
||||||
standalone: request.standalone ?? true,
|
standalone: request.standalone ?? true,
|
||||||
|
options: request.options === undefined ? undefined : (request.options as object),
|
||||||
// Prisma's Bytes maps to Uint8Array<ArrayBuffer>; a Node Buffer's
|
// Prisma's Bytes maps to Uint8Array<ArrayBuffer>; a Node Buffer's
|
||||||
// backing store is ArrayBufferLike, so copy into a plain Uint8Array.
|
// backing store is ArrayBufferLike, so copy into a plain Uint8Array.
|
||||||
input: new Uint8Array(request.input),
|
input: new Uint8Array(request.input),
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
import { Module, OnModuleInit } from '@nestjs/common';
|
import { Module, OnModuleInit } from '@nestjs/common';
|
||||||
|
|
||||||
import { FilesModule } from '../files/files.module';
|
import { FilesModule } from '../files/files.module';
|
||||||
|
import { LabelsModule } from '../labels/labels.module';
|
||||||
import { PagesModule } from '../pages/pages.module';
|
import { PagesModule } from '../pages/pages.module';
|
||||||
import { PluginsModule } from '../plugins/plugins.module';
|
import { PluginsModule } from '../plugins/plugins.module';
|
||||||
import { SchedulerModule } from '../scheduler/scheduler.module';
|
import { SchedulerModule } from '../scheduler/scheduler.module';
|
||||||
@ -30,7 +31,7 @@ const EXPORT_PURGE_CADENCE_SECONDS = 60 * 60;
|
|||||||
* feature exports (#65/#67), and the GDPR account data export (#68).
|
* feature exports (#65/#67), and the GDPR account data export (#68).
|
||||||
*/
|
*/
|
||||||
@Module({
|
@Module({
|
||||||
imports: [FilesModule, PagesModule, PluginsModule, SchedulerModule],
|
imports: [FilesModule, LabelsModule, PagesModule, PluginsModule, SchedulerModule],
|
||||||
controllers: [JobsController, ImportController, ExportController, DataExportController],
|
controllers: [JobsController, ImportController, ExportController, DataExportController],
|
||||||
providers: [
|
providers: [
|
||||||
ConversionJobService,
|
ConversionJobService,
|
||||||
|
|||||||
284
apps/api/src/import-export/import-vault.e2e.db.test.ts
Normal file
284
apps/api/src/import-export/import-vault.e2e.db.test.ts
Normal file
@ -0,0 +1,284 @@
|
|||||||
|
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
||||||
|
import { join, relative } from 'node:path';
|
||||||
|
|
||||||
|
import { INestApplication } from '@nestjs/common';
|
||||||
|
import { PrismaClient } from '@prisma/client';
|
||||||
|
import { zipSync } from 'fflate';
|
||||||
|
import request from 'supertest';
|
||||||
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { AuthTokensService } from '../auth/auth-tokens.service';
|
||||||
|
import { createTestApp, sessionCookieOf } from '../testing/test-app';
|
||||||
|
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
|
||||||
|
import { UsersService } from '../users/users.service';
|
||||||
|
|
||||||
|
import { ConversionWorker } from './conversion-worker.service';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Obsidian vault import (issue #117): endpoint gating, the whole fixture
|
||||||
|
* import (tree, slugs, links, labels, assets), collision handling, quota
|
||||||
|
* rollback, and the size ceilings. Runs the real worker (drained per test)
|
||||||
|
* against the checked-in fixture vault.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const FIXTURE_DIR = join(__dirname, '../../../../fixtures/import/obsidian-vault');
|
||||||
|
|
||||||
|
function fixtureZip(): Buffer {
|
||||||
|
const entries: Record<string, Uint8Array> = {};
|
||||||
|
const walk = (dir: string): void => {
|
||||||
|
for (const name of readdirSync(dir)) {
|
||||||
|
const path = join(dir, name);
|
||||||
|
if (statSync(path).isDirectory()) walk(path);
|
||||||
|
else entries[relative(FIXTURE_DIR, path)] = new Uint8Array(readFileSync(path));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
walk(FIXTURE_DIR);
|
||||||
|
return Buffer.from(zipSync(entries));
|
||||||
|
}
|
||||||
|
|
||||||
|
describe.skipIf(!hasTestDb)('vault import (e2e, issue #117)', () => {
|
||||||
|
let app: INestApplication;
|
||||||
|
let prisma: PrismaClient;
|
||||||
|
let worker: ConversionWorker;
|
||||||
|
const suffix = uniqueSuffix();
|
||||||
|
const password = 'vault voller notizen 12';
|
||||||
|
|
||||||
|
let ownerId: string;
|
||||||
|
let pondId: string;
|
||||||
|
let ownerCookie: string;
|
||||||
|
let editorCookie: string;
|
||||||
|
|
||||||
|
const api = () => request(app.getHttpServer());
|
||||||
|
|
||||||
|
async function loginOf(username: string): Promise<string> {
|
||||||
|
const res = await api()
|
||||||
|
.post('/api/v1/auth/login')
|
||||||
|
.send({ usernameOrEmail: username, password })
|
||||||
|
.expect(200);
|
||||||
|
return sessionCookieOf(res);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function importVault(
|
||||||
|
options: object,
|
||||||
|
cookie = ownerCookie,
|
||||||
|
zip: Buffer = fixtureZip(),
|
||||||
|
): Promise<request.Response> {
|
||||||
|
const enqueued = await api()
|
||||||
|
.post(`/api/v1/ponds/${pondId}/import/vault`)
|
||||||
|
.set('Cookie', cookie)
|
||||||
|
.field('options', JSON.stringify(options))
|
||||||
|
.attach('file', zip, 'vault.zip')
|
||||||
|
.expect(201);
|
||||||
|
await worker.drain();
|
||||||
|
return api().get(`/api/v1/jobs/${enqueued.body.id}`).set('Cookie', cookie).expect(200);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function pageBySlug(slug: string) {
|
||||||
|
return prisma.page.findFirst({ where: { pondId, slug } });
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
prisma = createTestPrisma();
|
||||||
|
await prisma.rateLimit.deleteMany({});
|
||||||
|
app = await createTestApp();
|
||||||
|
worker = app.get(ConversionWorker);
|
||||||
|
|
||||||
|
const users = app.get(UsersService);
|
||||||
|
const tokens = app.get(AuthTokensService);
|
||||||
|
const ownerUser = await users.createUser({
|
||||||
|
username: `vera-vault-${suffix}`,
|
||||||
|
email: `vera-vault-${suffix}@example.org`,
|
||||||
|
displayName: 'Vera Vault',
|
||||||
|
password,
|
||||||
|
locale: 'en',
|
||||||
|
});
|
||||||
|
ownerId = ownerUser.id;
|
||||||
|
const verify = await tokens.issue(ownerUser.id, 'EMAIL_VERIFICATION', 600);
|
||||||
|
await api().post('/api/v1/auth/verify-email').send({ token: verify }).expect(204);
|
||||||
|
pondId = (await prisma.pond.findFirstOrThrow({ where: { ownerId, type: 'PERSONAL' } })).id;
|
||||||
|
|
||||||
|
// A pond-wide editor (NOT pond admin) — vault import must refuse them.
|
||||||
|
// Grant created before any permission resolution warms the pond's cache.
|
||||||
|
const editorUser = await users.createUser({
|
||||||
|
username: `egon-editor-${suffix}`,
|
||||||
|
email: `egon-editor-${suffix}@example.org`,
|
||||||
|
displayName: 'Egon Editor',
|
||||||
|
password,
|
||||||
|
locale: 'en',
|
||||||
|
});
|
||||||
|
await users.markEmailVerified(editorUser.id);
|
||||||
|
await prisma.roleGrant.create({
|
||||||
|
data: {
|
||||||
|
pondId,
|
||||||
|
subjectType: 'USER',
|
||||||
|
subjectId: editorUser.id,
|
||||||
|
role: 'EDITOR',
|
||||||
|
scopeType: 'POND',
|
||||||
|
effect: 'ALLOW',
|
||||||
|
createdBy: ownerId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
ownerCookie = await loginOf(`vera-vault-${suffix}`);
|
||||||
|
editorCookie = await loginOf(`egon-editor-${suffix}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await prisma.conversionJob.deleteMany({ where: { ownerId } });
|
||||||
|
await prisma.attachment.deleteMany({ where: { pondId } });
|
||||||
|
await prisma.roleGrant.deleteMany({ where: { pondId } });
|
||||||
|
await prisma.label.deleteMany({ where: { pondId } });
|
||||||
|
await prisma.page.deleteMany({ where: { pondId } });
|
||||||
|
await prisma.quotaOverride.deleteMany({ where: { subjectId: pondId } });
|
||||||
|
await prisma.pond.deleteMany({ where: { ownerId } });
|
||||||
|
await prisma.user.deleteMany({ where: { username: { contains: suffix } } });
|
||||||
|
await prisma.$disconnect();
|
||||||
|
await app.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects non-admins, garbage archives, and unknown options up front', async () => {
|
||||||
|
await api()
|
||||||
|
.post(`/api/v1/ponds/${pondId}/import/vault`)
|
||||||
|
.set('Cookie', editorCookie)
|
||||||
|
.attach('file', fixtureZip(), 'vault.zip')
|
||||||
|
.expect(403);
|
||||||
|
|
||||||
|
const garbage = await api()
|
||||||
|
.post(`/api/v1/ponds/${pondId}/import/vault`)
|
||||||
|
.set('Cookie', ownerCookie)
|
||||||
|
.attach('file', Buffer.from('not a zip'), 'vault.zip')
|
||||||
|
.expect(400);
|
||||||
|
expect(garbage.body.code).toBe('import_vault_invalid_zip');
|
||||||
|
|
||||||
|
await api()
|
||||||
|
.post(`/api/v1/ponds/${pondId}/import/vault`)
|
||||||
|
.set('Cookie', ownerCookie)
|
||||||
|
.field('options', JSON.stringify({ parentPageId: 'missing' }))
|
||||||
|
.attach('file', fixtureZip(), 'vault.zip')
|
||||||
|
.expect(404);
|
||||||
|
|
||||||
|
await api()
|
||||||
|
.post(`/api/v1/ponds/${pondId}/import/vault`)
|
||||||
|
.set('Cookie', ownerCookie)
|
||||||
|
.attach('file', Buffer.from('x'), 'vault.rar')
|
||||||
|
.expect(400);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('imports the fixture vault: tree, links, tags, assets, extra label', async () => {
|
||||||
|
// Pre-existing page that collides with a vault note's slug.
|
||||||
|
await api()
|
||||||
|
.post(`/api/v1/ponds/${pondId}/pages`)
|
||||||
|
.set('Cookie', ownerCookie)
|
||||||
|
.send({ title: 'Startseite' })
|
||||||
|
.expect(201);
|
||||||
|
const extra = await api()
|
||||||
|
.post(`/api/v1/ponds/${pondId}/labels`)
|
||||||
|
.set('Cookie', ownerCookie)
|
||||||
|
.send({ name: `importiert-${suffix}` })
|
||||||
|
.expect(201);
|
||||||
|
|
||||||
|
const mount = await api()
|
||||||
|
.post(`/api/v1/ponds/${pondId}/pages`)
|
||||||
|
.set('Cookie', ownerCookie)
|
||||||
|
.send({ title: `Vault Mount ${suffix}` })
|
||||||
|
.expect(201);
|
||||||
|
|
||||||
|
const job = await importVault({
|
||||||
|
parentPageId: mount.body.id,
|
||||||
|
labelIds: [extra.body.id],
|
||||||
|
frontmatterMode: 'strip',
|
||||||
|
});
|
||||||
|
expect(job.body.status).toBe('succeeded');
|
||||||
|
expect(job.body.resultPageId).toBe(mount.body.id);
|
||||||
|
|
||||||
|
// Tree: containers under the mount page. Mount depth 1 leaves exactly 4
|
||||||
|
// container levels — the 4-folder chain fits unmerged (the merge itself
|
||||||
|
// is pinned by the transform's unit tests).
|
||||||
|
const projekte = await pageBySlug('projekte');
|
||||||
|
expect(projekte?.parentId).toBe(mount.body.id);
|
||||||
|
const ebene4 = await pageBySlug('ebene4');
|
||||||
|
const ebene5 = await pageBySlug('ebene5');
|
||||||
|
expect(ebene5?.parentId).toBe(ebene4?.id);
|
||||||
|
const deepNote = await pageBySlug('tiefe-notiz');
|
||||||
|
expect(deepNote?.parentId).toBe(ebene5?.id);
|
||||||
|
|
||||||
|
// Slug collision with the pre-existing page → suffix; the vault-internal
|
||||||
|
// link follows the FINAL slug.
|
||||||
|
const start = await pageBySlug('startseite-2');
|
||||||
|
expect(start).toBeTruthy();
|
||||||
|
const startCache = await prisma.pageContentCache.findUniqueOrThrow({
|
||||||
|
where: { pageId: start!.id },
|
||||||
|
});
|
||||||
|
expect(startCache.markdown).toContain('[[projekt-a|Projekt A]]');
|
||||||
|
|
||||||
|
// Outgoing links are indexed at creation: resolved edge + phantom.
|
||||||
|
const links = await prisma.pageLink.findMany({ where: { fromPageId: start!.id } });
|
||||||
|
const projektA = await pageBySlug('projekt-a');
|
||||||
|
expect(links.find((l) => l.targetSlug === 'projekt-a')?.toPageId).toBe(projektA!.id);
|
||||||
|
expect(links.find((l) => l.targetSlug === 'fehlt-noch')?.toPageId).toBeNull();
|
||||||
|
|
||||||
|
// Tags → labels: nested #status/aktiv becomes a hierarchy; the extra
|
||||||
|
// dialog label is on every imported page.
|
||||||
|
const status = await prisma.label.findFirst({ where: { pondId, name: 'status' } });
|
||||||
|
const aktiv = await prisma.label.findFirst({ where: { pondId, name: 'aktiv' } });
|
||||||
|
expect(aktiv?.parentId).toBe(status?.id);
|
||||||
|
const projektALabels = await prisma.pageLabel.findMany({
|
||||||
|
where: { pageId: projektA!.id },
|
||||||
|
include: { label: true },
|
||||||
|
});
|
||||||
|
const names = projektALabels.map((row) => row.label.name).sort();
|
||||||
|
expect(names).toEqual(expect.arrayContaining(['aktiv', 'projekt', `importiert-${suffix}`]));
|
||||||
|
const startLabels = await prisma.pageLabel.findMany({
|
||||||
|
where: { pageId: start!.id },
|
||||||
|
include: { label: true },
|
||||||
|
});
|
||||||
|
expect(startLabels.map((row) => row.label.name)).toEqual(
|
||||||
|
expect.arrayContaining(['willkommen', `importiert-${suffix}`]),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Frontmatter stripped; inline tag gone from the text.
|
||||||
|
const aCache = await prisma.pageContentCache.findUniqueOrThrow({
|
||||||
|
where: { pageId: projektA!.id },
|
||||||
|
});
|
||||||
|
expect(aCache.markdown).not.toContain('tags:');
|
||||||
|
expect(aCache.markdown).not.toContain('#status');
|
||||||
|
|
||||||
|
// The image became a pond file embedded in the page; the PDF an attachment.
|
||||||
|
const attachments = await prisma.attachment.findMany({ where: { pondId } });
|
||||||
|
expect(attachments.some((a) => a.fileName === 'teich.png')).toBe(true);
|
||||||
|
const pdf = attachments.find((a) => a.fileName === 'unterlagen.pdf');
|
||||||
|
expect(pdf?.pageId).toBe((await pageBySlug('projekt-b'))!.id);
|
||||||
|
expect(aCache.markdown).toMatch(/!\[[^\]]*\]\([0-9a-f-]{36}\)/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rolls back everything when the pond runs out of storage', async () => {
|
||||||
|
const before = {
|
||||||
|
pages: await prisma.page.count({ where: { pondId } }),
|
||||||
|
files: await prisma.attachment.count({ where: { pondId } }),
|
||||||
|
labels: await prisma.label.count({ where: { pondId } }),
|
||||||
|
};
|
||||||
|
// Storage quota 1 byte → the first asset upload fails mid-import.
|
||||||
|
await prisma.quotaOverride.create({
|
||||||
|
data: { subjectType: 'POND', subjectId: pondId, quotaKey: 'storage_bytes', value: 1 },
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const job = await importVault({ frontmatterMode: 'strip' });
|
||||||
|
expect(job.body.status).toBe('failed');
|
||||||
|
expect(job.body.errorCode).toBe('quota_exceeded');
|
||||||
|
|
||||||
|
expect(await prisma.page.count({ where: { pondId } })).toBe(before.pages);
|
||||||
|
expect(await prisma.attachment.count({ where: { pondId } })).toBe(before.files);
|
||||||
|
} finally {
|
||||||
|
await prisma.quotaOverride.deleteMany({
|
||||||
|
where: { subjectId: pondId, quotaKey: 'storage_bytes' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('re-imports cleanly after a failure (no duplicates, fresh suffixes)', async () => {
|
||||||
|
const job = await importVault({ frontmatterMode: 'strip' });
|
||||||
|
expect(job.body.status).toBe('succeeded');
|
||||||
|
// The second import suffixes against the first one's slugs.
|
||||||
|
expect(await pageBySlug('projekt-a-2')).toBeTruthy();
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -1,5 +1,6 @@
|
|||||||
import {
|
import {
|
||||||
BadRequestException,
|
BadRequestException,
|
||||||
|
Body,
|
||||||
Controller,
|
Controller,
|
||||||
Param,
|
Param,
|
||||||
Post,
|
Post,
|
||||||
@ -8,9 +9,14 @@ import {
|
|||||||
UseInterceptors,
|
UseInterceptors,
|
||||||
} from '@nestjs/common';
|
} from '@nestjs/common';
|
||||||
import { FileInterceptor } from '@nestjs/platform-express';
|
import { FileInterceptor } from '@nestjs/platform-express';
|
||||||
import { ConversionJobView, MAX_UPLOAD_PARSE_BYTES } from '@dorfteich/shared';
|
import {
|
||||||
|
ConversionJobView,
|
||||||
|
MAX_UPLOAD_PARSE_BYTES,
|
||||||
|
importVaultOptionsSchema,
|
||||||
|
} from '@dorfteich/shared';
|
||||||
|
|
||||||
import { AuthedRequest } from '../auth/auth.guard';
|
import { AuthedRequest } from '../auth/auth.guard';
|
||||||
|
import { ZodValidationPipe } from '../common/zod-validation.pipe';
|
||||||
import { RequiresPondRole } from '../permissions/permission.decorators';
|
import { RequiresPondRole } from '../permissions/permission.decorators';
|
||||||
|
|
||||||
import { ImportService } from './import.service';
|
import { ImportService } from './import.service';
|
||||||
@ -36,4 +42,33 @@ export class ImportController {
|
|||||||
if (!file) throw new BadRequestException({ code: 'bad_request' });
|
if (!file) throw new BadRequestException({ code: 'bad_request' });
|
||||||
return this.imports.enqueue(request.user!, pondId, file);
|
return this.imports.enqueue(request.user!, pondId, file);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Obsidian vault import (issue #117): a `.zip` of the whole vault plus a
|
||||||
|
* JSON `options` field ({parentPageId?, labelIds?, frontmatterMode}) in the
|
||||||
|
* same multipart form. Pond-admin-gated — a vault import creates a whole
|
||||||
|
* subtree, uploads files, and creates labels, which is pond administration
|
||||||
|
* rather than everyday editing.
|
||||||
|
*/
|
||||||
|
@Post('import/vault')
|
||||||
|
@RequiresPondRole('pond_admin', { idParam: 'pondId' })
|
||||||
|
@UseInterceptors(FileInterceptor('file', { limits: { fileSize: MAX_UPLOAD_PARSE_BYTES } }))
|
||||||
|
async importVault(
|
||||||
|
@Param('pondId') pondId: string,
|
||||||
|
@UploadedFile() file: Express.Multer.File | undefined,
|
||||||
|
@Body() body: { options?: string },
|
||||||
|
@Req() request: AuthedRequest,
|
||||||
|
): Promise<ConversionJobView> {
|
||||||
|
if (!file) throw new BadRequestException({ code: 'bad_request' });
|
||||||
|
let raw: unknown = {};
|
||||||
|
if (body.options) {
|
||||||
|
try {
|
||||||
|
raw = JSON.parse(body.options);
|
||||||
|
} catch {
|
||||||
|
throw new BadRequestException({ code: 'bad_request' });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const options = new ZodValidationPipe(importVaultOptionsSchema).transform(raw);
|
||||||
|
return this.imports.enqueueVault(request.user!, pondId, file, options);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,15 +1,37 @@
|
|||||||
import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/common';
|
import {
|
||||||
|
BadRequestException,
|
||||||
|
ForbiddenException,
|
||||||
|
Injectable,
|
||||||
|
NotFoundException,
|
||||||
|
} from '@nestjs/common';
|
||||||
import { ConversionJob, Page, User } from '@prisma/client';
|
import { ConversionJob, Page, User } from '@prisma/client';
|
||||||
import { ConversionJobView, editorSchema, markdownToDoc } from '@dorfteich/shared';
|
import {
|
||||||
|
ConversionJobView,
|
||||||
|
ImportVaultOptions,
|
||||||
|
MAX_LABEL_DEPTH,
|
||||||
|
MAX_UPLOAD_PARSE_BYTES,
|
||||||
|
editorSchema,
|
||||||
|
importVaultOptionsSchema,
|
||||||
|
markdownToDoc,
|
||||||
|
nodeDepth,
|
||||||
|
} from '@dorfteich/shared';
|
||||||
import { Node } from 'prosemirror-model';
|
import { Node } from 'prosemirror-model';
|
||||||
import { PinoLogger } from 'nestjs-pino';
|
import { PinoLogger } from 'nestjs-pino';
|
||||||
|
|
||||||
import { FilesService } from '../files/files.service';
|
import { FilesService } from '../files/files.service';
|
||||||
|
import { LabelsService } from '../labels/labels.service';
|
||||||
import { PagesService } from '../pages/pages.service';
|
import { PagesService } from '../pages/pages.service';
|
||||||
import { docToState } from '../pages/yjs-content';
|
import { docToState, emptyPageState } from '../pages/yjs-content';
|
||||||
import { PrismaService } from '../prisma/prisma.service';
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
|
|
||||||
import { ImportProcessor } from './import.constants';
|
import { ImportProcessor } from './import.constants';
|
||||||
|
import {
|
||||||
|
ASSET_PLACEHOLDER_PREFIX,
|
||||||
|
VaultError,
|
||||||
|
VaultImportPlan,
|
||||||
|
parseVaultZip,
|
||||||
|
planVaultImport,
|
||||||
|
} from './obsidian-vault';
|
||||||
import { ConversionError, PandocConverter } from './pandoc.converter';
|
import { ConversionError, PandocConverter } from './pandoc.converter';
|
||||||
import { ConversionJobService } from './conversion-job.service';
|
import { ConversionJobService } from './conversion-job.service';
|
||||||
|
|
||||||
@ -108,6 +130,7 @@ export class ImportService implements ImportProcessor {
|
|||||||
private readonly converter: PandocConverter,
|
private readonly converter: PandocConverter,
|
||||||
private readonly files: FilesService,
|
private readonly files: FilesService,
|
||||||
private readonly pages: PagesService,
|
private readonly pages: PagesService,
|
||||||
|
private readonly labels: LabelsService,
|
||||||
private readonly logger: PinoLogger,
|
private readonly logger: PinoLogger,
|
||||||
) {
|
) {
|
||||||
this.logger.setContext(ImportService.name);
|
this.logger.setContext(ImportService.name);
|
||||||
@ -180,6 +203,51 @@ export class ImportService implements ImportProcessor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate and enqueue an Obsidian vault import (issue #117). The ZIP is
|
||||||
|
* parsed once here for fast feedback (an invalid archive 400s instead of
|
||||||
|
* failing a job later); the worker re-parses when the job runs. The mount
|
||||||
|
* parent is validated up front too, and again at run time.
|
||||||
|
*/
|
||||||
|
async enqueueVault(
|
||||||
|
user: User,
|
||||||
|
pondId: string,
|
||||||
|
file: { buffer: Buffer; originalname: string },
|
||||||
|
options: ImportVaultOptions,
|
||||||
|
): Promise<ConversionJobView> {
|
||||||
|
if (fileExtension(file.originalname) !== 'zip') {
|
||||||
|
throw new BadRequestException({ code: 'import_unsupported_format' });
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
parseVaultZip(new Uint8Array(file.buffer));
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof VaultError) {
|
||||||
|
throw new BadRequestException({ code: error.code });
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
if (options.parentPageId) await this.requireLivePageInPond(pondId, options.parentPageId);
|
||||||
|
for (const labelId of options.labelIds) {
|
||||||
|
const label = await this.prisma.label.findFirst({ where: { id: labelId, pondId } });
|
||||||
|
if (!label) throw new NotFoundException();
|
||||||
|
}
|
||||||
|
|
||||||
|
const job = await this.jobs.enqueue({
|
||||||
|
ownerId: user.id,
|
||||||
|
pondId,
|
||||||
|
kind: 'import_vault',
|
||||||
|
from: 'zip',
|
||||||
|
to: 'pages',
|
||||||
|
input: file.buffer,
|
||||||
|
sourceName: file.originalname,
|
||||||
|
standalone: false,
|
||||||
|
options,
|
||||||
|
// The 25 MiB default guards the pandoc sidecar; a vault never goes there.
|
||||||
|
maxInputBytes: MAX_UPLOAD_PARSE_BYTES,
|
||||||
|
});
|
||||||
|
return this.jobs.viewOf(job);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Run one import job to completion (called by the worker). Throws a
|
* Run one import job to completion (called by the worker). Throws a
|
||||||
* {@link ConversionError} on failure so the worker applies the queue's
|
* {@link ConversionError} on failure so the worker applies the queue's
|
||||||
@ -195,6 +263,9 @@ export class ImportService implements ImportProcessor {
|
|||||||
if (!user) {
|
if (!user) {
|
||||||
throw new ConversionError('conversion_failed', false, 'import job owner is gone');
|
throw new ConversionError('conversion_failed', false, 'import job owner is gone');
|
||||||
}
|
}
|
||||||
|
if (job.kind === 'import_vault') {
|
||||||
|
return this.runVault(job, user, job.pondId);
|
||||||
|
}
|
||||||
|
|
||||||
const rawMarkdown = await convertImportedDocument(
|
const rawMarkdown = await convertImportedDocument(
|
||||||
this.converter,
|
this.converter,
|
||||||
@ -212,6 +283,251 @@ export class ImportService implements ImportProcessor {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a vault import job (issue #117): transform (issue #116) → containers
|
||||||
|
* top-down → notes → assets → labels, ALL-OR-NOTHING. On any failure every
|
||||||
|
* created page is hard-deleted (children before parents) and every stored
|
||||||
|
* file removed (quota restored), so the worker's retry policy is safe and a
|
||||||
|
* failed import leaves no trace.
|
||||||
|
*/
|
||||||
|
private async runVault(job: ConversionJob, user: User, pondId: string): Promise<void> {
|
||||||
|
const options = importVaultOptionsSchema.parse(job.options ?? {});
|
||||||
|
|
||||||
|
// Re-validate the mount parent (it may have been trashed since enqueue)
|
||||||
|
// and compute its depth for the folder budget.
|
||||||
|
let mountDepth = 0;
|
||||||
|
if (options.parentPageId) {
|
||||||
|
const live = await this.prisma.page.findFirst({
|
||||||
|
where: { id: options.parentPageId, pondId, deletedAt: null },
|
||||||
|
select: { id: true },
|
||||||
|
});
|
||||||
|
if (!live) {
|
||||||
|
throw new ConversionError('conversion_failed', false, 'vault mount parent is gone');
|
||||||
|
}
|
||||||
|
const tree = await this.prisma.page.findMany({
|
||||||
|
where: { pondId, deletedAt: null },
|
||||||
|
select: { id: true, parentId: true },
|
||||||
|
});
|
||||||
|
mountDepth = nodeDepth(tree, options.parentPageId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Slugs are unique across live AND trashed pages (the unique index does
|
||||||
|
// not care about deletedAt), so the reservation set includes everything.
|
||||||
|
const existing = await this.prisma.page.findMany({
|
||||||
|
where: { pondId },
|
||||||
|
select: { slug: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
let plan: VaultImportPlan;
|
||||||
|
let assetBytes: Map<string, { data: Uint8Array; name: string }>;
|
||||||
|
try {
|
||||||
|
const zip = new Uint8Array(job.input);
|
||||||
|
plan = planVaultImport(zip, {
|
||||||
|
frontmatterMode: options.frontmatterMode,
|
||||||
|
existingSlugs: new Set(existing.map((row) => row.slug)),
|
||||||
|
mountDepth,
|
||||||
|
});
|
||||||
|
const vault = parseVaultZip(zip);
|
||||||
|
assetBytes = new Map(
|
||||||
|
vault.assets.map((asset) => [asset.path, { data: asset.data, name: asset.name }]),
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof VaultError) {
|
||||||
|
throw new ConversionError(error.code, false, error.message);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const createdPageIds: string[] = [];
|
||||||
|
const storedFileIds: string[] = [];
|
||||||
|
try {
|
||||||
|
// Container pages top-down (parents exist before their children).
|
||||||
|
const pageIdByKey = new Map<string, string>();
|
||||||
|
const parentOf = (key: string | null): string | null =>
|
||||||
|
key ? pageIdByKey.get(key)! : (options.parentPageId ?? null);
|
||||||
|
for (const container of plan.containers) {
|
||||||
|
const page = await this.pages.createWithState(
|
||||||
|
user,
|
||||||
|
pondId,
|
||||||
|
container.title,
|
||||||
|
emptyPageState(),
|
||||||
|
parentOf(container.parentKey),
|
||||||
|
container.slug,
|
||||||
|
);
|
||||||
|
createdPageIds.push(page.id);
|
||||||
|
pageIdByKey.set(container.key, page.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Referenced assets, uploaded once; the attachment row links to the
|
||||||
|
// first page that references the asset (done per note below).
|
||||||
|
const fileIdByAsset = new Map<string, string>();
|
||||||
|
for (const path of plan.referencedAssets) {
|
||||||
|
const asset = assetBytes.get(path);
|
||||||
|
if (!asset) continue;
|
||||||
|
const view = await this.uploadVaultAsset(user, pondId, asset, storedFileIds);
|
||||||
|
if (view) fileIdByAsset.set(path, view);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Notes: replace asset placeholders, parse, create, link, label.
|
||||||
|
const labelIdByPath = new Map<string, string>();
|
||||||
|
const linkedAssets = new Set<string>();
|
||||||
|
for (const note of plan.notes) {
|
||||||
|
const placeholder = new RegExp(
|
||||||
|
`!\\[([^\\]]*)\\]\\(${ASSET_PLACEHOLDER_PREFIX}([^)\\s]+)\\)`,
|
||||||
|
'g',
|
||||||
|
);
|
||||||
|
const markdown = note.markdown.replaceAll(
|
||||||
|
placeholder,
|
||||||
|
(_whole, alt: string, path: string) => {
|
||||||
|
const fileId = fileIdByAsset.get(path);
|
||||||
|
return fileId ? `` : '';
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const json = markdownToDoc(markdown).toJSON() as unknown as PmNode;
|
||||||
|
const state = docToState(Node.fromJSON(editorSchema, json));
|
||||||
|
const page = await this.pages.createWithState(
|
||||||
|
user,
|
||||||
|
pondId,
|
||||||
|
note.title,
|
||||||
|
state,
|
||||||
|
parentOf(note.parentKey),
|
||||||
|
note.slug,
|
||||||
|
);
|
||||||
|
createdPageIds.push(page.id);
|
||||||
|
|
||||||
|
const noteAssets = [...note.imageAssets, ...note.attachmentAssets]
|
||||||
|
.filter((path) => !linkedAssets.has(path))
|
||||||
|
.map((path) => fileIdByAsset.get(path))
|
||||||
|
.filter((id): id is string => Boolean(id));
|
||||||
|
for (const path of [...note.imageAssets, ...note.attachmentAssets]) {
|
||||||
|
linkedAssets.add(path);
|
||||||
|
}
|
||||||
|
await this.files.linkAttachmentsToPage(noteAssets, page.id);
|
||||||
|
|
||||||
|
for (const tag of note.tags) {
|
||||||
|
const labelId = await this.ensureLabelPath(user, pondId, tag, labelIdByPath);
|
||||||
|
if (labelId) await this.labels.assign(user, page.id, labelId);
|
||||||
|
}
|
||||||
|
for (const labelId of options.labelIds) {
|
||||||
|
await this.labels.assign(user, page.id, labelId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.prisma.conversionJob.update({
|
||||||
|
where: { id: job.id },
|
||||||
|
data: {
|
||||||
|
status: 'SUCCEEDED',
|
||||||
|
resultPageId: options.parentPageId ?? null,
|
||||||
|
errorCode: null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
this.logger.info(
|
||||||
|
{ jobId: job.id, pondId, pages: createdPageIds.length, files: storedFileIds.length },
|
||||||
|
'audit: vault imported',
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
await this.rollbackPages(createdPageIds);
|
||||||
|
await this.rollbackMedia(user, storedFileIds);
|
||||||
|
if (error instanceof ForbiddenException && errorCode(error) === 'quota_exceeded') {
|
||||||
|
throw new ConversionError('quota_exceeded', false, 'pond storage exhausted during import');
|
||||||
|
}
|
||||||
|
if (error instanceof ConversionError || error instanceof VaultError) {
|
||||||
|
throw error instanceof VaultError
|
||||||
|
? new ConversionError(error.code, false, error.message)
|
||||||
|
: error;
|
||||||
|
}
|
||||||
|
throw new ConversionError(
|
||||||
|
'conversion_failed',
|
||||||
|
true,
|
||||||
|
error instanceof Error ? error.message : 'vault import failed',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Uploads one vault asset as a pond file; a type the instance rejects is
|
||||||
|
* dropped (structure over layout), an exhausted pond fails the import. */
|
||||||
|
private async uploadVaultAsset(
|
||||||
|
user: User,
|
||||||
|
pondId: string,
|
||||||
|
asset: { data: Uint8Array; name: string },
|
||||||
|
storedFileIds: string[],
|
||||||
|
): Promise<string | null> {
|
||||||
|
try {
|
||||||
|
const view = await this.files.upload(user, pondId, {
|
||||||
|
buffer: Buffer.from(asset.data),
|
||||||
|
size: asset.data.length,
|
||||||
|
originalname: asset.name,
|
||||||
|
});
|
||||||
|
storedFileIds.push(view.id);
|
||||||
|
return view.id;
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof ForbiddenException && errorCode(error) === 'quota_exceeded') {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
this.logger.warn({ pondId, code: errorCode(error) }, 'vault import: dropped an asset');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find-or-create the label chain for a nested tag (`a/b` → label `a` with
|
||||||
|
* child `b`), returning the deepest label's id. Creation goes through
|
||||||
|
* LabelsService so locking and permission-cache invalidation apply; chains
|
||||||
|
* deeper than the label limit are clamped to it.
|
||||||
|
*/
|
||||||
|
private async ensureLabelPath(
|
||||||
|
user: User,
|
||||||
|
pondId: string,
|
||||||
|
tag: string[],
|
||||||
|
cache: Map<string, string>,
|
||||||
|
): Promise<string | null> {
|
||||||
|
const path = tag.slice(0, MAX_LABEL_DEPTH);
|
||||||
|
let parentId: string | null = null;
|
||||||
|
let key = '';
|
||||||
|
for (const name of path) {
|
||||||
|
key = key ? `${key}/${name}` : name;
|
||||||
|
const cached = cache.get(key);
|
||||||
|
if (cached) {
|
||||||
|
parentId = cached;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const found: { id: string } | null = await this.prisma.label.findFirst({
|
||||||
|
where: { pondId, parentId, name },
|
||||||
|
select: { id: true },
|
||||||
|
});
|
||||||
|
const id: string = found
|
||||||
|
? found.id
|
||||||
|
: (await this.labels.create(user, pondId, { name, parentId })).id;
|
||||||
|
cache.set(key, id);
|
||||||
|
parentId = id;
|
||||||
|
}
|
||||||
|
return parentId;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Hard-deletes the pages a failed vault import created — children before
|
||||||
|
* parents (creation order reversed), attachments already gone via the media
|
||||||
|
* rollback, everything else cascades. */
|
||||||
|
private async rollbackPages(pageIds: string[]): Promise<void> {
|
||||||
|
for (const id of [...pageIds].reverse()) {
|
||||||
|
try {
|
||||||
|
await this.prisma.page.delete({ where: { id } });
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn(
|
||||||
|
{ pageId: id, error: error instanceof Error ? error.message : String(error) },
|
||||||
|
'vault import: page rollback failed',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async requireLivePageInPond(pondId: string, pageId: string): Promise<void> {
|
||||||
|
const page = await this.prisma.page.findFirst({
|
||||||
|
where: { id: pageId, pondId, deletedAt: null },
|
||||||
|
select: { id: true },
|
||||||
|
});
|
||||||
|
if (!page) throw new NotFoundException();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The shared tail of every import: store embedded images, parse the Markdown
|
* The shared tail of every import: store embedded images, parse the Markdown
|
||||||
* into an editor document, and create a page from it. Media stored during a
|
* into an editor document, and create a page from it. Media stored during a
|
||||||
|
|||||||
@ -30,7 +30,10 @@ export type ConversionErrorCode =
|
|||||||
| 'conversion_failed'
|
| 'conversion_failed'
|
||||||
// The pond ran out of storage while an import stored the document's media
|
// The pond ran out of storage while an import stored the document's media
|
||||||
// (#63) — final, and surfaced the same way through the worker.
|
// (#63) — final, and surfaced the same way through the worker.
|
||||||
| 'quota_exceeded';
|
| 'quota_exceeded'
|
||||||
|
// Vault import (#117): rejected archive / unpacked ceiling exceeded.
|
||||||
|
| 'import_vault_invalid_zip'
|
||||||
|
| 'import_vault_too_large';
|
||||||
|
|
||||||
/** A conversion failure with a stable, localizable code. `retryable` marks
|
/** A conversion failure with a stable, localizable code. `retryable` marks
|
||||||
* the transient causes (sidecar down / timed out) the worker retries before
|
* the transient causes (sidecar down / timed out) the worker retries before
|
||||||
|
|||||||
@ -225,8 +225,9 @@ export class PagesService {
|
|||||||
title: string,
|
title: string,
|
||||||
state: Uint8Array<ArrayBuffer>,
|
state: Uint8Array<ArrayBuffer>,
|
||||||
parentId: string | null = null,
|
parentId: string | null = null,
|
||||||
|
presetSlug?: string,
|
||||||
): Promise<Page> {
|
): Promise<Page> {
|
||||||
return this.insertPage(user, pondId, title, state, parentId);
|
return this.insertPage(user, pondId, title, state, parentId, presetSlug);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -250,12 +251,15 @@ export class PagesService {
|
|||||||
title: string,
|
title: string,
|
||||||
state: Uint8Array<ArrayBuffer>,
|
state: Uint8Array<ArrayBuffer>,
|
||||||
parentId: string | null = null,
|
parentId: string | null = null,
|
||||||
|
presetSlug?: string,
|
||||||
): Promise<Page> {
|
): Promise<Page> {
|
||||||
const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } });
|
const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } });
|
||||||
if (!pond) throw new NotFoundException();
|
if (!pond) throw new NotFoundException();
|
||||||
if (parentId) this.assertValidParent(await this.livePageTree(pond.id), parentId, null);
|
if (parentId) this.assertValidParent(await this.livePageTree(pond.id), parentId, null);
|
||||||
|
|
||||||
const slug = await this.generateUniqueSlugInPond(pond.id, title);
|
// A batch import (#117) reserves its slugs up front against the pond
|
||||||
|
// snapshot ∪ batch — the unique index stays the final arbiter.
|
||||||
|
const slug = presetSlug ?? (await this.generateUniqueSlugInPond(pond.id, title));
|
||||||
const last = await this.prisma.page.findFirst({
|
const last = await this.prisma.page.findFirst({
|
||||||
where: { pondId: pond.id },
|
where: { pondId: pond.id },
|
||||||
orderBy: { sortKey: 'desc' },
|
orderBy: { sortKey: 'desc' },
|
||||||
@ -278,12 +282,35 @@ export class PagesService {
|
|||||||
});
|
});
|
||||||
// A new page may satisfy phantom wikilinks that referenced its slug (#47).
|
// A new page may satisfy phantom wikilinks that referenced its slug (#47).
|
||||||
await this.resolvePhantomLinks(pond.id, slug, page.id);
|
await this.resolvePhantomLinks(pond.id, slug, page.id);
|
||||||
|
// Seed the page's own outgoing links (issue #117): pages born with
|
||||||
|
// content (imports) would otherwise stay invisible to backlinks and the
|
||||||
|
// graph until their first collab save — collab rewrites these rows on
|
||||||
|
// every later save, exactly as it does for hand-typed links.
|
||||||
|
if (content.wikilinkSlugs.length > 0) {
|
||||||
|
await this.indexOutgoingLinks(pond.id, page.id, content.wikilinkSlugs);
|
||||||
|
}
|
||||||
// Index the page so a title-only match is findable immediately (#49).
|
// Index the page so a title-only match is findable immediately (#49).
|
||||||
await this.search.indexPage(page.id);
|
await this.search.indexPage(page.id);
|
||||||
this.logger.info({ pageId: page.id, pondId: pond.id, userId: user.id }, 'audit: page created');
|
this.logger.info({ pageId: page.id, pondId: pond.id, userId: user.id }, 'audit: page created');
|
||||||
return page;
|
return page;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Insert the outgoing `page_links` rows of a freshly created page (issue
|
||||||
|
* #117), resolving each target slug against the pond's live pages — the
|
||||||
|
* same shape the collab persistence writes on every save (#47). Insert-only:
|
||||||
|
* a brand-new page has no rows to replace.
|
||||||
|
*/
|
||||||
|
private async indexOutgoingLinks(pondId: string, pageId: string, slugs: string[]): Promise<void> {
|
||||||
|
await this.prisma.$executeRaw`
|
||||||
|
INSERT INTO page_links (id, from_page_id, to_page_id, target_slug)
|
||||||
|
SELECT gen_random_uuid(), ${pageId}, target.id, s.link_slug
|
||||||
|
FROM unnest(${slugs}::text[]) AS s(link_slug)
|
||||||
|
LEFT JOIN pages AS target
|
||||||
|
ON target.pond_id = ${pondId} AND target.slug = s.link_slug
|
||||||
|
AND target.deleted_at IS NULL`;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Point phantom `page_links` (unresolved, `to_page_id` null) whose
|
* Point phantom `page_links` (unresolved, `to_page_id` null) whose
|
||||||
* `target_slug` matches `slug` within the pond at `pageId` (issue #47).
|
* `target_slug` matches `slug` within the pond at `pageId` (issue #47).
|
||||||
|
|||||||
@ -5,6 +5,7 @@ import {
|
|||||||
editorSchema,
|
editorSchema,
|
||||||
extractOutline,
|
extractOutline,
|
||||||
OutlineEntry,
|
OutlineEntry,
|
||||||
|
extractWikilinkSlugs,
|
||||||
} from '@dorfteich/shared';
|
} from '@dorfteich/shared';
|
||||||
import { Node } from 'prosemirror-model';
|
import { Node } from 'prosemirror-model';
|
||||||
import { prosemirrorJSONToYXmlFragment, yXmlFragmentToProseMirrorRootNode } from 'y-prosemirror';
|
import { prosemirrorJSONToYXmlFragment, yXmlFragmentToProseMirrorRootNode } from 'y-prosemirror';
|
||||||
@ -68,6 +69,10 @@ export interface DerivedPageContent {
|
|||||||
* embeds the file, which is what the trash-purge job uses to find a
|
* embeds the file, which is what the trash-purge job uses to find a
|
||||||
* purged page's files. */
|
* purged page's files. */
|
||||||
imageFileIds: string[];
|
imageFileIds: string[];
|
||||||
|
/** Outgoing wikilink target slugs (issue #117): pages created through the
|
||||||
|
* api (imports, phantom-create) seed their `page_links` rows from this —
|
||||||
|
* collab, the content writer, rewrites them on every later save. */
|
||||||
|
wikilinkSlugs: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
function imageFileIdsOf(doc: Node): string[] {
|
function imageFileIdsOf(doc: Node): string[] {
|
||||||
@ -93,5 +98,6 @@ export function deriveContent(state: Uint8Array): DerivedPageContent {
|
|||||||
html: docToHtml(doc),
|
html: docToHtml(doc),
|
||||||
outline: extractOutline(doc),
|
outline: extractOutline(doc),
|
||||||
imageFileIds: imageFileIdsOf(doc),
|
imageFileIds: imageFileIdsOf(doc),
|
||||||
|
wikilinkSlugs: extractWikilinkSlugs(doc),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@ -39,6 +39,8 @@
|
|||||||
"renderer_unavailable": "Der PDF-Renderer ist derzeit nicht verfügbar. Bitte versuche es später erneut.",
|
"renderer_unavailable": "Der PDF-Renderer ist derzeit nicht verfügbar. Bitte versuche es später erneut.",
|
||||||
"render_failed": "Diese Seite konnte nicht als PDF gerendert werden.",
|
"render_failed": "Diese Seite konnte nicht als PDF gerendert werden.",
|
||||||
"import_unsupported_format": "Nur Word- (.docx) und OpenDocument-Dokumente (.odt) können importiert werden.",
|
"import_unsupported_format": "Nur Word- (.docx) und OpenDocument-Dokumente (.odt) können importiert werden.",
|
||||||
|
"import_vault_invalid_zip": "Das Archiv ist kein gültiger Obsidian-Vault (ZIP mit Markdown-Notizen).",
|
||||||
|
"import_vault_too_large": "Der entpackte Vault ist zu groß für einen Import.",
|
||||||
"network": "Der Server war nicht erreichbar.",
|
"network": "Der Server war nicht erreichbar.",
|
||||||
"grant_exists": "Diese Berechtigung existiert bereits.",
|
"grant_exists": "Diese Berechtigung existiert bereits.",
|
||||||
"plugin_invalid_zip": "Die hochgeladene Datei ist kein gültiges ZIP-Archiv.",
|
"plugin_invalid_zip": "Die hochgeladene Datei ist kein gültiges ZIP-Archiv.",
|
||||||
|
|||||||
@ -39,6 +39,8 @@
|
|||||||
"renderer_unavailable": "The PDF renderer is currently unavailable. Please try again later.",
|
"renderer_unavailable": "The PDF renderer is currently unavailable. Please try again later.",
|
||||||
"render_failed": "This page could not be rendered as a PDF.",
|
"render_failed": "This page could not be rendered as a PDF.",
|
||||||
"import_unsupported_format": "Only Word (.docx) and OpenDocument (.odt) documents can be imported.",
|
"import_unsupported_format": "Only Word (.docx) and OpenDocument (.odt) documents can be imported.",
|
||||||
|
"import_vault_invalid_zip": "The archive is not a valid Obsidian vault (ZIP with Markdown notes).",
|
||||||
|
"import_vault_too_large": "The unpacked vault is too large for an import.",
|
||||||
"network": "The server could not be reached.",
|
"network": "The server could not be reached.",
|
||||||
"grant_exists": "This grant already exists.",
|
"grant_exists": "This grant already exists.",
|
||||||
"plugin_invalid_zip": "The uploaded file is not a valid ZIP archive.",
|
"plugin_invalid_zip": "The uploaded file is not a valid ZIP archive.",
|
||||||
|
|||||||
@ -35,6 +35,24 @@ export interface ConversionJobView {
|
|||||||
export const IMPORT_EXTENSIONS = ['docx', 'odt', 'md', 'markdown'] as const;
|
export const IMPORT_EXTENSIONS = ['docx', 'odt', 'md', 'markdown'] as const;
|
||||||
export type ImportExtension = (typeof IMPORT_EXTENSIONS)[number];
|
export type ImportExtension = (typeof IMPORT_EXTENSIONS)[number];
|
||||||
|
|
||||||
|
/** What happens to a note's YAML frontmatter on vault import (issue #117):
|
||||||
|
* dropped, or preserved as a yaml code block at the top of the page. */
|
||||||
|
export const VAULT_FRONTMATTER_MODES = ['strip', 'preserve'] as const;
|
||||||
|
export type VaultFrontmatterMode = (typeof VAULT_FRONTMATTER_MODES)[number];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Options of an Obsidian vault import (issue #117), sent as a JSON string in
|
||||||
|
* the multipart `options` field and stored on the job row. `parentPageId`
|
||||||
|
* mounts the vault under an existing page (null/absent = the pond root);
|
||||||
|
* `labelIds` are assigned to every imported page on top of the tag labels.
|
||||||
|
*/
|
||||||
|
export const importVaultOptionsSchema = z.object({
|
||||||
|
parentPageId: z.string().min(1).nullish(),
|
||||||
|
labelIds: z.array(z.string().min(1)).max(20).default([]),
|
||||||
|
frontmatterMode: z.enum(VAULT_FRONTMATTER_MODES).default('strip'),
|
||||||
|
});
|
||||||
|
export type ImportVaultOptions = z.infer<typeof importVaultOptionsSchema>;
|
||||||
|
|
||||||
/** Formats a single page exports to via a conversion job whose result is
|
/** Formats a single page exports to via a conversion job whose result is
|
||||||
* downloaded from `GET /jobs/:id/result` (issues #65/#67). `docx`/`odt` run
|
* downloaded from `GET /jobs/:id/result` (issues #65/#67). `docx`/`odt` run
|
||||||
* `markdown → pandoc`; `pdf` renders HTML through Gotenberg. Markdown export is
|
* `markdown → pandoc`; `pdf` renders HTML through Gotenberg. Markdown export is
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user