dorfteich/apps/api/src/import-export/conversion-job.service.ts
Claude Fable 5 8ae010218e
Some checks failed
CD / Build and push images (push) Successful in 4m5s
CD / Deploy to Test (push) Successful in 9s
CI / Lint, typecheck, test (push) Failing after 4m21s
CI / Auth e2e pack (push) Has been skipped
CI / Import/export fidelity gate (push) Has been skipped
CI / Build container images (push) Has been skipped
CD / Smoke tests against Test (push) Successful in 1m22s
CD / Promote to Int (push) Successful in 11s
Obsidian vault import: endpoint, job orchestration, rollback (#117)
POST /ponds/:pondId/import/vault (pond-admin-gated; a vault import
creates a subtree, uploads files, and creates labels — administration,
not everyday editing) takes the ZIP plus a JSON options field
{parentPageId?, labelIds?, frontmatterMode}. The archive is parsed at
enqueue for fast 400s; the job (new kind import_vault, riding the
existing isImportKind worker routing) re-parses and runs the #116
transform, then: containers top-down → notes (asset placeholders →
uploaded pond files; non-images become page attachments) → tags to
labels (nested tags build a label hierarchy via LabelsService, so
locking and cache invalidation apply) plus the dialog labels.

All-or-nothing: any failure hard-deletes the created pages (children
first) and removes the stored files (quota restored), then surfaces as
import_vault_invalid_zip / import_vault_too_large / quota_exceeded /
conversion_failed — and makes the worker's retry policy safe.

Supporting changes:
- conversion_jobs gains a nullable options jsonb column; enqueue takes
  kind-specific options and a maxInputBytes override (the 25 MiB
  default protects the pandoc sidecar, which a vault never touches —
  vaults use the 64 MiB upload limit).
- insertPage accepts a pre-reserved slug (the batch reserves all slugs
  up front against pond ∪ batch).
- NEW: pages born with content seed their outgoing page_links rows
  (deriveContent now returns wikilinkSlugs) — imported pages would
  otherwise stay invisible to backlinks and the graph until their
  first collab save. Collab still rewrites the rows on every save, and
  the existing phantom resolution heals batch creation order.

import-vault.e2e.db.test.ts (4 tests, real worker drained): gating +
input rejection, the full fixture import (tree under a mount page,
collision suffixes, link rows incl. phantom, nested tag labels, extra
label everywhere, frontmatter stripped, image embedded + PDF attached),
complete quota rollback, and a clean re-import with fresh suffixes.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-14 17:13:11 +02:00

132 lines
4.8 KiB
TypeScript

import { Injectable, NotFoundException, PayloadTooLargeException } from '@nestjs/common';
import { ConversionJob, ConversionJobStatus as PrismaStatus } from '@prisma/client';
import { ConversionJobStatus, ConversionJobView } from '@dorfteich/shared';
import { PinoLogger } from 'nestjs-pino';
import { PrismaService } from '../prisma/prisma.service';
import { ConversionWorker } from './conversion-worker.service';
import { MAX_CONVERSION_INPUT_BYTES } from './pandoc.converter';
export interface EnqueueConversion {
ownerId: string;
kind: string;
from: string;
to: string;
input: Buffer;
standalone?: boolean;
/** Set for import jobs (#63): the pond the document is imported into and the
* original upload file name (a title fallback). */
pondId?: string;
sourceName?: string;
/** Kind-specific options persisted with the job (issue #117). */
options?: unknown;
/** Input ceiling override (issue #117): the 25 MiB default protects the
* pandoc sidecar; a vault import never touches it and may use the full
* upload limit instead. */
maxInputBytes?: number;
}
export interface ConversionResultPayload {
bytes: Buffer;
mimeType: string;
}
const STATUS_VIEW: Record<PrismaStatus, ConversionJobStatus> = {
PENDING: 'pending',
RUNNING: 'running',
SUCCEEDED: 'succeeded',
FAILED: 'failed',
};
/**
* Enqueues import/export conversions and answers owner-scoped polling
* (ADR 0009, issue #62). Enqueue persists the job (so it survives a restart)
* and wakes the worker for interactive latency; the worker
* ({@link ConversionWorker}) does the sidecar call out of band.
*/
@Injectable()
export class ConversionJobService {
constructor(
private readonly prisma: PrismaService,
private readonly worker: ConversionWorker,
private readonly logger: PinoLogger,
) {
this.logger.setContext(ConversionJobService.name);
}
async enqueue(request: EnqueueConversion): Promise<ConversionJob> {
const maxInputBytes = request.maxInputBytes ?? MAX_CONVERSION_INPUT_BYTES;
if (request.input.byteLength > maxInputBytes) {
throw new PayloadTooLargeException({
code: 'file_too_large',
details: { limitBytes: maxInputBytes },
});
}
const job = await this.prisma.conversionJob.create({
data: {
ownerId: request.ownerId,
pondId: request.pondId ?? null,
sourceName: request.sourceName ?? null,
kind: request.kind,
sourceFormat: request.from,
targetFormat: request.to,
standalone: request.standalone ?? true,
options: request.options === undefined ? undefined : (request.options as object),
// Prisma's Bytes maps to Uint8Array<ArrayBuffer>; a Node Buffer's
// backing store is ArrayBufferLike, so copy into a plain Uint8Array.
input: new Uint8Array(request.input),
},
});
this.logger.info(
{ jobId: job.id, kind: job.kind, ownerId: request.ownerId },
'audit: conversion enqueued',
);
this.worker.wake();
return job;
}
/** The job if it belongs to `userId`, else 404 — existence stays hidden from
* anyone but its owner (permissions.md §non-page objects). */
async getForOwner(id: string, userId: string): Promise<ConversionJobView> {
return this.viewOf(await this.ownedJob(id, userId));
}
/** The finished output bytes, or 404 while the job is not yet succeeded (so a
* caller cannot distinguish "still running" from "never existed"). */
async resultForOwner(id: string, userId: string): Promise<ConversionResultPayload> {
const job = await this.ownedJob(id, userId);
if (job.status !== 'SUCCEEDED' || !job.result) throw new NotFoundException();
// A data-export link expires (#68): past its window the result is treated as
// gone (a scheduled purge deletes the bytes), so the download 404s.
if (job.expiresAt && job.expiresAt.getTime() < Date.now()) throw new NotFoundException();
return {
bytes: Buffer.from(job.result),
mimeType: job.resultMimeType ?? 'application/octet-stream',
};
}
private async ownedJob(id: string, userId: string): Promise<ConversionJob> {
const job = await this.prisma.conversionJob.findFirst({ where: { id, ownerId: userId } });
if (!job) throw new NotFoundException();
return job;
}
viewOf(job: ConversionJob): ConversionJobView {
return {
id: job.id,
status: STATUS_VIEW[job.status],
kind: job.kind,
sourceFormat: job.sourceFormat,
targetFormat: job.targetFormat,
errorCode: job.errorCode,
// Set once an import job succeeds (#63) so the client can open the page.
resultPageId: job.resultPageId,
// Set for a data-export job (#68): when its download link stops working.
expiresAt: job.expiresAt?.toISOString() ?? null,
createdAt: job.createdAt.toISOString(),
updatedAt: job.updatedAt.toISOString(),
};
}
}