dorfteich/apps/api/src/files/files.service.ts
Claude Opus 4.8 546e8279ac
All checks were successful
CD / Build and push images (push) Successful in 3m19s
CI / Lint, typecheck, test (push) Successful in 2m55s
CI / Auth e2e pack (push) Successful in 3m45s
CI / Build container images (push) Has been skipped
CD / Deploy to Test (push) Successful in 9s
CD / Smoke tests against Test (push) Successful in 1m13s
CD / Promote to Int (push) Successful in 11s
Import .docx and .odt documents as new pages (#63)
Uploading a Word/OpenOffice document to POST /ponds/:id/import enqueues a
conversion job (the #62 queue) that produces a new page in the pond; the
client polls GET /jobs/:id for the created resultPageId.

Pipeline (ImportService, ADR 0009): pandoc-server is stateless and hands
back a document's media no other way, so we convert in two passes —
docx/odt → html with embed-resources inlines every image as a data: URI,
then html → gfm produces clean structural Markdown with those data URIs
still inline. Embedded images are stored as pond files (with quota
accounting) and their references rewritten to file ids on the Markdown
text before parsing (the editor parser only admits png/jpeg/gif/webp data
URIs); an image whose bytes the upload pipeline rejects is dropped, not
fatal. The title comes from a leading top-level heading (removed from the
body) else the file name. The page is created from the resulting Yjs state.

The shared conversion worker routes import-kind jobs to the pipeline via a
token (breaking a module cycle), so import inherits the queue's locking,
retry, and restart-survival. Media stored during a failed attempt is rolled
back; a pond that runs out of storage fails the job with quota_exceeded.

- schema: ConversionJob gains pond_id / source_name / result_page_id
  (migration 20260710041215_import_pages_conversion); ConversionJobView
  gains resultPageId.
- PagesService.createWithState / yjs-content docToState build a page from a
  prepared document; FilesService.linkAttachmentsToPage links import media.
- fixtures/import/: representative .docx/.odt corpus (headings, lists,
  nested lists, tables, images, links, bold/italic) with expected-Markdown
  snapshots; scripts/gen-import-fixtures.mjs regenerates them.
- tests: import.service.db.test.ts drives the full pipeline with a fake
  converter (CI); import.fixtures.test.ts runs the real two-pass conversion
  over the corpus and a 50-page timing check against a reachable sidecar.
- i18n: import_unsupported_format (de+en). Limits documented (25 MiB input,
  60 s per pass).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EwZ4jR4KFAPvpjWevfUGX1
2026-07-10 07:34:43 +02:00

265 lines
9.4 KiB
TypeScript

import { randomUUID } from 'node:crypto';
import type { Readable } from 'node:stream';
import {
BadRequestException,
Injectable,
NotFoundException,
PayloadTooLargeException,
} from '@nestjs/common';
import {
ATTACHMENT_EXTENSION_MIME_TYPES,
AttachmentListItemView,
AttachmentView,
PondFilesView,
SVG_MIME_TYPE,
fileExtension,
isImageMimeType,
} from '@dorfteich/shared';
import { Attachment, User } from '@prisma/client';
import { PinoLogger } from 'nestjs-pino';
import { PrismaService } from '../prisma/prisma.service';
import { QuotaService } from '../quotas/quota.service';
import { InstanceSettingsService } from '../settings/instance-settings.service';
import { FileStorageService } from './file-storage.service';
import { looksLikeSvg, sniffImageMimeType } from './magic-bytes';
import { sanitizeSvg } from './svg-sanitize';
export interface FileDownload {
attachment: Attachment;
stream: Readable;
/** Raster images render inline (they are embedded in pages); everything
* else — office files, PDFs, and SVG — is always sent as a download so it
* can never execute inline (ADR 0011, security.md §Uploads). */
inline: boolean;
}
/** What the upload bytes resolved to after allowlist + SVG handling. */
interface ResolvedUpload {
mimeType: string;
/** The bytes to store — identical to the input except for a sanitized SVG. */
buffer: Buffer;
}
/** File storage, upload, and attachment listing API (issue #27, #61, ADR 0011). */
@Injectable()
export class FilesService {
constructor(
private readonly prisma: PrismaService,
private readonly quotas: QuotaService,
private readonly storage: FileStorageService,
private readonly settings: InstanceSettingsService,
private readonly logger: PinoLogger,
) {
this.logger.setContext(FilesService.name);
}
viewOf(attachment: Attachment): AttachmentView {
return {
id: attachment.id,
pondId: attachment.pondId,
pageId: attachment.pageId,
fileName: attachment.fileName,
mimeType: attachment.mimeType,
sizeBytes: attachment.sizeBytes,
createdAt: attachment.createdAt.toISOString(),
};
}
/**
* Decide the served MIME type and the bytes to persist. Raster images pass
* the magic-byte sniff and are always allowed. An SVG is sanitized (its
* scripts/handlers stripped) or rejected per the instance policy. Anything
* else is admitted only if its extension is on the configurable allowlist;
* its bytes are never trusted to be inert, but it is only ever downloaded,
* never rendered.
*/
private async resolveUpload(fileName: string, buffer: Buffer): Promise<ResolvedUpload> {
const rasterMime = sniffImageMimeType(buffer);
if (rasterMime) return { mimeType: rasterMime, buffer };
if (looksLikeSvg(buffer)) {
const policy = await this.settings.get('upload.svgPolicy');
if (policy === 'reject') {
throw new BadRequestException({ code: 'upload_type_not_allowed' });
}
let clean: string;
try {
clean = sanitizeSvg(buffer.toString('utf8'));
} catch {
throw new BadRequestException({ code: 'upload_type_not_allowed' });
}
return { mimeType: SVG_MIME_TYPE, buffer: Buffer.from(clean, 'utf8') };
}
const ext = fileExtension(fileName);
const allowed = await this.settings.get('upload.allowedExtensions');
if (!ext || !allowed.includes(ext)) {
throw new BadRequestException({
code: 'upload_type_not_allowed',
details: { allowed },
});
}
return { mimeType: ATTACHMENT_EXTENSION_MIME_TYPES[ext] ?? 'application/octet-stream', buffer };
}
/**
* Store an upload against a pond, optionally linked to a page. `pageId` is
* set for a page attachment (uploaded from the page's attachments section)
* so it is listed there and purged with the page; image pastes leave it null
* and the collab persistence hook links them to whichever page embeds them.
*/
async upload(
user: User,
pondId: string,
file: { buffer: Buffer; size: number; originalname: string },
pageId?: string,
): Promise<AttachmentView> {
const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } });
if (!pond) throw new NotFoundException();
const maxFileBytes = await this.quotas.getEffective('max_file_bytes', {
userId: pond.ownerId,
pondId: pond.id,
});
if (file.size > maxFileBytes) {
throw new PayloadTooLargeException({
code: 'file_too_large',
details: { limitBytes: maxFileBytes },
});
}
const resolved = await this.resolveUpload(file.originalname, file.buffer);
// Account for what actually lands on the volume (a sanitized SVG is
// usually smaller than the upload) so pond_usage matches disk exactly.
const sizeBytes = resolved.buffer.length;
const id = randomUUID();
// Consume the storage budget before touching the disk so a race never
// leaves bytes written without a matching reservation.
await this.quotas.checkAndConsume(pond.id, pond.ownerId, sizeBytes);
try {
await this.storage.save(pond.id, id, resolved.buffer);
const attachment = await this.prisma.attachment.create({
data: {
id,
pondId: pond.id,
pageId: pageId ?? null,
fileName: file.originalname,
mimeType: resolved.mimeType,
sizeBytes,
storagePath: `${pond.id}/${id}`,
uploadedBy: user.id,
},
});
this.logger.info(
{ attachmentId: id, pondId: pond.id, pageId: pageId ?? null, userId: user.id },
'audit: file uploaded',
);
return this.viewOf(attachment);
} catch (error) {
// Roll back the reservation and any bytes already written so usage
// never drifts from what is actually on the volume/in the database.
await this.quotas.release(pond.id, sizeBytes);
await this.storage.delete(pond.id, id);
throw error;
}
}
/**
* Point a set of just-uploaded attachments at the page that now embeds them
* (issue #63 import). The import worker stores a document's media before the
* page exists (the page's Yjs state references their ids), then calls this so
* they list under the page and are purged with it (#31), the same invariant
* the collab persistence hook maintains for pasted images.
*/
async linkAttachmentsToPage(attachmentIds: string[], pageId: string): Promise<void> {
if (attachmentIds.length === 0) return;
await this.prisma.attachment.updateMany({
where: { id: { in: attachmentIds } },
data: { pageId },
});
}
/** Upload against the pond of `pageId`, linked to that page (#61). */
async uploadToPage(
user: User,
pageId: string,
file: { buffer: Buffer; size: number; originalname: string },
): Promise<AttachmentView> {
const page = await this.prisma.page.findFirst({ where: { id: pageId } });
if (!page) throw new NotFoundException();
return this.upload(user, page.pondId, file, page.id);
}
async download(_user: User | null, id: string): Promise<FileDownload> {
const attachment = await this.prisma.attachment.findFirst({ where: { id } });
if (!attachment) throw new NotFoundException();
return {
attachment,
stream: this.storage.createReadStream(attachment.pondId, attachment.id),
inline: isImageMimeType(attachment.mimeType),
};
}
/** Attachments linked to a page, for its attachments section (#61). */
async listForPage(pageId: string): Promise<AttachmentListItemView[]> {
const rows = await this.prisma.attachment.findMany({
where: { pageId, deletedAt: null },
orderBy: { createdAt: 'desc' },
include: { uploader: true, page: true },
});
return rows.map((row) => this.listItemOf(row));
}
/** Every file in a pond, for the Pond Admin file manager (#61). */
async listForPond(pondId: string): Promise<PondFilesView> {
const [rows, usage, storageBytesLimit] = await Promise.all([
this.prisma.attachment.findMany({
where: { pondId, deletedAt: null },
orderBy: { createdAt: 'desc' },
include: { uploader: true, page: true },
}),
this.prisma.pondUsage.findUnique({ where: { pondId } }),
this.pondStorageLimit(pondId),
]);
return {
files: rows.map((row) => this.listItemOf(row)),
storageBytesUsed: Number(usage?.storageBytesUsed ?? 0n),
storageBytesLimit,
};
}
private async pondStorageLimit(pondId: string): Promise<number> {
const pond = await this.prisma.pond.findUnique({ where: { id: pondId } });
if (!pond) throw new NotFoundException();
return this.quotas.getEffective('storage_bytes', { userId: pond.ownerId, pondId });
}
private listItemOf(
row: Attachment & { uploader: User; page: { title: string } | null },
): AttachmentListItemView {
return {
...this.viewOf(row),
uploaderName: row.uploader.displayName,
pageTitle: row.page?.title ?? null,
};
}
async remove(user: User, id: string): Promise<void> {
const attachment = await this.prisma.attachment.findFirst({ where: { id } });
if (!attachment) throw new NotFoundException();
await this.prisma.attachment.delete({ where: { id: attachment.id } });
await this.storage.delete(attachment.pondId, attachment.id);
await this.quotas.release(attachment.pondId, attachment.sizeBytes);
this.logger.info(
{ attachmentId: id, pondId: attachment.pondId, userId: user.id },
'audit: file deleted',
);
}
}