Nightly sweep with two directions: attachments still unclaimed (pageId null) after a 24 h grace period - claimed by no collab persist, page upload, or import - are reclaimed (row, file, quota released); files on the uploads volume without a database row (drift after a crashed upload) are removed once older than the grace period. The grace period protects the paste-then-insert window. Deliberate deviation from the issue's content-reference idea, documented in schema comment and operations.md: claimed attachments whose page content no longer embeds them are NOT auto-deleted. The page attachments panel lists claimed files as user-managed objects (inserting into the document is optional there), so 'not embedded' is not 'unused' - an auto-delete would destroy panel assets. Humans clean those up in the panel or the pond file manager, which flags orphans already. Attachment.deletedAt is removed by migration - deletion is hard everywhere (sweep, purge, manual), there is no soft-delete state; the never-true deletedAt:null filters in files/export queries went with it. Refs #194 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0168Ph5uBmHm8X28CSVpbpnJ
265 lines
9.4 KiB
TypeScript
265 lines
9.4 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import type { Readable } from 'node:stream';
|
|
|
|
import {
|
|
BadRequestException,
|
|
Injectable,
|
|
NotFoundException,
|
|
PayloadTooLargeException,
|
|
} from '@nestjs/common';
|
|
import {
|
|
ATTACHMENT_EXTENSION_MIME_TYPES,
|
|
AttachmentListItemView,
|
|
AttachmentView,
|
|
PondFilesView,
|
|
SVG_MIME_TYPE,
|
|
fileExtension,
|
|
isImageMimeType,
|
|
} from '@dorfteich/shared';
|
|
import { Attachment, User } from '@prisma/client';
|
|
import { PinoLogger } from 'nestjs-pino';
|
|
|
|
import { PrismaService } from '../prisma/prisma.service';
|
|
import { QuotaService } from '../quotas/quota.service';
|
|
import { InstanceSettingsService } from '../settings/instance-settings.service';
|
|
|
|
import { FileStorageService } from './file-storage.service';
|
|
import { looksLikeSvg, sniffImageMimeType } from './magic-bytes';
|
|
import { sanitizeSvg } from './svg-sanitize';
|
|
|
|
export interface FileDownload {
|
|
attachment: Attachment;
|
|
stream: Readable;
|
|
/** Raster images render inline (they are embedded in pages); everything
|
|
* else — office files, PDFs, and SVG — is always sent as a download so it
|
|
* can never execute inline (ADR 0011, security.md §Uploads). */
|
|
inline: boolean;
|
|
}
|
|
|
|
/** What the upload bytes resolved to after allowlist + SVG handling. */
|
|
interface ResolvedUpload {
|
|
mimeType: string;
|
|
/** The bytes to store — identical to the input except for a sanitized SVG. */
|
|
buffer: Buffer;
|
|
}
|
|
|
|
/** File storage, upload, and attachment listing API (issue #27, #61, ADR 0011). */
|
|
@Injectable()
|
|
export class FilesService {
|
|
constructor(
|
|
private readonly prisma: PrismaService,
|
|
private readonly quotas: QuotaService,
|
|
private readonly storage: FileStorageService,
|
|
private readonly settings: InstanceSettingsService,
|
|
private readonly logger: PinoLogger,
|
|
) {
|
|
this.logger.setContext(FilesService.name);
|
|
}
|
|
|
|
viewOf(attachment: Attachment): AttachmentView {
|
|
return {
|
|
id: attachment.id,
|
|
pondId: attachment.pondId,
|
|
pageId: attachment.pageId,
|
|
fileName: attachment.fileName,
|
|
mimeType: attachment.mimeType,
|
|
sizeBytes: attachment.sizeBytes,
|
|
createdAt: attachment.createdAt.toISOString(),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Decide the served MIME type and the bytes to persist. Raster images pass
|
|
* the magic-byte sniff and are always allowed. An SVG is sanitized (its
|
|
* scripts/handlers stripped) or rejected per the instance policy. Anything
|
|
* else is admitted only if its extension is on the configurable allowlist;
|
|
* its bytes are never trusted to be inert, but it is only ever downloaded,
|
|
* never rendered.
|
|
*/
|
|
private async resolveUpload(fileName: string, buffer: Buffer): Promise<ResolvedUpload> {
|
|
const rasterMime = sniffImageMimeType(buffer);
|
|
if (rasterMime) return { mimeType: rasterMime, buffer };
|
|
|
|
if (looksLikeSvg(buffer)) {
|
|
const policy = await this.settings.get('upload.svgPolicy');
|
|
if (policy === 'reject') {
|
|
throw new BadRequestException({ code: 'upload_type_not_allowed' });
|
|
}
|
|
let clean: string;
|
|
try {
|
|
clean = sanitizeSvg(buffer.toString('utf8'));
|
|
} catch {
|
|
throw new BadRequestException({ code: 'upload_type_not_allowed' });
|
|
}
|
|
return { mimeType: SVG_MIME_TYPE, buffer: Buffer.from(clean, 'utf8') };
|
|
}
|
|
|
|
const ext = fileExtension(fileName);
|
|
const allowed = await this.settings.get('upload.allowedExtensions');
|
|
if (!ext || !allowed.includes(ext)) {
|
|
throw new BadRequestException({
|
|
code: 'upload_type_not_allowed',
|
|
details: { allowed },
|
|
});
|
|
}
|
|
return { mimeType: ATTACHMENT_EXTENSION_MIME_TYPES[ext] ?? 'application/octet-stream', buffer };
|
|
}
|
|
|
|
/**
|
|
* Store an upload against a pond, optionally linked to a page. `pageId` is
|
|
* set for a page attachment (uploaded from the page's attachments section)
|
|
* so it is listed there and purged with the page; image pastes leave it null
|
|
* and the collab persistence hook links them to whichever page embeds them.
|
|
*/
|
|
async upload(
|
|
user: User,
|
|
pondId: string,
|
|
file: { buffer: Buffer; size: number; originalname: string },
|
|
pageId?: string,
|
|
): Promise<AttachmentView> {
|
|
const pond = await this.prisma.pond.findFirst({ where: { id: pondId, deletedAt: null } });
|
|
if (!pond) throw new NotFoundException();
|
|
|
|
const maxFileBytes = await this.quotas.getEffective('max_file_bytes', {
|
|
userId: pond.ownerId,
|
|
pondId: pond.id,
|
|
});
|
|
if (file.size > maxFileBytes) {
|
|
throw new PayloadTooLargeException({
|
|
code: 'file_too_large',
|
|
details: { limitBytes: maxFileBytes },
|
|
});
|
|
}
|
|
|
|
const resolved = await this.resolveUpload(file.originalname, file.buffer);
|
|
// Account for what actually lands on the volume (a sanitized SVG is
|
|
// usually smaller than the upload) so pond_usage matches disk exactly.
|
|
const sizeBytes = resolved.buffer.length;
|
|
|
|
const id = randomUUID();
|
|
// Consume the storage budget before touching the disk so a race never
|
|
// leaves bytes written without a matching reservation.
|
|
await this.quotas.checkAndConsume(pond.id, pond.ownerId, sizeBytes);
|
|
|
|
try {
|
|
await this.storage.save(pond.id, id, resolved.buffer);
|
|
const attachment = await this.prisma.attachment.create({
|
|
data: {
|
|
id,
|
|
pondId: pond.id,
|
|
pageId: pageId ?? null,
|
|
fileName: file.originalname,
|
|
mimeType: resolved.mimeType,
|
|
sizeBytes,
|
|
storagePath: `${pond.id}/${id}`,
|
|
uploadedBy: user.id,
|
|
},
|
|
});
|
|
this.logger.info(
|
|
{ attachmentId: id, pondId: pond.id, pageId: pageId ?? null, userId: user.id },
|
|
'audit: file uploaded',
|
|
);
|
|
return this.viewOf(attachment);
|
|
} catch (error) {
|
|
// Roll back the reservation and any bytes already written so usage
|
|
// never drifts from what is actually on the volume/in the database.
|
|
await this.quotas.release(pond.id, sizeBytes);
|
|
await this.storage.delete(pond.id, id);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Point a set of just-uploaded attachments at the page that now embeds them
|
|
* (issue #63 import). The import worker stores a document's media before the
|
|
* page exists (the page's Yjs state references their ids), then calls this so
|
|
* they list under the page and are purged with it (#31), the same invariant
|
|
* the collab persistence hook maintains for pasted images.
|
|
*/
|
|
async linkAttachmentsToPage(attachmentIds: string[], pageId: string): Promise<void> {
|
|
if (attachmentIds.length === 0) return;
|
|
await this.prisma.attachment.updateMany({
|
|
where: { id: { in: attachmentIds } },
|
|
data: { pageId },
|
|
});
|
|
}
|
|
|
|
/** Upload against the pond of `pageId`, linked to that page (#61). */
|
|
async uploadToPage(
|
|
user: User,
|
|
pageId: string,
|
|
file: { buffer: Buffer; size: number; originalname: string },
|
|
): Promise<AttachmentView> {
|
|
const page = await this.prisma.page.findFirst({ where: { id: pageId } });
|
|
if (!page) throw new NotFoundException();
|
|
return this.upload(user, page.pondId, file, page.id);
|
|
}
|
|
|
|
async download(_user: User | null, id: string): Promise<FileDownload> {
|
|
const attachment = await this.prisma.attachment.findFirst({ where: { id } });
|
|
if (!attachment) throw new NotFoundException();
|
|
return {
|
|
attachment,
|
|
stream: this.storage.createReadStream(attachment.pondId, attachment.id),
|
|
inline: isImageMimeType(attachment.mimeType),
|
|
};
|
|
}
|
|
|
|
/** Attachments linked to a page, for its attachments section (#61). */
|
|
async listForPage(pageId: string): Promise<AttachmentListItemView[]> {
|
|
const rows = await this.prisma.attachment.findMany({
|
|
where: { pageId },
|
|
orderBy: { createdAt: 'desc' },
|
|
include: { uploader: true, page: true },
|
|
});
|
|
return rows.map((row) => this.listItemOf(row));
|
|
}
|
|
|
|
/** Every file in a pond, for the Pond Admin file manager (#61). */
|
|
async listForPond(pondId: string): Promise<PondFilesView> {
|
|
const [rows, usage, storageBytesLimit] = await Promise.all([
|
|
this.prisma.attachment.findMany({
|
|
where: { pondId },
|
|
orderBy: { createdAt: 'desc' },
|
|
include: { uploader: true, page: true },
|
|
}),
|
|
this.prisma.pondUsage.findUnique({ where: { pondId } }),
|
|
this.pondStorageLimit(pondId),
|
|
]);
|
|
return {
|
|
files: rows.map((row) => this.listItemOf(row)),
|
|
storageBytesUsed: Number(usage?.storageBytesUsed ?? 0n),
|
|
storageBytesLimit,
|
|
};
|
|
}
|
|
|
|
private async pondStorageLimit(pondId: string): Promise<number> {
|
|
const pond = await this.prisma.pond.findUnique({ where: { id: pondId } });
|
|
if (!pond) throw new NotFoundException();
|
|
return this.quotas.getEffective('storage_bytes', { userId: pond.ownerId, pondId });
|
|
}
|
|
|
|
private listItemOf(
|
|
row: Attachment & { uploader: User; page: { title: string } | null },
|
|
): AttachmentListItemView {
|
|
return {
|
|
...this.viewOf(row),
|
|
uploaderName: row.uploader.displayName,
|
|
pageTitle: row.page?.title ?? null,
|
|
};
|
|
}
|
|
|
|
async remove(user: User, id: string): Promise<void> {
|
|
const attachment = await this.prisma.attachment.findFirst({ where: { id } });
|
|
if (!attachment) throw new NotFoundException();
|
|
|
|
await this.prisma.attachment.delete({ where: { id: attachment.id } });
|
|
await this.storage.delete(attachment.pondId, attachment.id);
|
|
await this.quotas.release(attachment.pondId, attachment.sizeBytes);
|
|
this.logger.info(
|
|
{ attachmentId: id, pondId: attachment.pondId, userId: user.id },
|
|
'audit: file deleted',
|
|
);
|
|
}
|
|
}
|