#193: pond purge (retention job + manual Site-Admin endpoint) #248

Merged
fable-5 merged 1 commits from feat/193-pond-purge into main 2026-07-30 12:59:52 +02:00
6 changed files with 394 additions and 4 deletions
Showing only changes of commit 402b22e05f - Show all commits

View File

@ -0,0 +1,269 @@
import { existsSync } from 'node:fs';
import { join } from 'node:path';
import { INestApplication } from '@nestjs/common';
import { PrismaClient } from '@prisma/client';
import request from 'supertest';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { createTestApp, sessionCookieOf } from '../testing/test-app';
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
import { UsersService } from '../users/users.service';
import { TrashService } from './trash.service';
/**
* Pond purge end to end (issue #193): deletion must actually delete
* after the purge NOTHING referencing the pond survives (rows, files on
* disk, search index), the operation is Site-Admin-only, idempotent, and
* both the manual and the retention path leave an audit event.
*/
describe.skipIf(!hasTestDb)('pond purge (e2e, issue #193)', () => {
let app: INestApplication;
let prisma: PrismaClient;
const suffix = uniqueSuffix();
const password = 'pond purge pass 1';
const ids: Record<string, string> = {};
const cookies: Record<string, string> = {};
let pondId: string;
let pageAId: string;
let fileId: string;
const needle = `zzpurgeneedle${suffix.replaceAll('-', '')}`;
const api = () => request(app.getHttpServer());
async function makeUser(handle: string, siteAdmin = false): Promise<void> {
const users = app.get(UsersService);
const username = `pp-${handle}-${suffix}`;
const user = await users.createUser({
username,
email: `${username}@example.org`,
displayName: `Purge ${handle}`,
password,
locale: 'en',
});
ids[handle] = user.id;
await users.markEmailVerified(user.id);
if (siteAdmin) {
await prisma.user.update({ where: { id: user.id }, data: { isSiteAdmin: true } });
}
cookies[handle] = sessionCookieOf(
await api()
.post('/api/v1/auth/login')
.send({ usernameOrEmail: username, password })
.expect(200),
);
}
beforeAll(async () => {
prisma = createTestPrisma();
await prisma.rateLimit.deleteMany({});
app = await createTestApp();
await makeUser('owner');
await makeUser('admin', true);
await api()
.put(`/api/v1/admin/quotas/user/${ids.owner!}/additional_ponds`)
.set('Cookie', cookies.admin!)
.send({ value: 5 })
.expect(200);
// The pond and two pages through the real API (grants, usage, slugs).
const pond = await api()
.post('/api/v1/ponds')
.set('Cookie', cookies.owner!)
.send({ name: `Purge Pond ${suffix}` })
.expect(201);
pondId = pond.body.id;
const pageA = await api()
.post(`/api/v1/ponds/${pondId}/pages`)
.set('Cookie', cookies.owner!)
.send({ title: `Purge Page A ${suffix}` })
.expect(201);
pageAId = pageA.body.id;
const pageB = await api()
.post(`/api/v1/ponds/${pondId}/pages`)
.set('Cookie', cookies.owner!)
.send({ title: `Purge Page B ${suffix}` })
.expect(201);
// A real uploaded file (disk + quota usage), attached to page A.
const upload = await api()
.post(`/api/v1/pages/${pageAId}/files`)
.set('Cookie', cookies.owner!)
.attach('file', Buffer.from('purge me'), 'purge.txt')
.expect(201);
fileId = upload.body.id;
// Every remaining representation, seeded directly: content cache with a
// searchable vector, update log, version, comment, label + assignment,
// favorite, watches (page + pond), a page link, a pond quota override.
// Page creation already seeded a cache row — fill it with content.
await prisma.pageContentCache.upsert({
where: { pageId: pageAId },
create: {
pageId: pageAId,
plainText: `text with ${needle}`,
markdown: needle,
html: `<p>${needle}</p>`,
outline: [],
},
update: { plainText: `text with ${needle}`, markdown: needle, html: `<p>${needle}</p>` },
});
await prisma.$executeRawUnsafe(
`UPDATE page_content_cache SET search_vector = to_tsvector('simple', plain_text) WHERE page_id = $1`,
pageAId,
);
await prisma.pageUpdate.create({
data: { pageId: pageAId, seq: 1, update: new Uint8Array([1, 2, 3]) },
});
await prisma.pageVersion.create({
data: {
pageId: pageAId,
ydocSnapshot: new Uint8Array(),
trigger: 'MANUAL',
createdBy: ids.owner!,
},
});
await prisma.comment.create({
data: { pageId: pageAId, authorId: ids.owner!, body: 'purge comment', anchor: null },
});
const label = await prisma.label.create({
data: { pondId, name: `purge-label-${suffix}`, color: '#00aa00' },
});
await prisma.pageLabel.create({ data: { pageId: pageAId, labelId: label.id } });
await prisma.pageFavorite.create({ data: { pageId: pageAId, userId: ids.owner! } });
// The API page creation auto-watched page A already (autoWatchOwnPages).
await prisma.watch.createMany({
data: [
{ userId: ids.owner!, targetType: 'PAGE', targetId: pageAId },
{ userId: ids.owner!, targetType: 'POND', targetId: pondId },
],
skipDuplicates: true,
});
await prisma.pageLink.create({
data: { fromPageId: pageAId, toPageId: pageB.body.id, targetSlug: pageB.body.slug },
});
await prisma.quotaOverride.create({
data: {
subjectType: 'POND',
subjectId: pondId,
quotaKey: 'storage_bytes',
value: 123456789n,
},
});
});
afterAll(async () => {
const all = Object.values(ids);
await prisma.quotaOverride.deleteMany({ where: { subjectId: { in: [...all, pondId] } } });
await prisma.auditEntry.deleteMany({
where: { OR: [{ actorId: { in: all } }, { targetId: pondId }] },
});
const ponds = await prisma.pond.findMany({
where: { ownerId: { in: all } },
select: { id: true },
});
const pondIds = ponds.map((p) => p.id);
await prisma.attachment.deleteMany({ where: { pondId: { in: pondIds } } });
await prisma.page.deleteMany({ where: { pondId: { in: pondIds } } });
await prisma.label.deleteMany({ where: { pondId: { in: pondIds } } });
await prisma.pond.deleteMany({ where: { id: { in: pondIds } } });
await prisma.watch.deleteMany({ where: { userId: { in: all } } });
await prisma.session.deleteMany({ where: { userId: { in: all } } });
await prisma.userIdentity.deleteMany({ where: { userId: { in: all } } });
await prisma.user.deleteMany({ where: { id: { in: all } } });
await prisma.$disconnect();
await app.close();
});
it('purges a trashed pond without leaving anything behind', async () => {
// Pre-flight: the content is findable and the file exists on disk.
const hitsBefore = await api()
.get(`/api/v1/search?q=${needle}`)
.set('Cookie', cookies.owner!)
.expect(200);
expect(hitsBefore.body.length).toBeGreaterThan(0);
const filePath = join(process.env.UPLOADS_DIR!, pondId, fileId);
expect(existsSync(filePath)).toBe(true);
// Only a TRASHED pond can be purged, and only by a Site Admin.
await api().delete(`/api/v1/ponds/${pondId}/purge`).set('Cookie', cookies.admin!).expect(404);
await api().delete(`/api/v1/ponds/${pondId}`).set('Cookie', cookies.owner!).expect(204);
await api().delete(`/api/v1/ponds/${pondId}/purge`).set('Cookie', cookies.owner!).expect(403);
await api().delete(`/api/v1/ponds/${pondId}/purge`).set('Cookie', cookies.admin!).expect(204);
// Nothing referencing the pond survives.
expect(await prisma.pond.findUnique({ where: { id: pondId } })).toBeNull();
expect(await prisma.page.count({ where: { pondId } })).toBe(0);
expect(await prisma.attachment.count({ where: { pondId } })).toBe(0);
expect(await prisma.label.count({ where: { pondId } })).toBe(0);
expect(await prisma.roleGrant.count({ where: { pondId } })).toBe(0);
expect(await prisma.pondUsage.count({ where: { pondId } })).toBe(0);
expect(await prisma.pageContentCache.count({ where: { pageId: pageAId } })).toBe(0);
expect(await prisma.pageUpdate.count({ where: { pageId: pageAId } })).toBe(0);
expect(await prisma.pageVersion.count({ where: { pageId: pageAId } })).toBe(0);
expect(await prisma.comment.count({ where: { pageId: pageAId } })).toBe(0);
expect(await prisma.pageLabel.count({ where: { pageId: pageAId } })).toBe(0);
expect(await prisma.pageFavorite.count({ where: { pageId: pageAId } })).toBe(0);
expect(await prisma.pageLink.count({ where: { fromPageId: pageAId } })).toBe(0);
expect(
await prisma.watch.count({
where: { targetId: { in: [pondId, pageAId] } },
}),
).toBe(0);
expect(
await prisma.quotaOverride.count({ where: { subjectType: 'POND', subjectId: pondId } }),
).toBe(0);
expect(existsSync(filePath)).toBe(false);
// The follow-up search finds nothing of the purged pond.
const hitsAfter = await api()
.get(`/api/v1/search?q=${needle}`)
.set('Cookie', cookies.owner!)
.expect(200);
expect(hitsAfter.body).toEqual([]);
// Idempotent: a second purge is a clean 404, not an error.
await api().delete(`/api/v1/ponds/${pondId}/purge`).set('Cookie', cookies.admin!).expect(404);
// The manual purge left an audit event.
const audit = await prisma.auditEntry.findFirst({
where: { action: 'pond.purged', targetId: pondId },
});
expect(audit).not.toBeNull();
expect(audit!.details).toMatchObject({ trigger: 'manual', pages: 2, attachments: 1 });
});
it('purges due ponds on the retention path with an audit event', async () => {
const trashedLongAgo = await prisma.pond.create({
data: {
slug: `pp-ret-${suffix}`,
name: 'Retention Pond',
type: 'SHARED',
ownerId: ids.owner!,
deletedAt: new Date('2020-01-01T00:00:00Z'),
deletedBy: ids.owner!,
},
});
await prisma.page.create({
data: {
pondId: trashedLongAgo.id,
slug: `pp-ret-page-${suffix}`,
title: 'Retention Page',
createdBy: ids.owner!,
sortKey: 'a0',
ydocState: new Uint8Array(),
},
});
await app.get(TrashService).purgeDuePonds();
expect(await prisma.pond.findUnique({ where: { id: trashedLongAgo.id } })).toBeNull();
expect(await prisma.page.count({ where: { pondId: trashedLongAgo.id } })).toBe(0);
const audit = await prisma.auditEntry.findFirst({
where: { action: 'pond.purged', targetId: trashedLongAgo.id },
});
expect(audit).not.toBeNull();
expect(audit!.details).toMatchObject({ trigger: 'retention' });
});
});

View File

@ -1,12 +1,13 @@
import { Controller, Delete, Get, HttpCode, Param, Post, Req } from '@nestjs/common';
import { Controller, Delete, Get, HttpCode, Param, Post, Req, UseGuards } from '@nestjs/common';
import { PageView } from '@dorfteich/shared';
import { SiteAdminGuard } from '../admin/site-admin.guard';
import { AuthedRequest } from '../auth/auth.guard';
import { RequiresPagePermission, RequiresPondRole } from '../permissions/permission.decorators';
import { TrashService } from './trash.service';
/** Page trash: list, restore, purge-single (issue #31). */
/** Page trash: list, restore, purge-single (issue #31); pond purge (#193). */
@Controller()
export class TrashController {
constructor(private readonly trash: TrashService) {}
@ -29,4 +30,15 @@ export class TrashController {
async purge(@Param('id') id: string, @Req() request: AuthedRequest): Promise<void> {
await this.trash.purgeNow(request.user!, id);
}
/**
* Manual pond purge (issue #193) Site-Admin-only, like pond restore:
* the pond trash is a Site-Admin surface (ponds.controller.ts).
*/
@Delete('ponds/:id/purge')
@HttpCode(204)
@UseGuards(SiteAdminGuard)
async purgePond(@Param('id') id: string, @Req() request: AuthedRequest): Promise<void> {
await this.trash.purgePondNow(request.user!, id);
}
}

View File

@ -38,7 +38,11 @@ export class TrashModule implements OnModuleInit {
this.scheduler.register({
name: 'trash-purge',
cadenceSeconds: TRASH_PURGE_CADENCE_SECONDS,
run: () => this.trash.purgeDuePages(),
// Pages first, then whole ponds (issue #193) — same retention clock.
run: async () => {
await this.trash.purgeDuePages();
await this.trash.purgeDuePonds();
},
});
}
}

View File

@ -3,6 +3,7 @@ import { PageView } from '@dorfteich/shared';
import { User } from '@prisma/client';
import { PinoLogger } from 'nestjs-pino';
import { AuditService } from '../audit/audit.service';
import { ClockService } from '../common/clock.service';
import { PagesService } from '../pages/pages.service';
import { PermissionService } from '../permissions/permission.service';
@ -32,6 +33,7 @@ export class TrashService {
private readonly storage: FileStorageService,
private readonly clock: ClockService,
private readonly watches: WatchesService,
private readonly audit: AuditService,
private readonly logger: PinoLogger,
) {
this.logger.setContext(TrashService.name);
@ -113,6 +115,98 @@ export class TrashService {
}
}
/** Manual pond purge (issue #193) — Site-Admin-only, guarded at the controller. */
async purgePondNow(actor: User, pondId: string): Promise<void> {
const pond = await this.prisma.pond.findFirst({
where: { id: pondId, deletedAt: { not: null } },
});
if (!pond) throw new NotFoundException();
const counts = await this.purgePond(pondId);
if (!counts) throw new NotFoundException(); // restored or raced away meanwhile
await this.audit.record({
action: 'pond.purged',
actorId: actor.id,
targetType: 'pond',
targetId: pondId,
details: { trigger: 'manual', ...counts },
});
}
/** Scheduled half of the pond purge (issue #193) — same retention as pages. */
async purgeDuePonds(): Promise<void> {
const retentionDays = await this.settings.get('trash.retentionDays');
const cutoff = new Date(this.clock.now().getTime() - retentionDays * MS_PER_DAY);
const due = await this.prisma.pond.findMany({
where: { deletedAt: { lte: cutoff } },
select: { id: true },
});
for (const { id } of due) {
const counts = await this.purgePond(id);
if (counts) {
await this.audit.record({
action: 'pond.purged',
targetType: 'pond',
targetId: id,
details: { trigger: 'retention', ...counts },
});
}
}
}
/**
* Deletes a trashed pond with everything it holds (issue #193). Files go
* first `rm(force)` is idempotent, so a crash between files and rows
* leaves a resumable state (rows intact, next run retries). The rows go
* in ONE transaction, ordered around the FK actions: attachments and
* labels are `Restrict` against the pond and must precede it; the page
* delete cascades versions, comments, content cache (incl. the search
* vector), update log, mentions, pending contributors, label
* assignments, favorites, outgoing links, and open collab sessions; the
* pond delete cascades grants, usage counters (that IS the quota
* correction pond capacity derives from live pond rows), pond-plugin
* opt-ins, and conversion jobs. Watches and quota overrides are
* polymorphic (no FK) and are deleted explicitly. A purge racing a
* restore or another purge is a no-op (`null`).
*/
private async purgePond(pondId: string): Promise<{ pages: number; attachments: number } | null> {
const pond = await this.prisma.pond.findUnique({ where: { id: pondId } });
if (!pond || !pond.deletedAt) return null;
const attachments = await this.prisma.attachment.findMany({
where: { pondId },
select: { id: true },
});
for (const attachment of attachments) {
await this.storage.delete(pondId, attachment.id);
}
const pageIds = (
await this.prisma.page.findMany({ where: { pondId }, select: { id: true } })
).map((page) => page.id);
await this.prisma.$transaction([
this.prisma.attachment.deleteMany({ where: { pondId } }),
this.prisma.watch.deleteMany({
where: {
OR: [
{ targetType: 'POND', targetId: pondId },
{ targetType: 'PAGE', targetId: { in: pageIds } },
],
},
}),
this.prisma.quotaOverride.deleteMany({
where: { subjectType: 'POND', subjectId: pondId },
}),
this.prisma.page.deleteMany({ where: { pondId } }),
this.prisma.label.deleteMany({ where: { pondId } }),
this.prisma.pond.delete({ where: { id: pondId } }),
]);
this.logger.info(
{ pondId, pages: pageIds.length, attachments: attachments.length },
'audit: pond purged',
);
return { pages: pageIds.length, attachments: attachments.length };
}
/**
* Deletes state, content cache, files, and (once M3 exists) versions for
* one page the fixed set of things `#31`'s scope names. Version rows

View File

@ -110,6 +110,17 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack.
Job outcomes are visible in the Site Admin UI (last run, status) — that
panel is the operator's single glance for instance health.
Pond purge (issue #193): a trashed pond past the trash retention is
removed with everything it holds — pages (cascading versions, comments,
content cache incl. the search vector, update log, mentions, label
assignments, favorites, outgoing links, open collab sessions),
attachments (rows and files on disk), labels, grants, usage counters,
watches, pond quota overrides, pond-plugin opt-ins, and conversion jobs.
A Site Admin can purge a trashed pond immediately
(`DELETE /ponds/:id/purge`); both paths record a `pond.purged` audit
event. Known residues by design: `page_links.target_slug` in OTHER
ponds' pages (#235) and backups within their retention.
## Update strategy
- **Own stages**: pipeline-driven (see `deployment.md`); Prod only via

View File

@ -99,7 +99,7 @@ chain`_
abschaltbar · 2 AT · #191
- [x] **Backup-Ziele einschränkbar** — Allowlist, WebDAV/rsync per Deploy
vollständig deaktivierbar · 2 AT · #192
- [ ] **Pond-Purge implementieren** — getrashte Ponds bleiben ewig liegen · 3 AT · #193
- [x] **Pond-Purge implementieren** — getrashte Ponds bleiben ewig liegen · 3 AT · #193
- [ ] **Orphan-File-Sweep** implementieren, `Attachment.deletedAt` nutzen
oder entfernen · 2 AT · #194
- [ ] **Papierkorb aus dem Suchindex** entfernen statt query-seitig filtern · 2 AT · #195