#195: trashed content leaves the search index itself #250
@ -0,0 +1,10 @@
|
||||
-- Issue #195, one-off backfill: the full-text index must hold no trashed
|
||||
-- content. Clears the search vector of every page that is trashed itself
|
||||
-- or lives in a trashed pond; the application keeps this invariant from
|
||||
-- now on (trash hooks + reindex paths).
|
||||
UPDATE page_content_cache c
|
||||
SET search_vector = NULL
|
||||
FROM pages p
|
||||
LEFT JOIN ponds po ON po.id = p.pond_id
|
||||
WHERE p.id = c.page_id
|
||||
AND (p.deleted_at IS NOT NULL OR po.deleted_at IS NOT NULL);
|
||||
@ -4,6 +4,7 @@ import { AuthModule } from '../auth/auth.module';
|
||||
import { BackupModule } from '../backup/backup.module';
|
||||
import { QuotasModule } from '../quotas/quotas.module';
|
||||
import { SchedulerModule } from '../scheduler/scheduler.module';
|
||||
import { SearchModule } from '../search/search.module';
|
||||
import { UsersModule } from '../users/users.module';
|
||||
|
||||
import { AdminSettingsController } from './admin.controller';
|
||||
@ -18,7 +19,7 @@ import { UserAdminController } from './user-admin.controller';
|
||||
import { UserAdminService } from './user-admin.service';
|
||||
|
||||
@Module({
|
||||
imports: [QuotasModule, UsersModule, AuthModule, SchedulerModule, BackupModule],
|
||||
imports: [QuotasModule, UsersModule, AuthModule, SchedulerModule, BackupModule, SearchModule],
|
||||
controllers: [
|
||||
AdminSettingsController,
|
||||
BackupAdminController,
|
||||
|
||||
@ -4,6 +4,7 @@ import { PinoLogger } from 'nestjs-pino';
|
||||
|
||||
import { AuditService } from '../audit/audit.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { SearchProvider } from '../search/search.provider';
|
||||
|
||||
/**
|
||||
* GDPR account deletion (issue #59, security.md §Privacy). Rather than
|
||||
@ -17,6 +18,7 @@ export class PseudonymizationService {
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly audit: AuditService,
|
||||
private readonly search: SearchProvider,
|
||||
private readonly logger: PinoLogger,
|
||||
) {
|
||||
this.logger.setContext(PseudonymizationService.name);
|
||||
@ -45,6 +47,14 @@ export class PseudonymizationService {
|
||||
data: { deletedAt: new Date(), deletedBy: userId },
|
||||
});
|
||||
});
|
||||
// Trash path includes leaving the search index (issue #195).
|
||||
const personalPonds = await this.prisma.pond.findMany({
|
||||
where: { ownerId: userId, type: 'PERSONAL' },
|
||||
select: { id: true },
|
||||
});
|
||||
for (const pond of personalPonds) {
|
||||
await this.search.removePond(pond.id);
|
||||
}
|
||||
await this.audit.record({
|
||||
action: 'user.pseudonymized',
|
||||
targetType: 'user',
|
||||
|
||||
@ -588,6 +588,12 @@ export class PagesService {
|
||||
});
|
||||
}
|
||||
});
|
||||
// Trashed content leaves the search index (issue #195); the query-side
|
||||
// deleted_at guards in the provider stay as the second layer.
|
||||
const trashedIds = mode === 'subtree' ? [page.id, ...descendantIds] : [page.id];
|
||||
for (const trashedId of trashedIds) {
|
||||
await this.search.removePage(trashedId);
|
||||
}
|
||||
this.logger.info(
|
||||
{
|
||||
pageId: id,
|
||||
|
||||
@ -1,13 +1,14 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { QuotasModule } from '../quotas/quotas.module';
|
||||
import { SearchModule } from '../search/search.module';
|
||||
|
||||
import { PondAccessNotifier } from './pond-access-notifier.service';
|
||||
import { PondsController } from './ponds.controller';
|
||||
import { PondsService } from './ponds.service';
|
||||
|
||||
@Module({
|
||||
imports: [QuotasModule],
|
||||
imports: [QuotasModule, SearchModule],
|
||||
controllers: [PondsController],
|
||||
providers: [PondsService, PondAccessNotifier],
|
||||
exports: [PondsService, PondAccessNotifier],
|
||||
|
||||
@ -12,6 +12,7 @@ import { PinoLogger } from 'nestjs-pino';
|
||||
import { PermissionService } from '../permissions/permission.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { QuotaService } from '../quotas/quota.service';
|
||||
import { SearchProvider } from '../search/search.provider';
|
||||
import { PondAccessNotifier } from './pond-access-notifier.service';
|
||||
|
||||
@Injectable()
|
||||
@ -21,6 +22,7 @@ export class PondsService {
|
||||
private readonly permissions: PermissionService,
|
||||
private readonly quotas: QuotaService,
|
||||
private readonly accessNotifier: PondAccessNotifier,
|
||||
private readonly search: SearchProvider,
|
||||
private readonly logger: PinoLogger,
|
||||
) {
|
||||
this.logger.setContext(PondsService.name);
|
||||
@ -186,6 +188,9 @@ export class PondsService {
|
||||
where: { id },
|
||||
data: { deletedAt: new Date(), deletedBy: user.id },
|
||||
});
|
||||
// The whole pond leaves the search index (issue #195); the query-side
|
||||
// deleted_at guards in the provider stay as the second layer.
|
||||
await this.search.removePond(id);
|
||||
this.logger.info({ pondId: id, userId: user.id }, 'audit: pond trashed');
|
||||
// Revalidate any live collaboration sessions on the pond's pages
|
||||
// (issue #39); grant changes fire the same notification (#52/#53).
|
||||
@ -207,6 +212,9 @@ export class PondsService {
|
||||
where: { id },
|
||||
data: { deletedAt: null, deletedBy: null },
|
||||
});
|
||||
// Live pages return to the search index; pages trashed inside the pond
|
||||
// stay out (issue #195).
|
||||
await this.search.reindexPond(id);
|
||||
this.logger.info({ pondId: id }, 'audit: pond restored');
|
||||
return this.viewOf(pond);
|
||||
}
|
||||
|
||||
@ -73,14 +73,42 @@ export class PostgresSearchProvider extends SearchProvider {
|
||||
.$executeRaw`UPDATE page_content_cache SET search_vector = NULL WHERE page_id = ${pageId}`;
|
||||
}
|
||||
|
||||
async reindexAll(): Promise<number> {
|
||||
async removePond(pondId: string): Promise<void> {
|
||||
await this.prisma.$executeRaw`
|
||||
UPDATE page_content_cache SET search_vector = NULL
|
||||
WHERE page_id IN (SELECT id FROM pages WHERE pond_id = ${pondId})`;
|
||||
}
|
||||
|
||||
async reindexPond(pondId: string): Promise<number> {
|
||||
const sources = await this.prisma.$queryRaw<(IndexSource & { page_id: string })[]>`
|
||||
SELECT p.id AS page_id, p.title, c.plain_text,
|
||||
(SELECT string_agg(l.name, ' ')
|
||||
FROM page_labels pl JOIN labels l ON l.id = pl.label_id
|
||||
WHERE pl.page_id = p.id) AS labels
|
||||
FROM page_content_cache c
|
||||
JOIN pages p ON p.id = c.page_id`;
|
||||
JOIN pages p ON p.id = c.page_id
|
||||
WHERE p.pond_id = ${pondId} AND p.deleted_at IS NULL`;
|
||||
for (const source of sources) await this.writeVector(source.page_id, source);
|
||||
return sources.length;
|
||||
}
|
||||
|
||||
async reindexAll(): Promise<number> {
|
||||
// Converge to the issue-#195 invariant: trashed content leaves the
|
||||
// index entirely, live content is rebuilt.
|
||||
await this.prisma.$executeRaw`
|
||||
UPDATE page_content_cache c SET search_vector = NULL
|
||||
FROM pages p LEFT JOIN ponds po ON po.id = p.pond_id
|
||||
WHERE p.id = c.page_id
|
||||
AND (p.deleted_at IS NOT NULL OR po.deleted_at IS NOT NULL)`;
|
||||
const sources = await this.prisma.$queryRaw<(IndexSource & { page_id: string })[]>`
|
||||
SELECT p.id AS page_id, p.title, c.plain_text,
|
||||
(SELECT string_agg(l.name, ' ')
|
||||
FROM page_labels pl JOIN labels l ON l.id = pl.label_id
|
||||
WHERE pl.page_id = p.id) AS labels
|
||||
FROM page_content_cache c
|
||||
JOIN pages p ON p.id = c.page_id
|
||||
JOIN ponds po ON po.id = p.pond_id AND po.deleted_at IS NULL
|
||||
WHERE p.deleted_at IS NULL`;
|
||||
for (const source of sources) await this.writeVector(source.page_id, source);
|
||||
return sources.length;
|
||||
}
|
||||
|
||||
176
apps/api/src/search/search-trash.e2e.db.test.ts
Normal file
176
apps/api/src/search/search-trash.e2e.db.test.ts
Normal file
@ -0,0 +1,176 @@
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import request from 'supertest';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
import { createTestApp, sessionCookieOf } from '../testing/test-app';
|
||||
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
|
||||
import { UsersService } from '../users/users.service';
|
||||
import { SearchProvider } from './search.provider';
|
||||
|
||||
/**
|
||||
* Trash keeps content out of the search index itself (issue #195): the
|
||||
* vector rows are cleared on page/pond trash and rebuilt on restore, and
|
||||
* the query-side deleted_at guards stay as an INDEPENDENT second layer —
|
||||
* proven by writing a vector back onto a trashed page and asserting the
|
||||
* query still returns nothing.
|
||||
*/
|
||||
describe.skipIf(!hasTestDb)('search index vs. trash (e2e, issue #195)', () => {
|
||||
let app: INestApplication;
|
||||
let prisma: PrismaClient;
|
||||
const suffix = uniqueSuffix();
|
||||
const password = 'search trash pass 1';
|
||||
const ids: Record<string, string> = {};
|
||||
const cookies: Record<string, string> = {};
|
||||
let pondId: string;
|
||||
let pageId: string;
|
||||
let childId: string;
|
||||
const needle = `zzsearchtrash${suffix.replaceAll('-', '')}`;
|
||||
|
||||
const api = () => request(app.getHttpServer());
|
||||
|
||||
const vectorOf = async (id: string): Promise<string | null> => {
|
||||
const rows = await prisma.$queryRaw<{ v: string | null }[]>`
|
||||
SELECT search_vector::text AS v FROM page_content_cache WHERE page_id = ${id}`;
|
||||
return rows[0]?.v ?? null;
|
||||
};
|
||||
|
||||
const hits = async (): Promise<unknown[]> => {
|
||||
const res = await api()
|
||||
.get(`/api/v1/search?q=${needle}`)
|
||||
.set('Cookie', cookies.owner!)
|
||||
.expect(200);
|
||||
return res.body as unknown[];
|
||||
};
|
||||
|
||||
async function seedContent(id: string, text: string): Promise<void> {
|
||||
await prisma.pageContentCache.upsert({
|
||||
where: { pageId: id },
|
||||
create: { pageId: id, plainText: text, markdown: text, html: `<p>${text}</p>`, outline: [] },
|
||||
update: { plainText: text, markdown: text, html: `<p>${text}</p>` },
|
||||
});
|
||||
await app.get(SearchProvider).indexPage(id);
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
prisma = createTestPrisma();
|
||||
await prisma.rateLimit.deleteMany({});
|
||||
app = await createTestApp();
|
||||
const users = app.get(UsersService);
|
||||
for (const handle of ['owner', 'admin'] as const) {
|
||||
const username = `st-${handle}-${suffix}`;
|
||||
const user = await users.createUser({
|
||||
username,
|
||||
email: `${username}@example.org`,
|
||||
displayName: `Search ${handle}`,
|
||||
password,
|
||||
locale: 'en',
|
||||
});
|
||||
ids[handle] = user.id;
|
||||
await users.markEmailVerified(user.id);
|
||||
if (handle === 'admin') {
|
||||
await prisma.user.update({ where: { id: user.id }, data: { isSiteAdmin: true } });
|
||||
}
|
||||
cookies[handle] = sessionCookieOf(
|
||||
await api()
|
||||
.post('/api/v1/auth/login')
|
||||
.send({ usernameOrEmail: username, password })
|
||||
.expect(200),
|
||||
);
|
||||
}
|
||||
await api()
|
||||
.put(`/api/v1/admin/quotas/user/${ids.owner!}/additional_ponds`)
|
||||
.set('Cookie', cookies.admin!)
|
||||
.send({ value: 5 })
|
||||
.expect(200);
|
||||
const pond = await api()
|
||||
.post('/api/v1/ponds')
|
||||
.set('Cookie', cookies.owner!)
|
||||
.send({ name: `Search Trash Pond ${suffix}` })
|
||||
.expect(201);
|
||||
pondId = pond.body.id;
|
||||
const page = await api()
|
||||
.post(`/api/v1/ponds/${pondId}/pages`)
|
||||
.set('Cookie', cookies.owner!)
|
||||
.send({ title: `Search Trash Page ${suffix}` })
|
||||
.expect(201);
|
||||
pageId = page.body.id;
|
||||
const child = await api()
|
||||
.post(`/api/v1/ponds/${pondId}/pages`)
|
||||
.set('Cookie', cookies.owner!)
|
||||
.send({ title: `Search Trash Child ${suffix}`, parentId: pageId })
|
||||
.expect(201);
|
||||
childId = child.body.id;
|
||||
await seedContent(pageId, `parent text ${needle}`);
|
||||
await seedContent(childId, `child text ${needle}`);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
const all = Object.values(ids);
|
||||
await prisma.quotaOverride.deleteMany({ where: { subjectId: { in: all } } });
|
||||
await prisma.auditEntry.deleteMany({ where: { actorId: { in: all } } });
|
||||
const ponds = await prisma.pond.findMany({
|
||||
where: { ownerId: { in: all } },
|
||||
select: { id: true },
|
||||
});
|
||||
const pondIds = ponds.map((p) => p.id);
|
||||
await prisma.page.deleteMany({ where: { pondId: { in: pondIds } } });
|
||||
await prisma.pond.deleteMany({ where: { id: { in: pondIds } } });
|
||||
await prisma.watch.deleteMany({ where: { userId: { in: all } } });
|
||||
await prisma.session.deleteMany({ where: { userId: { in: all } } });
|
||||
await prisma.userIdentity.deleteMany({ where: { userId: { in: all } } });
|
||||
await prisma.user.deleteMany({ where: { id: { in: all } } });
|
||||
await prisma.$disconnect();
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it('clears the vector rows on subtree trash and rebuilds them on restore', async () => {
|
||||
expect(await vectorOf(pageId)).toContain(needle.toLowerCase());
|
||||
expect((await hits()).length).toBeGreaterThan(0);
|
||||
|
||||
await api()
|
||||
.delete(`/api/v1/pages/${pageId}?mode=subtree`)
|
||||
.set('Cookie', cookies.owner!)
|
||||
.expect(204);
|
||||
|
||||
// Layer 1: the index rows themselves hold nothing.
|
||||
expect(await vectorOf(pageId)).toBeNull();
|
||||
expect(await vectorOf(childId)).toBeNull();
|
||||
expect(await hits()).toEqual([]);
|
||||
|
||||
await api().post(`/api/v1/pages/${pageId}/restore`).set('Cookie', cookies.owner!).expect(201);
|
||||
expect(await vectorOf(pageId)).toContain(needle.toLowerCase());
|
||||
// The child stays trashed — and stays out of the index.
|
||||
expect(await vectorOf(childId)).toBeNull();
|
||||
expect((await hits()).length).toBe(1);
|
||||
await api().post(`/api/v1/pages/${childId}/restore`).set('Cookie', cookies.owner!).expect(201);
|
||||
expect((await hits()).length).toBe(2);
|
||||
});
|
||||
|
||||
it('keeps the query-side guard as an independent second layer', async () => {
|
||||
await api().delete(`/api/v1/pages/${childId}`).set('Cookie', cookies.owner!).expect(204);
|
||||
expect(await vectorOf(childId)).toBeNull();
|
||||
// Simulate a future code path that forgot to clear the vector.
|
||||
await prisma.$executeRaw`
|
||||
UPDATE page_content_cache SET search_vector = to_tsvector('simple', plain_text)
|
||||
WHERE page_id = ${childId}`;
|
||||
expect(await vectorOf(childId)).not.toBeNull();
|
||||
// The deleted_at join still hides it.
|
||||
expect((await hits()).length).toBe(1);
|
||||
await api().post(`/api/v1/pages/${childId}/restore`).set('Cookie', cookies.owner!).expect(201);
|
||||
});
|
||||
|
||||
it('clears every page vector on pond trash and reindexes live pages on restore', async () => {
|
||||
// One page goes into the page trash first — it must stay out after
|
||||
// the pond comes back.
|
||||
await api().delete(`/api/v1/pages/${childId}`).set('Cookie', cookies.owner!).expect(204);
|
||||
await api().delete(`/api/v1/ponds/${pondId}`).set('Cookie', cookies.owner!).expect(204);
|
||||
expect(await vectorOf(pageId)).toBeNull();
|
||||
expect(await vectorOf(childId)).toBeNull();
|
||||
|
||||
await api().post(`/api/v1/ponds/${pondId}/restore`).set('Cookie', cookies.admin!).expect(201);
|
||||
expect(await vectorOf(pageId)).toContain(needle.toLowerCase());
|
||||
expect(await vectorOf(childId)).toBeNull();
|
||||
expect((await hits()).length).toBe(1);
|
||||
});
|
||||
});
|
||||
@ -28,6 +28,8 @@ describe('SearchProvider DI seam (issue #49)', () => {
|
||||
const fake: SearchProvider = {
|
||||
indexPage: vi.fn(),
|
||||
removePage: vi.fn(),
|
||||
removePond: vi.fn(),
|
||||
reindexPond: vi.fn(),
|
||||
reindexAll: vi.fn(),
|
||||
search: vi.fn().mockResolvedValue([hit]),
|
||||
};
|
||||
|
||||
@ -13,8 +13,19 @@ export abstract class SearchProvider {
|
||||
abstract indexPage(pageId: string): Promise<void>;
|
||||
/** Drop a page from the index (its cache row is also removed on purge). */
|
||||
abstract removePage(pageId: string): Promise<void>;
|
||||
/** Drop every page of a pond from the index — pond trash (issue #195). */
|
||||
abstract removePond(pondId: string): Promise<void>;
|
||||
/**
|
||||
* Rebuild the entries of a pond's LIVE pages — pond restore (issue #195).
|
||||
* Pages trashed inside the pond stay out of the index.
|
||||
*/
|
||||
abstract reindexPond(pondId: string): Promise<number>;
|
||||
/** Ranked, permission-filtered results for `user`. */
|
||||
abstract search(query: SearchQuery, user: User): Promise<SearchResultView[]>;
|
||||
/** Rebuild the whole index from `page_content_cache`; returns rows indexed. */
|
||||
/**
|
||||
* Converge the whole index: rebuild entries for live pages of live ponds
|
||||
* and clear every trashed page's vector — the index holds no trashed
|
||||
* content (issue #195). Returns rows indexed.
|
||||
*/
|
||||
abstract reindexAll(): Promise<number>;
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import { FilesModule } from '../files/files.module';
|
||||
import { PagesModule } from '../pages/pages.module';
|
||||
import { PondsModule } from '../ponds/ponds.module';
|
||||
import { QuotasModule } from '../quotas/quotas.module';
|
||||
import { SearchModule } from '../search/search.module';
|
||||
import { WatchesModule } from '../watches/watches.module';
|
||||
import { SchedulerModule } from '../scheduler/scheduler.module';
|
||||
import { SchedulerService } from '../scheduler/scheduler.service';
|
||||
@ -23,6 +24,7 @@ const TRASH_PURGE_CADENCE_SECONDS = 24 * 60 * 60;
|
||||
FilesModule,
|
||||
PagesModule,
|
||||
SchedulerModule,
|
||||
SearchModule,
|
||||
WatchesModule,
|
||||
],
|
||||
controllers: [TrashController],
|
||||
|
||||
@ -5,6 +5,7 @@ import { PinoLogger } from 'nestjs-pino';
|
||||
|
||||
import { AuditService } from '../audit/audit.service';
|
||||
import { ClockService } from '../common/clock.service';
|
||||
import { SearchProvider } from '../search/search.provider';
|
||||
import { PagesService } from '../pages/pages.service';
|
||||
import { PermissionService } from '../permissions/permission.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
@ -34,6 +35,7 @@ export class TrashService {
|
||||
private readonly clock: ClockService,
|
||||
private readonly watches: WatchesService,
|
||||
private readonly audit: AuditService,
|
||||
private readonly search: SearchProvider,
|
||||
private readonly logger: PinoLogger,
|
||||
) {
|
||||
this.logger.setContext(TrashService.name);
|
||||
@ -90,6 +92,8 @@ export class TrashService {
|
||||
where: { id },
|
||||
data: { deletedAt: null, deletedBy: null, parentId },
|
||||
});
|
||||
// Back into the search index (issue #195) — trash had cleared its vector.
|
||||
await this.search.indexPage(id);
|
||||
this.logger.info({ pageId: id, userId: user.id, parentId }, 'audit: page restored from trash');
|
||||
return this.pages.viewOf(restored);
|
||||
}
|
||||
|
||||
@ -64,6 +64,14 @@ or sloppy plugin authors, compromised dependencies.
|
||||
- App CSP (strict): `default-src 'self'`; `font-src 'self'` (ADR 0016);
|
||||
no third-party origins at all — the GDPR posture is "zero external
|
||||
requests".
|
||||
- The full-text index holds **no trashed content** (issue #195): trashing
|
||||
a page or pond clears the affected `search_vector`s, restore rebuilds
|
||||
them, `reindexAll` converges to the same invariant, and a one-off
|
||||
migration backfilled pre-existing trash. The query-side
|
||||
`deleted_at IS NULL` joins stay in place as the second, independent
|
||||
layer — a future query path that forgets them still finds no trashed
|
||||
vectors. (The plaintext cache row itself remains until purge; the index
|
||||
is the concern here because it is queryable.)
|
||||
|
||||
## Plugin sandboxing (ADR 0008, operational)
|
||||
|
||||
|
||||
@ -102,7 +102,7 @@ chain`_
|
||||
- [x] **Pond-Purge implementieren** — getrashte Ponds bleiben ewig liegen · 3 AT · #193
|
||||
- [x] **Orphan-File-Sweep** implementieren, `Attachment.deletedAt` nutzen
|
||||
oder entfernen · 2 AT · #194
|
||||
- [ ] **Papierkorb aus dem Suchindex** entfernen statt query-seitig filtern · 2 AT · #195
|
||||
- [x] **Papierkorb aus dem Suchindex** entfernen statt query-seitig filtern · 2 AT · #195
|
||||
- [ ] **Retention-Job für `audit_log`** · 1 AT · #196
|
||||
- [ ] **Security-Header** (helmet), CORS explizit restriktiv · 1 AT · #197
|
||||
- [ ] **SBOM in CI** (CycloneDX/syft) + Lizenzreport als Artefakt · 1–2 AT · #202
|
||||
|
||||
Loading…
Reference in New Issue
Block a user