#234: retention for mail_outbox #255
104
apps/api/src/mail/mail-outbox-retention.e2e.db.test.ts
Normal file
104
apps/api/src/mail/mail-outbox-retention.e2e.db.test.ts
Normal file
@ -0,0 +1,104 @@
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
import { createTestApp } from '../testing/test-app';
|
||||
import { createTestPrisma, hasTestDb, uniqueSuffix } from '../testing/test-db';
|
||||
|
||||
import { MailRetentionService } from './mail-retention.service';
|
||||
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Mail outbox retention (issue #234): SENT and permanently FAILED rows past
|
||||
* `mail.outboxRetentionDays` are deleted; PENDING rows — including a
|
||||
* failed-but-retryable one — belong to the retry loop and stay untouched.
|
||||
*/
|
||||
describe.skipIf(!hasTestDb)('mail outbox retention (e2e, issue #234)', () => {
|
||||
let app: INestApplication;
|
||||
let prisma: PrismaClient;
|
||||
const suffix = uniqueSuffix();
|
||||
const address = (name: string) => `${name}-${suffix}@example.org`;
|
||||
|
||||
beforeAll(async () => {
|
||||
prisma = createTestPrisma();
|
||||
// A short period so ages are unambiguous; written straight to the row
|
||||
// BEFORE the app boots (the settings cache is in-process and fills on
|
||||
// first read). The key is cleaned afterAll.
|
||||
await prisma.instanceSetting.upsert({
|
||||
where: { key: 'mail.outboxRetentionDays' },
|
||||
create: { key: 'mail.outboxRetentionDays', value: 10 },
|
||||
update: { value: 10 },
|
||||
});
|
||||
app = await createTestApp();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await prisma.instanceSetting.deleteMany({ where: { key: 'mail.outboxRetentionDays' } });
|
||||
await prisma.mailOutbox.deleteMany({ where: { toAddress: { contains: suffix } } });
|
||||
await prisma.$disconnect();
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it('deletes terminal rows past the period, keeps the retry loop’s rows', async () => {
|
||||
await prisma.mailOutbox.createMany({
|
||||
data: [
|
||||
{
|
||||
toAddress: address('sent-old'),
|
||||
subject: 's',
|
||||
textBody: 't',
|
||||
htmlBody: 'h',
|
||||
status: 'SENT',
|
||||
sentAt: new Date(Date.now() - 15 * DAY),
|
||||
},
|
||||
{
|
||||
toAddress: address('sent-fresh'),
|
||||
subject: 's',
|
||||
textBody: 't',
|
||||
htmlBody: 'h',
|
||||
status: 'SENT',
|
||||
sentAt: new Date(Date.now() - 5 * DAY),
|
||||
},
|
||||
{
|
||||
// Gave up after MAX_ATTEMPTS; nextAttemptAt records the last try.
|
||||
toAddress: address('failed-old'),
|
||||
subject: 's',
|
||||
textBody: 't',
|
||||
htmlBody: 'h',
|
||||
status: 'FAILED',
|
||||
attempts: 5,
|
||||
nextAttemptAt: new Date(Date.now() - 15 * DAY),
|
||||
lastError: 'smtp gone',
|
||||
},
|
||||
{
|
||||
// Failed once but retryable: still PENDING, still the worker's.
|
||||
toAddress: address('pending-retry'),
|
||||
subject: 's',
|
||||
textBody: 't',
|
||||
htmlBody: 'h',
|
||||
status: 'PENDING',
|
||||
attempts: 2,
|
||||
nextAttemptAt: new Date(Date.now() - 15 * DAY),
|
||||
lastError: 'transient',
|
||||
createdAt: new Date(Date.now() - 15 * DAY),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const pruned = await app.get(MailRetentionService).pruneExpired();
|
||||
expect(pruned).toBeGreaterThanOrEqual(2);
|
||||
|
||||
const remaining = await prisma.mailOutbox.findMany({
|
||||
where: { toAddress: { contains: suffix } },
|
||||
});
|
||||
expect(remaining.map((row) => row.toAddress).sort()).toEqual([
|
||||
address('pending-retry'),
|
||||
address('sent-fresh'),
|
||||
]);
|
||||
});
|
||||
|
||||
it('is a no-op when nothing is due', async () => {
|
||||
expect(await app.get(MailRetentionService).pruneExpired()).toBe(0);
|
||||
expect(await prisma.mailOutbox.count({ where: { toAddress: { contains: suffix } } })).toBe(2);
|
||||
});
|
||||
});
|
||||
49
apps/api/src/mail/mail-retention.service.ts
Normal file
49
apps/api/src/mail/mail-retention.service.ts
Normal file
@ -0,0 +1,49 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { PinoLogger } from 'nestjs-pino';
|
||||
|
||||
import { ClockService } from '../common/clock.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { InstanceSettingsService } from '../settings/instance-settings.service';
|
||||
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Mail outbox retention (issue #234): the daily job deletes outbox rows that
|
||||
* reached a terminal state longer ago than `mail.outboxRetentionDays` —
|
||||
* SENT rows by their `sentAt`, permanently FAILED rows by their last attempt
|
||||
* (`nextAttemptAt`, written with the final failure). Digest bodies carry
|
||||
* page titles, so a sent mail is content-adjacent data whose copy must be
|
||||
* bounded. PENDING rows are the retry loop's ({@link MailWorker}) alone —
|
||||
* a failed-but-retryable mail stays PENDING and is never deleted here.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailRetentionService {
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly settings: InstanceSettingsService,
|
||||
private readonly clock: ClockService,
|
||||
private readonly logger: PinoLogger,
|
||||
) {
|
||||
this.logger.setContext(MailRetentionService.name);
|
||||
}
|
||||
|
||||
async pruneExpired(): Promise<number> {
|
||||
const retentionDays = await this.settings.get('mail.outboxRetentionDays');
|
||||
const cutoff = new Date(this.clock.now().getTime() - retentionDays * MS_PER_DAY);
|
||||
const { count } = await this.prisma.mailOutbox.deleteMany({
|
||||
where: {
|
||||
OR: [
|
||||
{ status: 'SENT', sentAt: { lt: cutoff } },
|
||||
{ status: 'FAILED', nextAttemptAt: { lt: cutoff } },
|
||||
],
|
||||
},
|
||||
});
|
||||
if (count > 0) {
|
||||
this.logger.info(
|
||||
{ pruned: count, cutoff: cutoff.toISOString(), retentionDays },
|
||||
'audit: mail outbox entries pruned',
|
||||
);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
@ -1,13 +1,24 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { Module, OnModuleInit } from '@nestjs/common';
|
||||
|
||||
import { CommonModule } from '../common/common.module';
|
||||
import { SchedulerModule } from '../scheduler/scheduler.module';
|
||||
import { SchedulerService } from '../scheduler/scheduler.service';
|
||||
import { SettingsModule } from '../settings/settings.module';
|
||||
|
||||
import { MailRetentionService } from './mail-retention.service';
|
||||
import { MAIL_TRANSPORT, MailTransport, MailWorker } from './mail-worker.service';
|
||||
import { MailService } from './mail.service';
|
||||
import { SmtpConfigService } from './smtp-config.service';
|
||||
|
||||
/** Daily, per operations.md's maintenance-jobs table (issue #234). */
|
||||
const OUTBOX_RETENTION_CADENCE_SECONDS = 24 * 60 * 60;
|
||||
|
||||
@Module({
|
||||
imports: [CommonModule, SchedulerModule, SettingsModule],
|
||||
providers: [
|
||||
MailService,
|
||||
MailWorker,
|
||||
MailRetentionService,
|
||||
SmtpConfigService,
|
||||
{
|
||||
// Delegates per send so the wizard's SMTP changes (SmtpConfigService.
|
||||
@ -21,4 +32,19 @@ import { SmtpConfigService } from './smtp-config.service';
|
||||
],
|
||||
exports: [MailService, SmtpConfigService],
|
||||
})
|
||||
export class MailModule {}
|
||||
export class MailModule implements OnModuleInit {
|
||||
constructor(
|
||||
private readonly scheduler: SchedulerService,
|
||||
private readonly retention: MailRetentionService,
|
||||
) {}
|
||||
|
||||
onModuleInit(): void {
|
||||
this.scheduler.register({
|
||||
name: 'mail-outbox-retention',
|
||||
cadenceSeconds: OUTBOX_RETENTION_CADENCE_SECONDS,
|
||||
run: async () => {
|
||||
await this.retention.pruneExpired();
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -45,6 +45,12 @@ export const INSTANCE_SETTINGS = {
|
||||
// before the daily prune job nulls them. The row itself survives for
|
||||
// status/audit purposes; PENDING/RUNNING jobs are never touched.
|
||||
'conversion.payloadRetentionDays': z.number().int().min(1).default(30),
|
||||
// Mail outbox retention (issue #234): days a SENT or permanently FAILED
|
||||
// outbox row is kept before the daily retention job deletes it. Digest
|
||||
// bodies carry page titles (content-adjacent data), so the copy must be
|
||||
// bounded. PENDING rows — including failed-but-retryable ones — are
|
||||
// never touched; the retry loop owns them.
|
||||
'mail.outboxRetentionDays': z.number().int().min(1).default(30),
|
||||
// Non-image upload allowlist (ADR 0011, issue #61): lowercase extensions
|
||||
// without the dot. Images are always allowed regardless; SVG is governed
|
||||
// by `upload.svgPolicy`. Normalized (lowercased, dot-stripped, deduped) so
|
||||
|
||||
@ -20,8 +20,8 @@ test('lists maintenance jobs and triggers one manually', async ({ browser }) =>
|
||||
// Keep in sync with the scheduler registrations: trash-purge,
|
||||
// version-thinning, page-compaction, data-export-purge,
|
||||
// notification-digest, orphan-file-sweep (#194), audit-retention (#196),
|
||||
// conversion-payload-prune (#233).
|
||||
await expect(jobsTable.locator('tbody tr')).toHaveCount(8);
|
||||
// conversion-payload-prune (#233), mail-outbox-retention (#234).
|
||||
await expect(jobsTable.locator('tbody tr')).toHaveCount(9);
|
||||
|
||||
const firstRow = jobsTable.locator('tbody tr').first();
|
||||
await firstRow.getByRole('button').click();
|
||||
|
||||
@ -107,6 +107,7 @@ monitoring, structured logs, backup alerting — no dedicated metrics stack.
|
||||
| orphan file sweep | nightly | volume ↔ DB consistency (ADR 0011) |
|
||||
| audit retention | daily | prune `audit_log` past `audit.retentionDays` (#196) |
|
||||
| conversion payload prune | daily | null finished conversion jobs' raw bytes (#233) |
|
||||
| mail outbox retention | daily | delete sent/failed outbox rows past period (#234) |
|
||||
| mail outbox retry | every minute | e-mail delivery with backoff |
|
||||
|
||||
Job outcomes are visible in the Site Admin UI (last run, status) — that
|
||||
|
||||
@ -183,6 +183,14 @@ job (out of scope, below).
|
||||
- IP addresses appear only in rate-limit counters (short TTL) and reverse
|
||||
proxy logs (host-level rotation) — documented in the privacy-policy
|
||||
template.
|
||||
- Sent mail is not kept forever (issue #234): SENT and permanently FAILED
|
||||
`mail_outbox` rows are deleted after `mail.outboxRetentionDays`
|
||||
(default 30) by a daily job. This bounds the copy of content-adjacent
|
||||
data — digest bodies name page titles and actors. That digest mails
|
||||
carry page titles at all is a recorded, accepted residue (issue #231):
|
||||
there is no per-page classification marking yet to key a suppression
|
||||
on (that lands with ADR 0022 / M32, revisit there), and a VS-NfD
|
||||
reference configuration (#227) can leave SMTP unconfigured entirely.
|
||||
|
||||
## Out of scope (v1, explicit)
|
||||
|
||||
|
||||
@ -251,7 +251,7 @@ fehlten — als Issues angelegt:
|
||||
|
||||
- [x] Conversion-Job-Payloads prunen — Rohbytes jedes Im-/Exports liegen
|
||||
unbefristet in `conversion_jobs` · 2 AT · #233 (M24, I-22)
|
||||
- [ ] Retention für `mail_outbox` — Digest-Mails tragen Seitentitel
|
||||
- [x] Retention für `mail_outbox` — Digest-Mails tragen Seitentitel
|
||||
· 1 AT · #234 (M24, I-23)
|
||||
- [ ] `page_links.target_slug`-Residuum nach Purge entscheiden
|
||||
· 0,5 AT · #235 (M24, I-24)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user