feat: force a seller scan over active conversations from the inbox

Adds a "Scan for sellers" button to the conversations header, backed by
POST /api/conversations/seller-scan.

The sweep reuses the existing seller detection rather than inventing a
second one: the configured keyword/domain list, the seller-link funnel
patterns, and the aiReplyScreening confidences all decide the verdict, so
a swept conversation lands in the state the live reply loop would have
given it. Being operator-forced, it deliberately ignores filterSellers
and aiReplyScreening.enabled.

AI is the last resort. Conversations are skipped when already flagged or
killed, settled for free by the deterministic rules, or skipped against a
durable sellerScreenedKey watermark when nothing new has arrived since
the last clean sweep. Only the remainder reaches the model, batched eight
at a time through a new screenConversationsForSellers, so the system
prompt and round trip are paid per batch instead of per conversation. A
classifier outage degrades the sweep instead of failing it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Jamie BruntonandClaude Opus 5 committed 2026-08-10 13:42:55 +01:00
1 parent 0a4e0064f2
commit 827eb3a1e0
15 files changed
+1169 -1

No files matched your search

+44
View File
@@ -604,4 +604,48 @@ describe('ConversationsPage', () => {
await user.click(within(dialog).getByRole('button', { name: 'Reveal image' }));
expect(dialogImage).not.toHaveClass('is-blurred');
});
it('runs a forced seller scan and reports what it cost', async () => {
let scanned = 0;
installConversationList();
server.use(
http.post('/api/conversations/seller-scan', () => {
scanned += 1;
return HttpResponse.json({
success: true, scanned: 12, alreadyFlagged: 2, unchanged: 6, deterministic: 1,
aiScreened: 3, aiRequests: 1, budgetSkipped: 0, emptySkipped: 0, degraded: false,
flagged: [{
conversationId: 'room-alice', username: 'alice', source: 'ai',
reason: 'AI screening classified this conversation as seller (confidence 0.90)',
confidence: 0.9, killed: true,
}],
});
}),
);
const user = userEvent.setup();
renderConversations();
await user.click(await screen.findByRole('button', { name: 'Scan for sellers' }));
const summary = await screen.findByRole('status');
expect(summary).toHaveTextContent('Scanned 12 conversations');
expect(summary).toHaveTextContent('1 flagged as sellers (1 ended)');
expect(summary).toHaveTextContent('3 needed AI over 1 request');
expect(summary).toHaveTextContent('6 unchanged since the last scan');
expect(scanned).toBe(1);
});
it('surfaces a failed seller scan without disturbing the list', async () => {
installConversationList();
server.use(
http.post('/api/conversations/seller-scan', () =>
HttpResponse.json({ error: 'Failed to scan conversations for sellers' }, { status: 500 })),
);
const user = userEvent.setup();
renderConversations();
await user.click(await screen.findByRole('button', { name: 'Scan for sellers' }));
expect(await screen.findByRole('alert')).toHaveTextContent('Seller scan failed');
expect(screen.getByRole('button', { name: /u\/alice/ })).toBeInTheDocument();
});
});
+50
View File
@@ -85,6 +85,56 @@ describe('OpenRouterClient', () => {
});
});
describe('screenConversationsForSellers', () => {
const batch = [
{ id: '1', username: 'alice', messages: [{ sender: 'alice', text: 'my menu is on my page', isSelf: false }] },
{ id: '2', username: 'bea', messages: [{ sender: 'bea', text: 'what are you up to today?', isSelf: false }] },
];
it('classifies a whole batch in one request and keys verdicts back by id', async () => {
mockCreate.mockResolvedValueOnce({
model: 'chat-model',
choices: [{ message: { content: '{"results":[{"id":"1","seller":true,"confidence":0.93,"evidence":"a menu"},{"id":"2","seller":false,"confidence":0.05,"evidence":"small talk"}]}' } }],
usage: {},
});
await expect(client.screenConversationsForSellers(batch)).resolves.toEqual([
{ id: '1', seller: true, confidence: 0.93, evidence: 'a menu' },
{ id: '2', seller: false, confidence: 0.05, evidence: 'small talk' },
]);
expect(mockCreate).toHaveBeenCalledTimes(1);
const request = mockCreate.mock.calls[0][0];
expect(request.temperature).toBe(0);
expect(request.messages[1].content).toContain('CONVERSATION id=1');
expect(request.messages[1].content).toContain('BEGIN UNTRUSTED CONVERSATION DATA');
});
it('drops verdicts for ids it never asked about', async () => {
mockCreate.mockResolvedValueOnce({
model: 'chat-model',
choices: [{ message: { content: '{"results":[{"id":"9","seller":true,"confidence":0.99,"evidence":"invented"}]}' } }],
usage: {},
});
await expect(client.screenConversationsForSellers(batch)).resolves.toEqual([]);
});
it('makes no request at all for an empty batch', async () => {
await expect(client.screenConversationsForSellers([])).resolves.toEqual([]);
expect(mockCreate).not.toHaveBeenCalled();
});
it('throws when no model returns usable JSON', async () => {
mockCreate.mockResolvedValue({
model: 'chat-model',
choices: [{ message: { content: 'they all look like sellers to me' } }],
usage: {},
});
await expect(client.screenConversationsForSellers(batch)).rejects.toThrow('no JSON object');
});
});
describe('isRefusal (internal)', () => {
// We test refusal detection by checking the analyzeImage behavior
// since isRefusal is private. We do this by mocking responses that
+271
View File
@@ -0,0 +1,271 @@
import { describe, expect, it, vi } from 'vitest';
import {
sweepConversationsForSellers,
type SellerSweepConversation,
type SellerSweepOptions,
} from '../src/services/seller-sweep.js';
import {
buildConversationState,
type ConversationState,
} from '../src/services/conversation-state.js';
const OPTIONS: SellerSweepOptions = {
sellerKeywords: ['onlyfans'],
sellerDomains: ['fansly.com'],
killConfidence: 0.8,
flagConfidence: 0.6,
killOnSellerKeyword: true,
};
function conversation(overrides: Partial<SellerSweepConversation> = {}): SellerSweepConversation {
return {
conversationId: 'room-1',
username: 'alice',
messages: [
{ sender: 'me', text: 'how has your week been?', isSelf: true },
{ sender: 'alice', text: 'busy, but good', isSelf: false },
],
watermark: 'w1',
...overrides,
};
}
/** In-memory stand-in for the durable conversation state repository. */
function stateStore(initial: Record<string, ConversationState> = {}) {
const saved = new Map<string, ConversationState>(Object.entries(initial));
return {
saved,
get: (id: string) => saved.get(id),
save: vi.fn((id: string, state: ConversationState) => { saved.set(id, structuredClone(state)); }),
};
}
function screener(verdicts: Array<{ id: string; seller: boolean; confidence: number; evidence: string }>) {
return { screenConversationsForSellers: vi.fn().mockResolvedValue(verdicts) };
}
describe('sweepConversationsForSellers', () => {
it('flags configured keywords and domains without spending an AI call', async () => {
const states = stateStore();
const ai = screener([]);
const summary = await sweepConversationsForSellers(
[
conversation({
conversationId: 'room-keyword',
messages: [{ sender: 'alice', text: 'everything is on my onlyfans', isSelf: false }],
}),
conversation({
conversationId: 'room-domain',
username: 'bea',
messages: [{ sender: 'bea', text: 'check fansly.com/bea for more', isSelf: false }],
}),
],
OPTIONS,
{ states, screener: ai },
);
expect(ai.screenConversationsForSellers).not.toHaveBeenCalled();
expect(summary.deterministic).toBe(2);
expect(summary.aiRequests).toBe(0);
expect(summary.flagged.map((flag) => flag.source)).toEqual(['keyword', 'domain']);
expect(summary.flagged.every((flag) => flag.killed)).toBe(true);
expect(states.saved.get('room-keyword')?.sellerDetected).toBe(true);
expect(states.saved.get('room-keyword')?.automation.status).toBe('killed');
});
it('catches obfuscated funnel language the keyword list does not enumerate', async () => {
const states = stateStore();
const summary = await sweepConversationsForSellers(
[conversation({ messages: [{ sender: 'alice', text: 'my links are in my bio babe', isSelf: false }] })],
OPTIONS,
{ states },
);
expect(summary.flagged).toHaveLength(1);
expect(summary.flagged[0].source).toBe('funnel');
});
it('hands the deterministic kill decision to the configured policy', async () => {
const states = stateStore();
const triage = { setStatus: vi.fn() };
const summary = await sweepConversationsForSellers(
[conversation({ messages: [{ sender: 'alice', text: 'see my onlyfans', isSelf: false }] })],
{ ...OPTIONS, killOnSellerKeyword: false },
{ states, triage },
);
expect(summary.flagged[0].killed).toBe(false);
expect(triage.setStatus).not.toHaveBeenCalled();
expect(states.saved.get('room-1')?.sellerDetected).toBe(true);
expect(states.saved.get('room-1')?.automation.status).toBe('human-required');
});
it('batches the remaining conversations into a single classifier request', async () => {
const states = stateStore();
const ai = screener([
{ id: '1', seller: true, confidence: 0.92, evidence: 'points at a paid page' },
{ id: '2', seller: false, confidence: 0.1, evidence: 'ordinary chat' },
]);
const triage = { setStatus: vi.fn() };
const summary = await sweepConversationsForSellers(
[
conversation({ conversationId: 'room-1', username: 'alice' }),
conversation({ conversationId: 'room-2', username: 'bea' }),
],
OPTIONS,
{ states, screener: ai, triage },
);
expect(ai.screenConversationsForSellers).toHaveBeenCalledTimes(1);
expect(summary.aiRequests).toBe(1);
expect(summary.aiScreened).toBe(2);
expect(summary.flagged).toEqual([expect.objectContaining({
conversationId: 'room-1', username: 'alice', source: 'ai', confidence: 0.92, killed: true,
})]);
expect(triage.setStatus).toHaveBeenCalledWith('room-1', 'done', null);
expect(states.saved.get('room-2')?.sellerDetected).toBe(false);
});
it('flags without killing between the flag and kill confidence thresholds', async () => {
const states = stateStore();
const ai = screener([{ id: '1', seller: true, confidence: 0.7, evidence: 'hints at a menu' }]);
const triage = { setStatus: vi.fn() };
const summary = await sweepConversationsForSellers([conversation()], OPTIONS, { states, screener: ai, triage });
expect(summary.flagged[0].killed).toBe(false);
expect(triage.setStatus).not.toHaveBeenCalled();
expect(states.saved.get('room-1')?.automation.status).toBe('human-required');
});
it('treats a low-confidence seller verdict as clean', async () => {
const states = stateStore();
const ai = screener([{ id: '1', seller: true, confidence: 0.2, evidence: 'weak' }]);
const summary = await sweepConversationsForSellers([conversation()], OPTIONS, { states, screener: ai });
expect(summary.flagged).toHaveLength(0);
expect(states.saved.get('room-1')?.sellerScreenedKey).toBe('w1');
});
it('skips conversations that are unchanged since the last clean sweep', async () => {
const cleared = { ...buildConversationState([]), sellerScreenedKey: 'w1' };
const states = stateStore({ 'room-1': cleared });
const ai = screener([]);
const summary = await sweepConversationsForSellers([conversation()], OPTIONS, { states, screener: ai });
expect(ai.screenConversationsForSellers).not.toHaveBeenCalled();
expect(summary.unchanged).toBe(1);
expect(summary.aiScreened).toBe(0);
});
it('re-screens a conversation once a newer inbound message arrives', async () => {
const cleared = { ...buildConversationState([]), sellerScreenedKey: 'w1' };
const states = stateStore({ 'room-1': cleared });
const ai = screener([{ id: '1', seller: false, confidence: 0.1, evidence: '' }]);
const summary = await sweepConversationsForSellers(
[conversation({ watermark: 'w2' })],
OPTIONS,
{ states, screener: ai },
);
expect(ai.screenConversationsForSellers).toHaveBeenCalledTimes(1);
expect(summary.aiScreened).toBe(1);
expect(states.saved.get('room-1')?.sellerScreenedKey).toBe('w2');
});
it('never re-screens conversations already flagged or killed', async () => {
const flagged = { ...buildConversationState([]), sellerDetected: true };
const killed = buildConversationState([]);
killed.automation.status = 'killed';
const states = stateStore({ 'room-1': flagged, 'room-2': killed });
const ai = screener([]);
const summary = await sweepConversationsForSellers(
[conversation(), conversation({ conversationId: 'room-2', username: 'bea' })],
OPTIONS,
{ states, screener: ai },
);
expect(ai.screenConversationsForSellers).not.toHaveBeenCalled();
expect(summary.alreadyFlagged).toBe(2);
expect(summary.scanned).toBe(2);
});
it('skips conversations with nothing inbound to judge', async () => {
const states = stateStore();
const ai = screener([]);
const summary = await sweepConversationsForSellers(
[conversation({ messages: [{ sender: 'me', text: 'hey there', isSelf: true }] })],
OPTIONS,
{ states, screener: ai },
);
expect(ai.screenConversationsForSellers).not.toHaveBeenCalled();
expect(summary.emptySkipped).toBe(1);
});
it('honours the per-sweep AI budget and reports what it left behind', async () => {
const states = stateStore();
const ai = screener([{ id: '1', seller: false, confidence: 0.1, evidence: '' }]);
const summary = await sweepConversationsForSellers(
[conversation({ conversationId: 'room-1' }), conversation({ conversationId: 'room-2' })],
{ ...OPTIONS, maxAiConversations: 1 },
{ states, screener: ai },
);
expect(summary.aiScreened).toBe(1);
expect(summary.budgetSkipped).toBe(1);
expect(states.saved.has('room-2')).toBe(false);
});
it('degrades rather than fails when the classifier is unavailable, keeping the old watermark', async () => {
const states = stateStore();
const ai = { screenConversationsForSellers: vi.fn().mockRejectedValue(new Error('router down')) };
const logger = { info: vi.fn(), warn: vi.fn() };
const summary = await sweepConversationsForSellers([conversation()], OPTIONS, { states, screener: ai, logger });
expect(summary.degraded).toBe(true);
expect(summary.flagged).toHaveLength(0);
expect(states.save).not.toHaveBeenCalled();
expect(logger.warn).toHaveBeenCalled();
});
it('leaves a conversation the model omitted for the next sweep', async () => {
const states = stateStore();
const ai = screener([{ id: '1', seller: false, confidence: 0.1, evidence: '' }]);
const summary = await sweepConversationsForSellers(
[conversation({ conversationId: 'room-1' }), conversation({ conversationId: 'room-2' })],
OPTIONS,
{ states, screener: ai },
);
expect(summary.aiScreened).toBe(1);
expect(states.saved.has('room-2')).toBe(false);
});
it('splits large sweeps into fixed-size batches', async () => {
const states = stateStore();
const ai = { screenConversationsForSellers: vi.fn().mockResolvedValue([]) };
await sweepConversationsForSellers(
Array.from({ length: 5 }, (_, index) => conversation({ conversationId: `room-${index}` })),
{ ...OPTIONS, batchSize: 2 },
{ states, screener: ai },
);
expect(ai.screenConversationsForSellers).toHaveBeenCalledTimes(3);
expect(ai.screenConversationsForSellers.mock.calls[0][0]).toHaveLength(2);
expect(ai.screenConversationsForSellers.mock.calls[2][0]).toHaveLength(1);
});
});
+130
View File
@@ -537,6 +537,136 @@ describe('conversations route', () => {
expect(await response.json()).toEqual({ success: true });
});
describe('forced seller sweep', () => {
const sweepConfig = {
reddit: { sellerKeywords: ['onlyfans'], sellerDomains: ['fansly.com'] },
conversations: {
killOnSellerKeyword: true,
aiReplyScreening: { enabled: false, killConfidence: 0.8, flagConfidence: 0.6 },
},
};
function chat(username: string, roomId: string, snippet: string) {
return {
username, roomName: username, roomId, lastMessageTime: 500,
lastMessageSnippet: snippet, isFromUs: false, unread: false,
messages: [{ sender: username, text: snippet, timestamp: 500, isSelf: false }],
};
}
function sweepContext(conversations: unknown[], ai?: unknown, states = new Map<string, unknown>()) {
const runtime = {
session: { accountId: '@t2_one:reddit.com', redditUsername: 'owner-one' },
snapshot: () => ({ state: 'running' }),
resources: {
chatService: { getCachedConversations: vi.fn().mockResolvedValue(conversations) },
conversationStateRepo: {
get: (id: string) => states.get(id),
save: vi.fn((id: string, state: unknown) => { states.set(id, state); }),
},
conversationReadStateRepo: { setStatus: vi.fn() },
},
};
const context = {
logger: { error: vi.fn(), warn: vi.fn(), info: vi.fn() },
broadcaster: { push: vi.fn() },
config: sweepConfig,
services: {
chat: { getCachedConversations: vi.fn().mockResolvedValue([]) },
ai,
accountRuntimes: {
list: () => [{ accountId: '@t2_one:reddit.com', redditUsername: 'owner-one', state: 'running' }],
get: () => runtime,
},
},
} as unknown as WebContext;
return { context, runtime, states };
}
it('flags keyword hits for free and sends only the unsettled conversations to the classifier', async () => {
const ai = { screenConversationsForSellers: vi.fn().mockResolvedValue([]) };
const { context, runtime } = sweepContext(
[chat('alice', 'room-alice', 'everything else is on my onlyfans'), chat('bob', 'room-bob', 'how was your day?')],
ai,
);
const app = express().use(express.json()).use('/api', createConversationsRouter(context));
const response = await request(app, '/api/conversations/seller-scan', { method: 'POST' });
const body = await response.json();
expect(response.status).toBe(200);
expect(body).toMatchObject({ success: true, scanned: 2, deterministic: 1, aiRequests: 1 });
expect(body.flagged).toEqual([expect.objectContaining({
// Flags carry the owned id so the dashboard can match its list rows.
conversationId: `${encodeURIComponent('@t2_one:reddit.com')}~room-alice`,
username: 'alice', source: 'keyword', killed: true,
})]);
// The keyword hit never reaches the model; only the clean row is screened.
expect(ai.screenConversationsForSellers.mock.calls[0][0]).toEqual([
expect.objectContaining({ username: 'bob' }),
]);
expect(runtime.resources.conversationReadStateRepo.setStatus).toHaveBeenCalledWith('room-alice', 'done', null);
});
it('batches the conversations the rules could not settle into one classifier request', async () => {
const ai = {
screenConversationsForSellers: vi.fn().mockResolvedValue([
{ id: '1', seller: true, confidence: 0.9, evidence: 'sells customs' },
{ id: '2', seller: false, confidence: 0.05, evidence: 'ordinary chat' },
]),
};
const { context } = sweepContext(
[chat('alice', 'room-alice', 'dm me about customs'), chat('bob', 'room-bob', 'how was your day?')],
ai,
);
const app = express().use(express.json()).use('/api', createConversationsRouter(context));
const body = await (await request(app, '/api/conversations/seller-scan', { method: 'POST' })).json();
expect(ai.screenConversationsForSellers).toHaveBeenCalledTimes(1);
expect(body).toMatchObject({ scanned: 2, aiScreened: 2, aiRequests: 1 });
expect(body.flagged).toEqual([expect.objectContaining({ username: 'alice', source: 'ai', confidence: 0.9 })]);
});
it('screens conversations whose transcript was never cached, using the summary snippet', async () => {
const ai = {
screenConversationsForSellers: vi.fn().mockResolvedValue([
{ id: '1', seller: true, confidence: 0.95, evidence: 'links to a paid page' },
]),
};
const { context } = sweepContext([{
username: 'alice', roomName: 'alice', roomId: 'room-alice', lastMessageTime: 500,
lastMessageSnippet: 'come see my page for the good stuff', isFromUs: false, unread: false,
}], ai);
const app = express().use(express.json()).use('/api', createConversationsRouter(context));
const body = await (await request(app, '/api/conversations/seller-scan', { method: 'POST' })).json();
expect(ai.screenConversationsForSellers.mock.calls[0][0][0].messages).toEqual([
{ sender: 'alice', text: 'come see my page for the good stuff', isSelf: false },
]);
expect(body.flagged).toHaveLength(1);
});
it('reports a degraded sweep instead of failing when screening is unavailable', async () => {
const ai = { screenConversationsForSellers: vi.fn().mockRejectedValue(new Error('router down')) };
const { context } = sweepContext([chat('alice', 'room-alice', 'hey there stranger')], ai);
const app = express().use(express.json()).use('/api', createConversationsRouter(context));
const response = await request(app, '/api/conversations/seller-scan', { method: 'POST' });
expect(response.status).toBe(200);
expect(await response.json()).toMatchObject({ success: true, degraded: true, flagged: [] });
});
it('refuses to scan when no chat service is initialized', async () => {
const context = { logger: { error: vi.fn() }, services: {} } as unknown as WebContext;
const app = express().use(express.json()).use('/api', createConversationsRouter(context));
expect((await request(app, '/api/conversations/seller-scan', { method: 'POST' })).status).toBe(503);
});
});
describe('operator read state', () => {
const conversation = {
username: 'alice', roomName: 'alice', roomId: 'room-alice',