Fix false autonomous safety handoffs

This commit is contained in:
jjbrunton committed 2026-08-09 21:27:53 +01:00
1 parent 84a133451f
commit c0cc661a2f
6 files changed
+144 -29

No files matched your search

+36 -4
View File
@@ -107,7 +107,7 @@ describe('conversation automation', () => {
expect((await service.processConversation(incoming(1)))?.remainingTurns).toBe(2);
const second = await service.processConversation(incoming(2));
expect(second).toMatchObject({ kind: 'human-required', remainingTurns: 0 });
expect(second).toMatchObject({ kind: 'human-required', remainingTurns: 2 });
expect(chat.sendChatMessage).toHaveBeenCalledOnce();
expect(getSaved()?.automation).toMatchObject({ turnsUsed: 1, status: 'human-required' });
expect(getSaved()?.automation.aiMessageFingerprints).toHaveLength(1);
@@ -135,7 +135,7 @@ describe('conversation automation', () => {
const result = await fixture.service.processConversation(conversation);
expect(result).toMatchObject({ kind: 'human-required', remainingTurns: 0 });
expect(result).toMatchObject({ kind: 'human-required', remainingTurns: 3 });
expect(fixture.ai.generateMediaAsk).not.toHaveBeenCalled();
expect(fixture.chat.sendChatMessage).not.toHaveBeenCalled();
expect(fixture.getSaved()?.automation.handoffReason).toContain('media request has already been made');
@@ -194,7 +194,7 @@ describe('conversation automation', () => {
const { service, chat, ai, getSaved } = harness(['onlyfans'], ['fansly.com']);
const result = await service.processConversation(incoming(1, 'My Fansly is https://www.fansly.com/example'));
expect(result).toMatchObject({ kind: 'human-required', remainingTurns: 0 });
expect(result).toMatchObject({ kind: 'human-required', remainingTurns: 3 });
expect(ai.generateMediaAsk).not.toHaveBeenCalled();
expect(chat.sendChatMessage).not.toHaveBeenCalled();
expect(getSaved()?.automation).toMatchObject({
@@ -550,7 +550,39 @@ describe('conversation automation', () => {
expect(result?.kind).toBe('human-required');
expect(fixture.chat.sendChatMessage).not.toHaveBeenCalled();
expect(fixture.getSaved()?.automation.handoffReason).toContain('failed autonomous safety checks');
expect(fixture.getSaved()?.automation.handoffReason)
.toBe('The generated reply was rejected: reply solicits media outside the configured media-ask phase.');
});
it('regenerates once when a legacy media ask contains no recognizable request', async () => {
const fixture = harness();
fixture.ai.generateMediaAsk
.mockResolvedValueOnce('That sounds fun. I would love to see that smile.')
.mockResolvedValueOnce('If you are up for it, snap a cute pic of yourself?');
const result = await fixture.service.processConversation(incoming(1));
expect(result).toMatchObject({ kind: 'sent', remainingTurns: 2 });
expect(fixture.ai.generateMediaAsk).toHaveBeenCalledTimes(2);
expect(fixture.chat.sendChatMessage).toHaveBeenCalledWith(
expect.any(String),
'If you are up for it, snap a cute pic of yourself?',
);
});
it('preserves unused turns and records the exact reason when regeneration is still invalid', async () => {
const fixture = harness();
fixture.ai.generateMediaAsk.mockResolvedValue('That sounds fun. I would love to see that smile.');
const result = await fixture.service.processConversation(incoming(1));
expect(result).toMatchObject({ kind: 'human-required', remainingTurns: 3 });
expect(fixture.ai.generateMediaAsk).toHaveBeenCalledTimes(2);
expect(fixture.getSaved()?.automation).toMatchObject({
turnsUsed: 0,
handoffReason: 'The generated media ask was rejected: media ask does not contain a recognizable image or video request.',
});
expect(fixture.chat.sendChatMessage).not.toHaveBeenCalled();
});
it('builds rapport before making exactly one optional media ask in hybrid mode', async () => {
+14
View File
@@ -3,6 +3,7 @@ import {
buildConversationState,
chooseConversationIntent,
formatConversationAsUntrustedData,
hybridMediaAskSafetyFailure,
mergeConversationState,
isSafeConversationReply,
isSafeHybridMediaAsk,
@@ -88,6 +89,15 @@ describe('conversation state', () => {
expect(chooseConversationIntent([], state, 'media-ask').kind).toBe('follow-up');
});
it('reports unused turns even when automation is paused for another reason', () => {
const state = buildConversationState([message('Hello', true), message('Hey')]);
state.automation.turnsUsed = 1;
state.automation.status = 'human-required';
expect(remainingAutomationTurns(state)).toBe(2);
expect(chooseConversationIntent([], state, 'media-ask').kind).toBe('follow-up');
});
it('migrates legacy saved state with a fresh automation counter', () => {
const state = parseConversationState(JSON.stringify({
persona: { age: null, location: null, relationshipStatus: 'unspecified' },
@@ -146,9 +156,13 @@ describe('conversation state', () => {
it('allows only optional non-explicit media asks in the hybrid phase', () => {
expect(isSafeHybridMediaAsk('Would you be comfortable sharing a selfie?')).toBe(true);
expect(isSafeHybridMediaAsk('If you are up for it, snap a cute pic of yourself today?')).toBe(true);
expect(isSafeHybridMediaAsk('You can attach or upload a photo if you like.')).toBe(true);
expect(isSafeHybridMediaAsk('Would you send me a nude?')).toBe(false);
expect(isSafeHybridMediaAsk('You have to prove it by sending a photo.')).toBe(false);
expect(isSafeHybridMediaAsk('What music are you into?')).toBe(false);
expect(hybridMediaAskSafetyFailure('What music are you into?'))
.toBe('media ask does not contain a recognizable image or video request');
});
it('rejects replies that imply shared location or encourage meeting in person', () => {
+23
View File
@@ -313,6 +313,29 @@ describe('ConversationsPage', () => {
expect(await screen.findByRole('textbox', { name: 'Reply to u/alice' })).toHaveValue('Would you like to share a photo?');
});
it('shows unused turns separately from a safety handoff', async () => {
server.use(
http.get('/api/conversations', () => HttpResponse.json([alice])),
http.get('/api/conversations/room-alice', () => HttpResponse.json({
...detailFixture(),
automation: {
turnsUsed: 0,
remainingTurns: 3,
status: 'human-required',
handoffReason: 'The generated media ask was rejected: media ask does not contain a recognizable image or video request.',
},
})),
);
const user = userEvent.setup();
renderConversations();
await user.click(await screen.findByRole('button', { name: /u\/alice/ }));
expect(await screen.findByText('3 automated turns remaining')).toBeInTheDocument();
expect(screen.getByText(/generated media ask was rejected/)).toBeInTheDocument();
expect(screen.getByRole('button', { name: 'Suggest media ask' })).toBeDisabled();
});
it('starts a conversation and preserves both fields after a recoverable failure', async () => {
installConversationList();
let startRequests = 0;