Fix false autonomous safety handoffs

This commit is contained in:
jjbrunton committed 2026-08-09 21:27:53 +01:00
1 parent 84a133451f
commit c0cc661a2f
6 files changed
+144 -29

No files matched your search

+49 -13
View File
@@ -88,7 +88,6 @@ export function conversationMessageFingerprint(message: string): string {
}
export function remainingAutomationTurns(state: ConversationState): number {
if (state.automation.status !== 'active') return 0;
return Math.max(0, state.automation.maxAutomatedTurns - state.automation.turnsUsed);
}
@@ -191,28 +190,65 @@ export function formatConversationAsUntrustedData(messages: ConversationMessage[
return `BEGIN UNTRUSTED CONVERSATION DATA\n${rows.join('\n')}\nEND UNTRUSTED CONVERSATION DATA`;
}
export type ConversationReplySafetyFailure =
| 'empty reply'
| 'reply exceeds 600 characters'
| 'reply contains instruction or prompt leakage'
| 'reply implies shared location or an in-person meeting'
| 'reply exceeds the three-sentence limit'
| 'reply solicits media outside the configured media-ask phase'
| 'media ask does not contain a recognizable image or video request'
| 'media ask requests explicit or sexual content'
| 'media ask uses coercive or pressuring language';
/** Explain why generated text violates the shared autonomous reply contract. */
export function conversationReplySafetyFailure(message: string): ConversationReplySafetyFailure | null {
const trimmed = message.trim();
if (!trimmed) return 'empty reply';
if (trimmed.length > 600) return 'reply exceeds 600 characters';
if (/\b(as an ai|system prompt|begin untrusted|end untrusted|ignore (all|previous) instructions)\b/i.test(trimmed)) {
return 'reply contains instruction or prompt leakage';
}
if (!isSafeOutboundMessage(trimmed)) return 'reply implies shared location or an in-person meeting';
const sentences = trimmed.match(/[^.!?]+[.!?]+/g) ?? [];
if (sentences.length > 3) return 'reply exceeds the three-sentence limit';
return null;
}
/** Reject obvious instruction leakage and outputs that disregard the reply length contract. */
export function isSafeConversationReply(message: string): boolean {
const trimmed = message.trim();
if (!trimmed || trimmed.length > 600) return false;
if (/\b(as an ai|system prompt|begin untrusted|end untrusted|ignore (all|previous) instructions)\b/i.test(trimmed)) return false;
if (!isSafeOutboundMessage(trimmed)) return false;
const sentences = trimmed.match(/[^.!?]+[.!?]+/g) ?? [];
return sentences.length <= 3;
return conversationReplySafetyFailure(message) === null;
}
export function r4rReplySafetyFailure(message: string): ConversationReplySafetyFailure | null {
const sharedFailure = conversationReplySafetyFailure(message);
if (sharedFailure) return sharedFailure;
return containsMediaRequest(message)
? 'reply solicits media outside the configured media-ask phase'
: null;
}
/** R4R automation may converse and flirt, but it must never solicit sexual media. */
export function isSafeR4RReply(message: string): boolean {
if (!isSafeConversationReply(message)) return false;
return !containsMediaRequest(message);
return r4rReplySafetyFailure(message) === null;
}
export function hybridMediaAskSafetyFailure(message: string): ConversationReplySafetyFailure | null {
const sharedFailure = conversationReplySafetyFailure(message);
if (sharedFailure) return sharedFailure;
if (!containsMediaRequest(message)) return 'media ask does not contain a recognizable image or video request';
if (/\b(nude|nudes|naked|explicit|nsfw|sexy pic|sexual)\b/i.test(message)) {
return 'media ask requests explicit or sexual content';
}
if (/\b(must|prove|dare|owe|have to|need to)\b/i.test(message)) {
return 'media ask uses coercive or pressuring language';
}
return null;
}
/** The hybrid policy gets one optional, non-explicit image request after rapport. */
export function isSafeHybridMediaAsk(message: string): boolean {
if (!isSafeConversationReply(message)) return false;
if (!containsMediaRequest(message)) return false;
if (/\b(nude|nudes|naked|explicit|nsfw|sexy pic|sexual)\b/i.test(message)) return false;
return !/\b(must|prove|dare|owe|have to|need to)\b/i.test(message);
return hybridMediaAskSafetyFailure(message) === null;
}
export function serializeConversationState(state: ConversationState): string {