Fix false autonomous safety handoffs
This commit is contained in:
1 parent
84a133451f
commit
c0cc661a2f
6 files changed
+144
-29
No files matched your search
@@ -88,7 +88,6 @@ export function conversationMessageFingerprint(message: string): string {
|
||||
}
|
||||
|
||||
export function remainingAutomationTurns(state: ConversationState): number {
|
||||
if (state.automation.status !== 'active') return 0;
|
||||
return Math.max(0, state.automation.maxAutomatedTurns - state.automation.turnsUsed);
|
||||
}
|
||||
|
||||
@@ -191,28 +190,65 @@ export function formatConversationAsUntrustedData(messages: ConversationMessage[
|
||||
return `BEGIN UNTRUSTED CONVERSATION DATA\n${rows.join('\n')}\nEND UNTRUSTED CONVERSATION DATA`;
|
||||
}
|
||||
|
||||
export type ConversationReplySafetyFailure =
|
||||
| 'empty reply'
|
||||
| 'reply exceeds 600 characters'
|
||||
| 'reply contains instruction or prompt leakage'
|
||||
| 'reply implies shared location or an in-person meeting'
|
||||
| 'reply exceeds the three-sentence limit'
|
||||
| 'reply solicits media outside the configured media-ask phase'
|
||||
| 'media ask does not contain a recognizable image or video request'
|
||||
| 'media ask requests explicit or sexual content'
|
||||
| 'media ask uses coercive or pressuring language';
|
||||
|
||||
/** Explain why generated text violates the shared autonomous reply contract. */
|
||||
export function conversationReplySafetyFailure(message: string): ConversationReplySafetyFailure | null {
|
||||
const trimmed = message.trim();
|
||||
if (!trimmed) return 'empty reply';
|
||||
if (trimmed.length > 600) return 'reply exceeds 600 characters';
|
||||
if (/\b(as an ai|system prompt|begin untrusted|end untrusted|ignore (all|previous) instructions)\b/i.test(trimmed)) {
|
||||
return 'reply contains instruction or prompt leakage';
|
||||
}
|
||||
if (!isSafeOutboundMessage(trimmed)) return 'reply implies shared location or an in-person meeting';
|
||||
const sentences = trimmed.match(/[^.!?]+[.!?]+/g) ?? [];
|
||||
if (sentences.length > 3) return 'reply exceeds the three-sentence limit';
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Reject obvious instruction leakage and outputs that disregard the reply length contract. */
|
||||
export function isSafeConversationReply(message: string): boolean {
|
||||
const trimmed = message.trim();
|
||||
if (!trimmed || trimmed.length > 600) return false;
|
||||
if (/\b(as an ai|system prompt|begin untrusted|end untrusted|ignore (all|previous) instructions)\b/i.test(trimmed)) return false;
|
||||
if (!isSafeOutboundMessage(trimmed)) return false;
|
||||
const sentences = trimmed.match(/[^.!?]+[.!?]+/g) ?? [];
|
||||
return sentences.length <= 3;
|
||||
return conversationReplySafetyFailure(message) === null;
|
||||
}
|
||||
|
||||
export function r4rReplySafetyFailure(message: string): ConversationReplySafetyFailure | null {
|
||||
const sharedFailure = conversationReplySafetyFailure(message);
|
||||
if (sharedFailure) return sharedFailure;
|
||||
return containsMediaRequest(message)
|
||||
? 'reply solicits media outside the configured media-ask phase'
|
||||
: null;
|
||||
}
|
||||
|
||||
/** R4R automation may converse and flirt, but it must never solicit sexual media. */
|
||||
export function isSafeR4RReply(message: string): boolean {
|
||||
if (!isSafeConversationReply(message)) return false;
|
||||
return !containsMediaRequest(message);
|
||||
return r4rReplySafetyFailure(message) === null;
|
||||
}
|
||||
|
||||
export function hybridMediaAskSafetyFailure(message: string): ConversationReplySafetyFailure | null {
|
||||
const sharedFailure = conversationReplySafetyFailure(message);
|
||||
if (sharedFailure) return sharedFailure;
|
||||
if (!containsMediaRequest(message)) return 'media ask does not contain a recognizable image or video request';
|
||||
if (/\b(nude|nudes|naked|explicit|nsfw|sexy pic|sexual)\b/i.test(message)) {
|
||||
return 'media ask requests explicit or sexual content';
|
||||
}
|
||||
if (/\b(must|prove|dare|owe|have to|need to)\b/i.test(message)) {
|
||||
return 'media ask uses coercive or pressuring language';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The hybrid policy gets one optional, non-explicit image request after rapport. */
|
||||
export function isSafeHybridMediaAsk(message: string): boolean {
|
||||
if (!isSafeConversationReply(message)) return false;
|
||||
if (!containsMediaRequest(message)) return false;
|
||||
if (/\b(nude|nudes|naked|explicit|nsfw|sexy pic|sexual)\b/i.test(message)) return false;
|
||||
return !/\b(must|prove|dare|owe|have to|need to)\b/i.test(message);
|
||||
return hybridMediaAskSafetyFailure(message) === null;
|
||||
}
|
||||
|
||||
export function serializeConversationState(state: ConversationState): string {
|
||||
|
||||
Reference in new issue
Block a user