Add human-led conversation learning loop

src/services/conversation-automation.ts

@@ -17,6 +17,7 @@ import {
   mergeConversationState,
   r4rReplySafetyFailure,
   remainingAutomationTurns,
+  type ConversationIntent,
   type ConversationMessage,
   type ConversationState,
 } from './conversation-state.js';
@@ -24,6 +25,8 @@ import { containsMediaRequest } from './conversation-boundaries.js';
 import { findOffPlatformSignal, offPlatformSignalReason } from './off-platform-signals.js';
 import type { MediaLibraryExchangeService } from './media-library-exchange.js';
 import type { ConversationReplyScreening } from '../ai/openrouter.js';
+import type { LearningConfig } from '../config/index.js';
+import type { ConversationLearningProvider, LearningStage, RetrievedLearningExample } from './conversation-learning-types.js';
 
 export type ConversationAutomationResultKind =
   | 'sent'
@@ -49,7 +52,7 @@ export interface ConversationAutomationClaims {
   listPending?(): ConversationAutomationClaimRecord[];
   markAttempt?(conversationId: string, incomingKey: string): boolean;
   releasePending?(conversationId: string, incomingKey: string): boolean;
-  markSent(conversationId: string, incomingKey: string): boolean;
+  markSent(conversationId: string, incomingKey: string, matrixEventId?: string | null): boolean;
   markSuperseded?(conversationId: string, incomingKey: string): boolean;
 }
 
@@ -87,6 +90,7 @@ export interface ConversationAutomationOptions {
   triage?: ConversationTriageWriter;
   screener?: ConversationReplyScreener;
   screening?: ConversationScreeningOptions;
+  learning?: { provider: ConversationLearningProvider; config: LearningConfig; accountId: string };
 }
 
 export interface ReplyPriorityStatus {
@@ -267,9 +271,16 @@ export class ConversationAutomationService {
     }
     const existingClaim = this.claims.getClaim?.(conversationId, incomingKey);
     if (existingClaim?.status === 'pending') {
-      const observedOutbound = messages.some((message) => message.isSelf && message.timestamp > latest.timestamp);
+      const expectedReplyFingerprint = existingClaim.replyText
+        ? conversationMessageFingerprint(existingClaim.replyText)
+        : null;
+      const observedOutbound = messages.find((message) =>
+        message.isSelf
+        && message.timestamp > latest.timestamp
+        && expectedReplyFingerprint !== null
+        && conversationMessageFingerprint(message.text) === expectedReplyFingerprint);
       if (observedOutbound) {
-        this.claims.markSent(conversationId, incomingKey);
+        this.claims.markSent(conversationId, incomingKey, observedOutbound.eventId ?? null);
         this.logger.info(`[automation] Reconciled interrupted reply for u/${conversation.username} from Matrix history`);
         return null;
       }
@@ -437,11 +448,20 @@ export class ConversationAutomationService {
     if (isLegacy && intent.kind !== 'media-ask') {
       return this.finish(conversation, conversationId, state, incomingKey, 'human-required', intent.reason);
     }
+    const learningExamples = this.learningExamples({
+      accountId: this.options.learning?.accountId ?? '',
+      conversationId,
+      subreddit: opener.subreddit ?? null,
+      policyMode: state.automation.policyMode,
+      stage: stageForIntent(intent.kind),
+      purpose: requestsMedia ? 'media-ask' : 'reply',
+      incomingEventId: incomingKey,
+    });
     const generatePlannedReply = () => isLegacy
-      ? this.ai.generateMediaAsk(mapped, conversation.username, state, intent)
+      ? this.ai.generateMediaAsk(mapped, conversation.username, state, intent, learningExamples)
       : isHybridMediaAsk
-        ? this.ai.generateR4RMediaAsk(mapped, conversation.username, state)
-        : this.ai.generateReply(mapped, conversation.username, state, intent);
+        ? this.ai.generateR4RMediaAsk(mapped, conversation.username, state, learningExamples)
+        : this.ai.generateReply(mapped, conversation.username, state, intent, learningExamples);
     let reply = await generatePlannedReply();
     let safetyFailure = isLegacy || isHybridMediaAsk
       ? hybridMediaAskSafetyFailure(reply)
@@ -486,6 +506,20 @@ export class ConversationAutomationService {
     );
   }
 
+  private learningExamples(query: Parameters<ConversationLearningProvider['getExamples']>[0]): readonly RetrievedLearningExample[] {
+    const learning = this.options.learning;
+    if (!learning) return [];
+    const examples = learning.provider.getExamples(query);
+    const injected = learning.config.generationEnabled && !learning.config.shadowMode;
+    learning.provider.recordGeneration({
+      query,
+      mode: injected ? 'injected' : 'shadow',
+      policyVersion: learning.config.policyVersion,
+      exampleIds: examples.map((example) => example.decisionId),
+    });
+    return injected ? examples : [];
+  }
+
   private actionableConversations(
     conversations: ActiveChat[],
     observedAfter: number | null = this.lastScanAt,
@@ -512,8 +546,13 @@ export class ConversationAutomationService {
         ? conversationMessageFingerprint(claim.replyText)
         : null;
       const observedClaimedReply = expectedFingerprint !== null
-        && laterSelfMessages.some((message) => conversationMessageFingerprint(message.text) === expectedFingerprint);
-      if (observedClaimedReply && this.claims.markSent(claim.conversationId, claim.incomingKey)) {
+        ? laterSelfMessages.find((message) => conversationMessageFingerprint(message.text) === expectedFingerprint)
+        : undefined;
+      if (observedClaimedReply && this.claims.markSent(
+        claim.conversationId,
+        claim.incomingKey,
+        observedClaimedReply.eventId ?? null,
+      )) {
         this.logger.info(`[automation] Reconciled interrupted reply for u/${conversation.username} from Matrix history`);
       } else if (!observedClaimedReply && this.claims.markSuperseded?.(claim.conversationId, claim.incomingKey)) {
         this.logger.info(`[automation] Superseded queued reply for u/${conversation.username}; an operator-authored message was observed`);
@@ -640,8 +679,14 @@ export class ConversationAutomationService {
           isHybridMediaAsk,
         );
         this.claims.markAttempt?.(conversationId, incomingKey);
-        const sent = await this.chat.sendChatMessage(latestConversation.url!, reply);
-        return { kind: sent ? 'sent' as const : 'failed' as const, exhausted, state: freshState };
+        const receipt = typeof this.chat.sendChatMessageDetailed === 'function'
+          ? await this.chat.sendChatMessageDetailed(latestConversation.url!, reply)
+          : await this.chat.sendChatMessage(latestConversation.url!, reply)
+            ? { eventId: null, sentAt: this.now() }
+            : null;
+        return receipt
+          ? { kind: 'sent' as const, exhausted, state: freshState, eventId: receipt.eventId }
+          : { kind: 'failed' as const, exhausted, state: freshState };
       },
       (result) => result.kind === 'sent',
       signal,
@@ -650,7 +695,8 @@ export class ConversationAutomationService {
     if (attempt.kind === 'superseded') return attempt.result;
     if (attempt.kind === 'failed') return null;
     const sentAt = this.now();
-    this.claims.markSent(conversation.roomId!, incomingKey);
+    if (attempt.eventId) this.claims.markSent(conversation.roomId!, incomingKey, attempt.eventId);
+    else this.claims.markSent(conversation.roomId!, incomingKey);
     this.lastReplySentAt = sentAt;
     this.lastReplyLatencyMs = Math.max(0, sentAt - incomingAt);
     this.logger.info(
@@ -859,6 +905,13 @@ function replyPlan(state: ConversationState): { isLegacy: boolean; isHybridMedia
   return { isLegacy, isHybridMediaAsk };
 }
 
+function stageForIntent(intent: ConversationIntent['kind']): LearningStage {
+  if (intent === 'media-ask') return 'media-invitation';
+  if (intent === 'answer-question') return 'answer';
+  if (intent === 'respect-boundary') return 'handoff';
+  return 'rapport';
+}
+
 /**
  * Handoffs that need a person rather than a kill, and that apply to every
  * policy mode. Messaging-app names are deliberately absent: those are handled