Files
RedditBot/src/database/conversation-learning-repository.ts
T

550 lines
21 KiB
TypeScript

import { createHash } from 'node:crypto';
import type Database from 'better-sqlite3';
import type { AttributionConfidence } from './conversation-event-repository.js';
import {
LearningTurnSchema,
type LearningCreditClass,
type LearningDecisionRecord,
type LearningDecisionStatus,
type LearningStage,
type LearningTrajectoryDetail,
type LearningTrajectoryRecord,
type LearningTrajectoryStatus,
type LearningTurn,
type RetrievedLearningExample,
} from '../services/conversation-learning-types.js';
import type { ConversationPolicyMode } from '../services/conversation-state.js';
export interface UpsertLearningTrajectory {
id: string;
conversationId: string;
sentMessageId: number | null;
subreddit: string | null;
sourceType: 'image' | 'chat' | null;
policyMode: ConversationPolicyMode | null;
firstHumanEventId: string | null;
outcomeEventId: string;
outcomeType: 'image-received' | 'reciprocal-engagement';
creditClass: LearningCreditClass;
attributionConfidence: AttributionConfidence;
status: LearningTrajectoryStatus;
maturityAt: number;
invalidatedAt?: number | null;
invalidationReason?: string | null;
now?: number;
}
export interface UpsertLearningDecision {
id: string;
trajectoryId: string;
decisionEventId: string;
stage: LearningStage;
context: LearningTurn[];
chosenResponse: string;
turnsUntilOutcome: number;
isDirectMediaInvitation: boolean;
qualityStatus: LearningDecisionStatus;
now?: number;
}
export interface LearningSummary {
total: number;
provisional: number;
eligible: number;
excluded: number;
invalidated: number;
needsReview: number;
humanCreated: number;
humanImproved: number;
humanCompletedAi: number;
autonomous: number;
recipientInitiated: number;
uncertain: number;
exact: number;
fingerprint: number;
inferred: number;
}
/** Persistence for derived, revocable learning trajectories and decisions. */
export class ConversationLearningRepository {
constructor(
private readonly db: Database.Database,
private readonly accountId?: string,
) {}
upsertTrajectory(input: UpsertLearningTrajectory): LearningTrajectoryRecord {
const accountId = this.requireAccountId();
const now = input.now ?? Date.now();
this.db.prepare(`
INSERT INTO conversation_learning_trajectories (
id, account_id, conversation_id, sent_message_id, subreddit, source_type,
policy_mode, first_human_event_id, outcome_event_id, outcome_type,
credit_class, attribution_confidence, status, maturity_at,
invalidated_at, invalidation_reason, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(account_id, conversation_id, outcome_event_id) DO UPDATE SET
sent_message_id = excluded.sent_message_id,
subreddit = excluded.subreddit,
source_type = excluded.source_type,
policy_mode = excluded.policy_mode,
first_human_event_id = excluded.first_human_event_id,
credit_class = excluded.credit_class,
attribution_confidence = excluded.attribution_confidence,
status = CASE
WHEN conversation_learning_trajectories.status = 'excluded' AND excluded.status != 'invalidated'
THEN 'excluded'
WHEN conversation_learning_trajectories.reviewed_at IS NOT NULL
AND conversation_learning_trajectories.status = 'eligible'
AND excluded.status != 'invalidated'
THEN 'eligible'
ELSE excluded.status
END,
maturity_at = excluded.maturity_at,
invalidated_at = excluded.invalidated_at,
invalidation_reason = excluded.invalidation_reason,
updated_at = excluded.updated_at
`).run(
input.id, accountId, input.conversationId, input.sentMessageId,
input.subreddit, input.sourceType, input.policyMode, input.firstHumanEventId,
input.outcomeEventId, input.outcomeType, input.creditClass,
input.attributionConfidence, input.status, input.maturityAt,
input.invalidatedAt ?? null, input.invalidationReason ?? null, now, now,
);
if (input.sentMessageId !== null && input.outcomeType === 'image-received') {
this.db.prepare(`
UPDATE sent_messages
SET image_outcome_credit = ?, image_outcome_at = ?
WHERE account_id = ? AND id = ?
`).run(input.creditClass, now, accountId, input.sentMessageId);
}
return this.getTrajectory(input.id)!;
}
replaceDecisions(trajectoryId: string, decisions: readonly UpsertLearningDecision[]): void {
const accountId = this.requireAccountId();
this.db.transaction(() => {
const keep = new Set(decisions.map((decision) => decision.id));
const existing = this.db.prepare(`
SELECT id FROM conversation_learning_decisions
WHERE account_id = ? AND trajectory_id = ?
`).all(accountId, trajectoryId) as Array<{ id: string }>;
for (const row of existing) {
if (!keep.has(row.id)) {
this.db.prepare('DELETE FROM conversation_learning_decisions WHERE account_id = ? AND id = ?')
.run(accountId, row.id);
}
}
for (const decision of decisions) this.upsertDecision(decision);
})();
}
upsertDecision(input: UpsertLearningDecision): LearningDecisionRecord {
const accountId = this.requireAccountId();
const now = input.now ?? Date.now();
this.db.prepare(`
INSERT INTO conversation_learning_decisions (
id, trajectory_id, account_id, decision_event_id, stage, context_json,
chosen_response, turns_until_outcome, is_direct_media_invitation,
quality_status, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(trajectory_id, decision_event_id) DO UPDATE SET
stage = excluded.stage,
context_json = excluded.context_json,
chosen_response = excluded.chosen_response,
turns_until_outcome = excluded.turns_until_outcome,
is_direct_media_invitation = excluded.is_direct_media_invitation,
quality_status = CASE
WHEN conversation_learning_decisions.quality_status = 'excluded' AND excluded.quality_status != 'invalidated'
THEN 'excluded'
WHEN EXISTS (
SELECT 1 FROM conversation_learning_trajectories t
WHERE t.id = excluded.trajectory_id AND t.reviewed_at IS NOT NULL AND t.status = 'eligible'
) AND excluded.quality_status != 'invalidated'
THEN 'eligible'
ELSE excluded.quality_status
END,
updated_at = excluded.updated_at
`).run(
input.id, input.trajectoryId, accountId, input.decisionEventId, input.stage,
JSON.stringify(input.context), input.chosenResponse, input.turnsUntilOutcome,
input.isDirectMediaInvitation ? 1 : 0, input.qualityStatus, now, now,
);
return this.getDecision(input.id)!;
}
getTrajectory(id: string): LearningTrajectoryDetail | undefined {
const where = this.accountId ? 'id = ? AND account_id = ?' : 'id = ?';
const params = this.accountId ? [id, this.accountId] : [id];
const row = this.db.prepare(`SELECT * FROM conversation_learning_trajectories WHERE ${where}`)
.get(...params) as TrajectoryRow | undefined;
if (!row) return undefined;
const decisions = (this.db.prepare(`
SELECT * FROM conversation_learning_decisions
WHERE trajectory_id = ? ORDER BY created_at ASC, id ASC
`).all(id) as DecisionRow[]).map(mapDecision);
return { ...mapTrajectory(row), decisions };
}
getDecision(id: string): LearningDecisionRecord | undefined {
const where = this.accountId ? 'id = ? AND account_id = ?' : 'id = ?';
const params = this.accountId ? [id, this.accountId] : [id];
const row = this.db.prepare(`SELECT * FROM conversation_learning_decisions WHERE ${where}`)
.get(...params) as DecisionRow | undefined;
return row ? mapDecision(row) : undefined;
}
listTrajectories(input: { status?: LearningTrajectoryStatus; limit?: number } = {}): LearningTrajectoryRecord[] {
const clauses: string[] = [];
const params: Array<string | number> = [];
if (this.accountId) {
clauses.push('account_id = ?');
params.push(this.accountId);
}
if (input.status) {
clauses.push('status = ?');
params.push(input.status);
}
params.push(Math.min(500, Math.max(1, input.limit ?? 100)));
return (this.db.prepare(`
SELECT * FROM conversation_learning_trajectories
${clauses.length ? `WHERE ${clauses.join(' AND ')}` : ''}
ORDER BY updated_at DESC, id DESC LIMIT ?
`).all(...params) as TrajectoryRow[]).map(mapTrajectory);
}
promoteMature(now = Date.now()): number {
const accountClause = this.accountId ? 'AND account_id = ?' : '';
const params = this.accountId ? [now, now, this.accountId] : [now, now];
return this.db.prepare(`
UPDATE conversation_learning_trajectories
SET status = 'eligible', updated_at = ?
WHERE status = 'provisional' AND maturity_at <= ? ${accountClause}
`).run(...params).changes;
}
setReviewStatus(id: string, status: 'eligible' | 'excluded', now = Date.now()): boolean {
const accountClause = this.accountId ? 'AND account_id = ?' : '';
const params = this.accountId ? [status, now, now, id, this.accountId] : [status, now, now, id];
const changed = this.db.prepare(`
UPDATE conversation_learning_trajectories
SET status = ?, reviewed_at = ?, updated_at = ?, invalidated_at = NULL, invalidation_reason = NULL
WHERE id = ? ${accountClause} AND status != 'invalidated'
`).run(...params).changes > 0;
if (changed) {
this.db.prepare(`
UPDATE conversation_learning_decisions
SET quality_status = ?, updated_at = ? WHERE trajectory_id = ?
`).run(status, now, id);
}
return changed;
}
invalidateBySentMessageId(sentMessageId: number, reason: string, now = Date.now()): number {
const accountId = this.requireAccountId();
const rows = this.db.prepare(`
SELECT id FROM conversation_learning_trajectories
WHERE account_id = ? AND sent_message_id = ? AND status != 'invalidated'
`).all(accountId, sentMessageId) as Array<{ id: string }>;
if (rows.length === 0) return 0;
this.db.transaction(() => {
this.db.prepare(`
UPDATE conversation_learning_trajectories
SET status = 'invalidated', invalidated_at = ?, invalidation_reason = ?, updated_at = ?
WHERE account_id = ? AND sent_message_id = ?
`).run(now, reason.slice(0, 500), now, accountId, sentMessageId);
for (const row of rows) {
this.db.prepare(`
UPDATE conversation_learning_decisions
SET quality_status = 'invalidated', updated_at = ? WHERE trajectory_id = ?
`).run(now, row.id);
}
})();
return rows.length;
}
getEligibleExamples(input: {
conversationId: string;
policyMode: ConversationPolicyMode;
stage: LearningStage;
subreddit: string | null;
minimumConfidence: AttributionConfidence;
humanLimit: number;
autonomousLimit: number;
totalLimit: number;
}): RetrievedLearningExample[] {
const accountId = this.requireAccountId();
const confidenceRank: Record<AttributionConfidence, number> = { inferred: 1, fingerprint: 2, exact: 3 };
const minimumRank = confidenceRank[input.minimumConfidence];
const rows = this.db.prepare(`
SELECT d.*, t.credit_class, t.subreddit, t.policy_mode, t.attribution_confidence
FROM conversation_learning_decisions d
JOIN conversation_learning_trajectories t ON t.id = d.trajectory_id
WHERE d.account_id = ? AND d.quality_status = 'eligible' AND t.status = 'eligible'
AND t.conversation_id != ?
AND (t.policy_mode = ? OR t.policy_mode IS NULL)
AND (d.stage = ? OR d.stage = 'unknown')
ORDER BY
CASE WHEN t.subreddit = ? THEN 0 ELSE 1 END,
CASE t.attribution_confidence WHEN 'exact' THEN 0 WHEN 'fingerprint' THEN 1 ELSE 2 END,
t.updated_at DESC,
d.updated_at DESC
LIMIT 200
`).all(accountId, input.conversationId, input.policyMode, input.stage, input.subreddit) as RetrievalRow[];
const eligible = rows.filter((row) => confidenceRank[row.attribution_confidence] >= minimumRank);
const human = eligible.filter((row) => row.credit_class.startsWith('human-')).slice(0, input.humanLimit);
const autonomous = eligible.filter((row) =>
row.credit_class === 'autonomous-success' || row.credit_class === 'recipient-initiated-media')
.slice(0, input.autonomousLimit);
return [...human, ...autonomous].slice(0, input.totalLimit).map(mapRetrieval);
}
recordGeneration(input: {
conversationId: string;
incomingEventId: string;
purpose: 'reply' | 'media-ask';
mode: 'shadow' | 'injected';
policyVersion: string;
exampleIds: readonly string[];
promptSha256?: string | null;
generatedAt?: number;
}): void {
const accountId = this.requireAccountId();
this.db.prepare(`
INSERT INTO learning_generation_context (
account_id, conversation_id, incoming_event_id, purpose, mode,
policy_version, example_ids_json, prompt_sha256, generated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(account_id, conversation_id, incoming_event_id, purpose, policy_version, mode)
DO UPDATE SET example_ids_json = excluded.example_ids_json,
prompt_sha256 = excluded.prompt_sha256, generated_at = excluded.generated_at
`).run(
accountId, input.conversationId, input.incomingEventId, input.purpose, input.mode,
input.policyVersion, JSON.stringify(input.exampleIds), input.promptSha256 ?? null,
input.generatedAt ?? Date.now(),
);
}
getSummary(): LearningSummary {
const accountClause = this.accountId ? 'WHERE account_id = ?' : '';
const params = this.accountId ? [this.accountId] : [];
const row = this.db.prepare(`
SELECT COUNT(*) AS total,
SUM(status = 'provisional') AS provisional,
SUM(status = 'eligible') AS eligible,
SUM(status = 'excluded') AS excluded,
SUM(status = 'invalidated') AS invalidated,
SUM(status = 'needs-review') AS needs_review,
SUM(credit_class = 'human-created-success') AS human_created,
SUM(credit_class = 'human-improved-success') AS human_improved,
SUM(credit_class = 'human-completed-ai-trajectory') AS human_completed_ai,
SUM(credit_class = 'autonomous-success') AS autonomous,
SUM(credit_class = 'recipient-initiated-media') AS recipient_initiated,
SUM(credit_class = 'uncertain') AS uncertain,
SUM(attribution_confidence = 'exact') AS exact,
SUM(attribution_confidence = 'fingerprint') AS fingerprint,
SUM(attribution_confidence = 'inferred') AS inferred
FROM conversation_learning_trajectories ${accountClause}
`).get(...params) as Record<string, number | null>;
return {
total: row.total ?? 0,
provisional: row.provisional ?? 0,
eligible: row.eligible ?? 0,
excluded: row.excluded ?? 0,
invalidated: row.invalidated ?? 0,
needsReview: row.needs_review ?? 0,
humanCreated: row.human_created ?? 0,
humanImproved: row.human_improved ?? 0,
humanCompletedAi: row.human_completed_ai ?? 0,
autonomous: row.autonomous ?? 0,
recipientInitiated: row.recipient_initiated ?? 0,
uncertain: row.uncertain ?? 0,
exact: row.exact ?? 0,
fingerprint: row.fingerprint ?? 0,
inferred: row.inferred ?? 0,
};
}
listEvaluationRuns(limit = 20): LearningEvaluationRun[] {
return (this.db.prepare(`
SELECT * FROM learning_evaluation_runs ORDER BY created_at DESC LIMIT ?
`).all(Math.min(100, Math.max(1, limit))) as EvaluationRow[]).map((row) => ({
id: row.id,
policyVersion: row.policy_version,
datasetVersion: row.dataset_version,
baselinePolicyVersion: row.baseline_policy_version,
status: row.status,
totalCases: row.total_cases,
hardGateFailures: row.hard_gate_failures,
summary: parseObject(row.summary_json),
createdAt: row.created_at,
completedAt: row.completed_at,
}));
}
saveEvaluationRun(run: LearningEvaluationRun): void {
this.db.prepare(`
INSERT INTO learning_evaluation_runs (
id, policy_version, dataset_version, baseline_policy_version, status,
total_cases, hard_gate_failures, summary_json, created_at, completed_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET status = excluded.status,
total_cases = excluded.total_cases,
hard_gate_failures = excluded.hard_gate_failures,
summary_json = excluded.summary_json,
completed_at = excluded.completed_at
`).run(
run.id, run.policyVersion, run.datasetVersion, run.baselinePolicyVersion,
run.status, run.totalCases, run.hardGateFailures, JSON.stringify(run.summary),
run.createdAt, run.completedAt,
);
}
private requireAccountId(): string {
if (!this.accountId) throw new Error('An account ID is required for this learning operation');
return this.accountId;
}
}
export interface LearningEvaluationRun {
id: string;
policyVersion: string;
datasetVersion: string;
baselinePolicyVersion: string | null;
status: 'running' | 'passed' | 'failed';
totalCases: number;
hardGateFailures: number;
summary: Record<string, unknown>;
createdAt: number;
completedAt: number | null;
}
interface TrajectoryRow {
id: string;
account_id: string;
conversation_id: string;
sent_message_id: number | null;
subreddit: string | null;
source_type: 'image' | 'chat' | null;
policy_mode: ConversationPolicyMode | null;
first_human_event_id: string | null;
outcome_event_id: string;
outcome_type: 'image-received' | 'reciprocal-engagement';
credit_class: LearningCreditClass;
attribution_confidence: AttributionConfidence;
status: LearningTrajectoryStatus;
maturity_at: number;
invalidated_at: number | null;
invalidation_reason: string | null;
reviewed_at: number | null;
created_at: number;
updated_at: number;
}
interface DecisionRow {
id: string;
trajectory_id: string;
account_id: string;
decision_event_id: string;
stage: LearningStage;
context_json: string;
chosen_response: string;
turns_until_outcome: number;
is_direct_media_invitation: number;
quality_status: LearningDecisionStatus;
created_at: number;
updated_at: number;
}
interface RetrievalRow extends DecisionRow {
credit_class: LearningCreditClass;
subreddit: string | null;
policy_mode: ConversationPolicyMode | null;
attribution_confidence: AttributionConfidence;
}
interface EvaluationRow {
id: string;
policy_version: string;
dataset_version: string;
baseline_policy_version: string | null;
status: 'running' | 'passed' | 'failed';
total_cases: number;
hard_gate_failures: number;
summary_json: string;
created_at: number;
completed_at: number | null;
}
function mapTrajectory(row: TrajectoryRow): LearningTrajectoryRecord {
return {
id: row.id,
accountId: row.account_id,
conversationId: row.conversation_id,
sentMessageId: row.sent_message_id,
subreddit: row.subreddit,
sourceType: row.source_type,
policyMode: row.policy_mode,
firstHumanEventId: row.first_human_event_id,
outcomeEventId: row.outcome_event_id,
outcomeType: row.outcome_type,
creditClass: row.credit_class,
attributionConfidence: row.attribution_confidence,
status: row.status,
maturityAt: row.maturity_at,
invalidatedAt: row.invalidated_at,
invalidationReason: row.invalidation_reason,
reviewedAt: row.reviewed_at,
createdAt: row.created_at,
updatedAt: row.updated_at,
};
}
function mapDecision(row: DecisionRow): LearningDecisionRecord {
const parsed = LearningTurnSchema.array().safeParse(parseJson(row.context_json));
return {
id: row.id,
trajectoryId: row.trajectory_id,
accountId: row.account_id,
decisionEventId: row.decision_event_id,
stage: row.stage,
context: parsed.success ? parsed.data : [],
chosenResponse: row.chosen_response,
turnsUntilOutcome: row.turns_until_outcome,
isDirectMediaInvitation: Boolean(row.is_direct_media_invitation),
qualityStatus: row.quality_status,
createdAt: row.created_at,
updatedAt: row.updated_at,
};
}
function mapRetrieval(row: RetrievalRow): RetrievedLearningExample {
const decision = mapDecision(row);
return {
decisionId: decision.id,
trajectoryId: decision.trajectoryId,
creditClass: row.credit_class,
stage: decision.stage,
context: decision.context,
chosenResponse: decision.chosenResponse,
turnsUntilOutcome: decision.turnsUntilOutcome,
subreddit: row.subreddit,
policyMode: row.policy_mode,
};
}
function parseJson(value: string): unknown {
try { return JSON.parse(value) as unknown; }
catch { return null; }
}
function parseObject(value: string): Record<string, unknown> {
const parsed = parseJson(value);
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
? parsed as Record<string, unknown>
: {};
}
export function stableLearningId(...parts: readonly string[]): string {
return createHash('sha256').update(parts.join('\u0000')).digest('hex').slice(0, 32);
}