Add human-led conversation learning loop
This commit is contained in:
1 parent
bc42c55555
commit
687abab82b
52 files changed
+2112
-53
No files matched your search
@@ -0,0 +1,83 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { runMigrations } from '../src/database/client.js';
|
||||
import { ConversationEventRepository } from '../src/database/conversation-event-repository.js';
|
||||
import { ConversationLearningRepository } from '../src/database/conversation-learning-repository.js';
|
||||
import { MessagesRepository } from '../src/database/repositories.js';
|
||||
import { RedditAccountsRepository } from '../src/database/account-repositories.js';
|
||||
import { attributeConversationMessages } from '../src/services/conversation-attribution.js';
|
||||
import { extractConversationLearning } from '../src/services/conversation-learning-extractor.js';
|
||||
import { redactLearningText } from '../src/services/conversation-learning-redactor.js';
|
||||
import { runConversationLearningEvaluation } from '../src/services/conversation-learning-evaluator.js';
|
||||
import { RepositoryConversationLearningProvider } from '../src/services/conversation-learning-provider.js';
|
||||
import type { LearningConfig } from '../src/config/index.js';
|
||||
|
||||
const learningConfig: LearningConfig = {
|
||||
captureEnabled: true, generationEnabled: false, shadowMode: true, maturityHours: 12,
|
||||
maxContextExamples: 3, maxTurnsPerExample: 8, humanExampleSlots: 2, autonomousExampleSlots: 1,
|
||||
minimumAttributionConfidence: 'exact', policyVersion: 'test-v1',
|
||||
};
|
||||
|
||||
describe('conversation learning loop', () => {
|
||||
let db: Database.Database;
|
||||
beforeEach(() => {
|
||||
db = new Database(':memory:');
|
||||
db.pragma('foreign_keys = ON');
|
||||
runMigrations(db);
|
||||
new RedditAccountsRepository(db).upsertVerified({
|
||||
accountId: 'account-a', redditUsername: 'account-a', matrixUserId: 'account-a',
|
||||
cookiesJson: '[]', matrixCredentialsJson: '{}', now: 1,
|
||||
});
|
||||
});
|
||||
afterEach(() => db.close());
|
||||
|
||||
it('credits a durable manual media invitation instead of the AI opener', () => {
|
||||
const messages = new MessagesRepository(db, 'account-a');
|
||||
const openerId = messages.insertChat({
|
||||
recipient: 'recipient', subject: 'recipient', message: 'hey', sentAt: 1_000,
|
||||
chatRoomId: 'room-a', matrixEventId: 'open-1',
|
||||
});
|
||||
const opener = messages.getByChatRoomId('room-a')!;
|
||||
expect(opener.id).toBe(openerId);
|
||||
const events = new ConversationEventRepository(db, 'account-a');
|
||||
events.upsert({
|
||||
conversationId: 'room-a', eventId: 'manual-1', eventTimestamp: 3_000,
|
||||
direction: 'outbound', actor: 'human', eventKind: 'manual-text', text: 'send a selfie if comfortable',
|
||||
sourceReference: 'dashboard:manual-reply', attributionConfidence: 'exact',
|
||||
});
|
||||
const attributed = attributeConversationMessages({
|
||||
conversationId: 'room-a', opener, claims: [], mediaSends: [], messages: [
|
||||
{ eventId: 'open-1', sender: 'Me', text: 'hey', timestamp: 1_000, isSelf: true },
|
||||
{ eventId: 'reply-1', sender: 'Them', text: 'hi', timestamp: 2_000, isSelf: false },
|
||||
{ eventId: 'manual-1', sender: 'Me', text: 'send a selfie if comfortable', timestamp: 3_000, isSelf: true },
|
||||
{ eventId: 'image-1', sender: 'Them', text: '[Image]', timestamp: 4_000, isSelf: false, media: [{ originalUrl: 'x', localPath: 'x', fileName: 'x' }] },
|
||||
],
|
||||
}, events);
|
||||
const learning = new ConversationLearningRepository(db, 'account-a');
|
||||
expect(extractConversationLearning({
|
||||
conversationId: 'room-a', messages: attributed, opener, state: undefined,
|
||||
canonicalOutcome: 'engaged', now: 4_000 + 13 * 60 * 60 * 1_000,
|
||||
}, learning, learningConfig)).toBe(1);
|
||||
const trajectory = learning.listTrajectories()[0];
|
||||
expect(trajectory).toMatchObject({ creditClass: 'human-created-success', status: 'eligible', attributionConfidence: 'exact' });
|
||||
expect(learning.getTrajectory(trajectory.id)?.decisions).toHaveLength(1);
|
||||
const provider = new RepositoryConversationLearningProvider(learning, learningConfig);
|
||||
const query = {
|
||||
accountId: 'account-a', conversationId: 'room-b', subreddit: null, policyMode: 'legacy' as const,
|
||||
stage: 'media-invitation' as const, purpose: 'media-ask' as const, incomingEventId: 'incoming-b',
|
||||
};
|
||||
const examples = provider.getExamples(query);
|
||||
expect(examples).toHaveLength(1);
|
||||
provider.recordGeneration({ query, mode: 'shadow', policyVersion: 'test-v1', exampleIds: [examples[0].decisionId] });
|
||||
expect((db.prepare('SELECT COUNT(*) AS count FROM learning_generation_context').get() as { count: number }).count).toBe(1);
|
||||
});
|
||||
|
||||
it('redacts direct identifiers and passes the offline hard gates', () => {
|
||||
expect(redactLearningText('email me at a@example.com or https://example.com u/person'))
|
||||
.toBe('email me at [email] or [url] [username]');
|
||||
const learning = new ConversationLearningRepository(db);
|
||||
const run = runConversationLearningEvaluation(learning, 'test-v1', null, 10);
|
||||
expect(run).toMatchObject({ status: 'passed', hardGateFailures: 0, totalCases: 0 });
|
||||
expect(learning.listEvaluationRuns()).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user