feat: show the conversation evidence behind each learning trajectory

Reviewers were asked to include or exclude learning trajectories while
only seeing metadata — credit class, confidence, status. The decisions
and their redacted context turns were already captured and stored, but
no endpoint returned them, so the evidence was unreachable from the UI.

Add GET /api/learning/trajectories/:id and an expandable row on the
learning page that fetches on demand.

Also widen SentMessageSchema to accept matrixEventId, imageOutcomeCredit
and imageOutcomeAt. The learning loop added these to SentMessageRecord
and they flow out of /api/messages, but the strict contract rejected
them, breaking the dashboard.

Enable generation with shadow mode off so eligible examples reach reply
prompts.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Jamie BruntonandClaude Opus 5 committed 2026-08-12 08:23:22 +01:00
1 parent 687abab82b
commit 142378df33
25 files changed
+836 -28

No files matched your search

+64
View File
@@ -134,6 +134,70 @@ describe('multi-account database ownership', () => {
expect(db.pragma('foreign_key_check')).toEqual([]);
});
it('removes an account with everything it owns and leaves other accounts intact', () => {
addAccount('t2_alice', 'alice-bot', 100);
addAccount('t2_bob', 'bob-bot', 110);
for (const accountId of ['t2_alice', 't2_bob']) {
db.prepare(`
INSERT INTO processed_posts (account_id, id, subreddit, author, title, processed_at)
VALUES (?, 'post-one', 'test', 'recipient', 'title', 1)
`).run(accountId);
db.prepare(`
INSERT INTO sent_messages (account_id, post_id, recipient, subject, message, sent_at)
VALUES (?, 'post-one', 'recipient', 'subject', 'message', 2)
`).run(accountId);
db.prepare(`
INSERT INTO successful_messages (account_id, original_message, created_at)
VALUES (?, 'message', 3)
`).run(accountId);
db.prepare(`
INSERT INTO delivery_outbox (
account_id, post_id, recipient, subject, message, idempotency_key,
status, created_at, updated_at
) VALUES (?, 'post-one', 'recipient', 'subject', 'message', 'key', 'prepared', 1, 1)
`).run(accountId);
db.prepare(`
INSERT INTO activity_log (account_id, timestamp, type, message) VALUES (?, 4, 'sent', 'activity')
`).run(accountId);
db.prepare(`
INSERT INTO conversation_states (account_id, conversation_id, state_json, updated_at)
VALUES (?, 'room', '{}', 5)
`).run(accountId);
content.upsertConversation({
accountId, conversationId: 'room', username: 'recipient', roomName: 'Recipient',
payloadJson: '{}', updatedAt: 6,
});
content.saveSyncState({
accountId, nextBatch: 'cursor', lastFullSyncAt: 6, lastDeltaSyncAt: 6,
cooldownJson: null, updatedAt: 6,
});
}
expect(accounts.delete('t2_alice')).toBe(true);
expect(accounts.get('t2_alice')).toBeNull();
expect(accounts.listPublic().map((account) => account.accountId)).toContain('t2_bob');
for (const table of [
'processed_posts', 'sent_messages', 'successful_messages', 'delivery_outbox',
'activity_log', 'conversation_states', 'reddit_conversations', 'reddit_account_sync_state',
]) {
expect({ table, ...db.prepare(`SELECT COUNT(*) count FROM ${table} WHERE account_id = ?`).get('t2_alice') as object })
.toEqual({ table, count: 0 });
expect({ table, ...db.prepare(`SELECT COUNT(*) count FROM ${table} WHERE account_id = ?`).get('t2_bob') as object })
.toEqual({ table, count: 1 });
}
expect(db.pragma('foreign_key_check')).toEqual([]);
});
it('reports an unknown account as not removed and rejects a blank id', () => {
addAccount('t2_alice', 'alice-bot', 100);
expect(accounts.delete('t2_missing')).toBe(false);
expect(() => accounts.delete(' ')).toThrow(/accountId/);
expect(accounts.get('t2_alice')).not.toBeNull();
});
it('scopes legacy account-produced table uniqueness by account', () => {
addAccount('t2_alice', 'alice-bot', 100);
addAccount('t2_bob', 'bob-bot', 110);
+112
View File
@@ -0,0 +1,112 @@
// @vitest-environment jsdom
import '@testing-library/jest-dom/vitest';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { cleanup, render, screen, waitFor } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import { HttpResponse, http } from 'msw';
import { setupServer } from 'msw/node';
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { LearningPage } from '../../apps/dashboard/src/features/learning/learning-page.js';
const TRAJECTORY_ID = 'traj-1';
let detailCalls: string[] = [];
function summary() {
return {
total: 1, provisional: 0, eligible: 1, excluded: 0, invalidated: 0, needsReview: 0,
humanCreated: 1, humanImproved: 0, humanCompletedAi: 0, autonomous: 0,
recipientInitiated: 0, uncertain: 0, exact: 1, fingerprint: 0, inferred: 0,
};
}
function trajectory() {
return {
id: TRAJECTORY_ID, accountId: '@t2_learner:reddit.com', conversationId: '!room:reddit.com',
sentMessageId: 7, subreddit: 'cats', sourceType: 'chat' as const, policyMode: 'r4r-media' as const,
firstHumanEventId: '$human1', outcomeEventId: '$image1', outcomeType: 'image-received' as const,
creditClass: 'human-created-success' as const, attributionConfidence: 'exact' as const,
status: 'eligible' as const, maturityAt: 1_700_000_100_000, invalidatedAt: null,
invalidationReason: null, reviewedAt: null, createdAt: 1_700_000_000_000, updatedAt: 1_700_000_050_000,
};
}
function decision() {
return {
id: 'dec-1', trajectoryId: TRAJECTORY_ID, accountId: '@t2_learner:reddit.com',
decisionEventId: '$human1', stage: 'media-invitation' as const,
context: [{
eventId: '$in1', timestamp: 1_700_000_010_000, role: 'recipient' as const,
actor: 'recipient' as const, kind: 'inbound-text' as const, text: 'hey how was your weekend',
}],
chosenResponse: 'no pressure, only if you want to',
turnsUntilOutcome: 2, isDirectMediaInvitation: true, qualityStatus: 'eligible' as const,
createdAt: 1_700_000_000_000, updatedAt: 1_700_000_050_000,
};
}
const server = setupServer(
http.get('/api/learning', () => HttpResponse.json({
summary: summary(), trajectories: [trajectory()],
captureEnabled: true, generationEnabled: false, shadowMode: true, policyVersion: 'baseline-v1',
})),
http.get('/api/learning/trajectories/:id', ({ params }) => {
detailCalls.push(String(params.id));
return HttpResponse.json({ trajectory: trajectory(), decisions: [decision()] });
}),
);
function renderLearning() {
const queryClient = new QueryClient({
defaultOptions: { queries: { retry: false }, mutations: { retry: false } },
});
return render(
<QueryClientProvider client={queryClient}>
<LearningPage />
</QueryClientProvider>,
);
}
beforeAll(() => server.listen({ onUnhandledRequest: 'error' }));
afterEach(() => {
server.resetHandlers();
cleanup();
});
afterAll(() => server.close());
beforeEach(() => {
detailCalls = [];
});
describe('learning page trajectory detail', () => {
it('does not fetch decisions until a row is expanded', async () => {
renderLearning();
await screen.findByText(/human created success/i);
expect(detailCalls).toEqual([]);
});
it('reveals the redacted conversation evidence when expanded', async () => {
renderLearning();
await screen.findByText(/human created success/i);
await userEvent.click(screen.getByRole('button', { name: /show conversation evidence/i }));
expect(await screen.findByText('no pressure, only if you want to')).toBeInTheDocument();
expect(screen.getByText('hey how was your weekend')).toBeInTheDocument();
expect(screen.getByText('!room:reddit.com')).toBeInTheDocument();
expect(screen.getByText(/direct media ask/i)).toBeInTheDocument();
await waitFor(() => expect(detailCalls).toEqual([TRAJECTORY_ID]));
});
it('collapses the evidence again on a second click', async () => {
renderLearning();
await screen.findByText(/human created success/i);
const toggle = screen.getByRole('button', { name: /show conversation evidence/i });
await userEvent.click(toggle);
await screen.findByText('no pressure, only if you want to');
await userEvent.click(toggle);
await waitFor(() => {
expect(screen.queryByText('no pressure, only if you want to')).not.toBeInTheDocument();
});
expect(toggle).toHaveAttribute('aria-expanded', 'false');
});
});
+48
View File
@@ -149,6 +149,54 @@ describe('Reddit session status', () => {
expect((await request(app, '/login/start', { method: 'POST' })).status).toBe(404);
});
it('removes an account and reports the account it deleted', async () => {
const remove = vi.fn().mockResolvedValue({ accountId: 't2_alice', redditUsername: 'alice-bot' });
const app = express();
app.use(createLoginRouter({
services: { redditAccounts: { remove } },
logger: { error: () => undefined, info: () => undefined },
} as never));
const response = await request(app, '/accounts/t2_alice', { method: 'DELETE' });
expect(response.status).toBe(200);
expect(await response.json()).toEqual({
ok: true,
account: { accountId: 't2_alice', redditUsername: 'alice-bot' },
});
expect(remove).toHaveBeenCalledWith('t2_alice');
});
it('answers 404 when the account to remove does not exist', async () => {
const app = express();
app.use(createLoginRouter({
services: {
redditAccounts: { remove: async () => { throw new Error('Reddit account not found'); } },
},
logger: { error: () => undefined, info: () => undefined },
} as never));
const response = await request(app, '/accounts/t2_missing', { method: 'DELETE' });
expect(response.status).toBe(404);
expect(await response.json()).toEqual({ ok: false, error: 'Reddit account not found' });
});
it('reports a failed removal as a server error rather than a silent success', async () => {
const app = express();
app.use(createLoginRouter({
services: {
redditAccounts: { remove: async () => { throw new Error('database is locked'); } },
},
logger: { error: () => undefined, info: () => undefined },
} as never));
const response = await request(app, '/accounts/t2_alice', { method: 'DELETE' });
expect(response.status).toBe(500);
expect(await response.json()).toEqual({ ok: false, error: 'database is locked' });
});
it('classifies Reddit navigation failures as unreachable', () => {
expect(
classifyRedditSessionError(
+104
View File
@@ -0,0 +1,104 @@
import Database from 'better-sqlite3';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { RedditAccountsRepository } from '../src/database/account-repositories.js';
import { runMigrations } from '../src/database/client.js';
import { AccountAuthContext } from '../src/services/account-auth-context.js';
import { RedditAccountsService } from '../src/services/reddit-accounts.js';
const silentLogger = {
info: () => undefined,
warn: () => undefined,
error: () => undefined,
debug: () => undefined,
} as never;
describe('removing a Reddit account', () => {
let db: Database.Database;
let accounts: RedditAccountsRepository;
let auth: AccountAuthContext;
let stopped: string[];
let service: RedditAccountsService;
beforeEach(() => {
db = new Database(':memory:');
db.pragma('foreign_keys = ON');
runMigrations(db);
accounts = new RedditAccountsRepository(db);
auth = new AccountAuthContext(silentLogger);
stopped = [];
const runtimes = {
get: () => null,
stop: async (accountId: string) => { stopped.push(accountId); },
};
service = new RedditAccountsService(
accounts,
{ import: async () => { throw new Error('not used'); } } as never,
auth,
runtimes as never,
);
accounts.upsertVerified({
accountId: 't2_alice',
redditUsername: 'alice-bot',
matrixUserId: '@t2_alice:reddit.com',
cookiesJson: JSON.stringify([{ name: 'reddit_session', value: 'secret' }]),
matrixCredentialsJson: JSON.stringify({ accessToken: 'matrix-secret' }),
now: 100,
});
});
afterEach(() => db.close());
it('stops the runtime before dropping the account and its stored credentials', async () => {
const removed = await service.remove('t2_alice');
expect(stopped).toEqual(['t2_alice']);
expect(removed).toMatchObject({
accountId: 't2_alice',
redditUsername: 'alice-bot',
isRunning: false,
status: 'login-required',
});
expect(accounts.get('t2_alice')).toBeNull();
expect(service.list()).toEqual([]);
});
it('drops the authenticated session when the removed account held it', async () => {
auth.markAuthenticated({
accountId: 't2_alice',
redditUsername: 'alice-bot',
matrixUserId: '@t2_alice:reddit.com',
});
expect(auth.getSnapshot().accountId).toBe('t2_alice');
await service.remove('t2_alice');
expect(auth.getSnapshot()).toMatchObject({ status: 'login-required', accountId: null });
});
it('leaves another account\'s session alone', async () => {
accounts.upsertVerified({
accountId: 't2_bob',
redditUsername: 'bob-bot',
matrixUserId: '@t2_bob:reddit.com',
cookiesJson: JSON.stringify([{ name: 'reddit_session', value: 'bob-secret' }]),
matrixCredentialsJson: JSON.stringify({ accessToken: 'bob-matrix-secret' }),
now: 110,
});
auth.markAuthenticated({
accountId: 't2_bob',
redditUsername: 'bob-bot',
matrixUserId: '@t2_bob:reddit.com',
});
await service.remove('t2_alice');
expect(auth.getSnapshot()).toMatchObject({ status: 'authenticated', accountId: 't2_bob' });
expect(service.list().map((account) => account.accountId)).toEqual(['t2_bob']);
});
it('refuses to remove an account it does not know, without stopping anything', async () => {
await expect(service.remove('t2_missing')).rejects.toThrow('Reddit account not found');
expect(stopped).toEqual([]);
expect(accounts.get('t2_alice')).not.toBeNull();
});
});
+118
View File
@@ -0,0 +1,118 @@
import express from 'express';
import Database from 'better-sqlite3';
import type { AddressInfo } from 'node:net';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { LearningTrajectoryDetailResponseSchema } from '../../packages/api-contracts/src/index.js';
import { runMigrations } from '../../src/database/client.js';
import { ConversationLearningRepository } from '../../src/database/conversation-learning-repository.js';
import type { WebContext } from '../../src/web/context.js';
import { createLearningRouter } from '../../src/web/routes/learning.js';
const ACCOUNT = '@t2_learner:reddit.com';
const CONVERSATION = '!room:reddit.com';
const servers: Array<ReturnType<express.Express['listen']>> = [];
async function call(app: express.Express, path: string): Promise<{ status: number; json: any }> {
const server = app.listen(0);
servers.push(server);
await new Promise<void>((resolve) => server.once('listening', resolve));
const { port } = server.address() as AddressInfo;
const response = await fetch(`http://127.0.0.1:${port}${path}`);
const text = await response.text();
return { status: response.status, json: text ? JSON.parse(text) : null };
}
afterEach(async () => {
await Promise.all(servers.splice(0).map((server) => (
new Promise<void>((resolve) => server.close(() => resolve()))
)));
});
describe('GET /api/learning/trajectories/:id', () => {
let db: Database.Database;
let repo: ConversationLearningRepository;
let app: express.Express;
beforeEach(() => {
db = new Database(':memory:');
db.pragma('foreign_keys = ON');
runMigrations(db);
db.prepare(`
INSERT INTO reddit_accounts (
account_id, reddit_username, matrix_user_id, status, is_active,
cookies_json, matrix_credentials_json, created_at, updated_at
) VALUES (?, 'learner', ?, 'verified', 1, '[]', '{}', 0, 0)
`).run(ACCOUNT, ACCOUNT);
repo = new ConversationLearningRepository(db, ACCOUNT);
repo.upsertTrajectory({
id: 'traj-1',
conversationId: CONVERSATION,
sentMessageId: null,
subreddit: 'cats',
sourceType: 'chat',
policyMode: 'r4r-media',
firstHumanEventId: '$human1',
outcomeEventId: '$image1',
outcomeType: 'image-received',
creditClass: 'human-created-success',
attributionConfidence: 'exact',
status: 'eligible',
maturityAt: 1_000,
invalidatedAt: null,
invalidationReason: null,
now: 900,
});
repo.upsertDecision({
id: 'dec-1',
trajectoryId: 'traj-1',
decisionEventId: '$human1',
stage: 'media-invitation',
context: [{
eventId: '$in1', timestamp: 500, role: 'recipient', actor: 'recipient',
kind: 'inbound-text', text: 'hey there',
}],
chosenResponse: 'no pressure, only if you want to',
turnsUntilOutcome: 2,
isDirectMediaInvitation: true,
qualityStatus: 'eligible',
now: 900,
});
const ctx = {
config: { learning: { captureEnabled: true, generationEnabled: false, shadowMode: true, policyVersion: 'baseline-v1' } },
services: {
accountRuntimes: {
list: () => [{ accountId: ACCOUNT }],
get: () => ({ resources: { conversationLearningRepo: repo } }),
},
},
} as unknown as WebContext;
app = express();
app.use(express.json());
app.use('/api', createLearningRouter(ctx));
});
it('returns the redacted decisions behind a trajectory', async () => {
const { status, json } = await call(app, '/api/learning/trajectories/traj-1');
expect(status).toBe(200);
const parsed = LearningTrajectoryDetailResponseSchema.parse(json);
expect(parsed.trajectory.conversationId).toBe(CONVERSATION);
expect(parsed.trajectory.creditClass).toBe('human-created-success');
expect(parsed.decisions).toHaveLength(1);
expect(parsed.decisions[0]?.chosenResponse).toBe('no pressure, only if you want to');
expect(parsed.decisions[0]?.context[0]?.text).toBe('hey there');
expect(parsed.decisions[0]?.isDirectMediaInvitation).toBe(true);
});
it('does not nest decisions inside the trajectory object', async () => {
const { json } = await call(app, '/api/learning/trajectories/traj-1');
expect(json.trajectory.decisions).toBeUndefined();
});
it('404s for an unknown trajectory', async () => {
const { status, json } = await call(app, '/api/learning/trajectories/missing');
expect(status).toBe(404);
expect(json.error).toBe('Learning trajectory not found');
});
});
+42 -1
View File
@@ -1,7 +1,7 @@
import express from 'express';
import type { AddressInfo } from 'node:net';
import { afterEach, describe, expect, it } from 'vitest';
import { StatsResponseSchema } from '../../packages/api-contracts/src/index.js';
import { MessagesResponseSchema, StatsResponseSchema } from '../../packages/api-contracts/src/index.js';
import type { WebContext } from '../../src/web/context.js';
import { createStatsRouter } from '../../src/web/routes/stats.js';
@@ -95,3 +95,44 @@ describe('stats route account status', () => {
expect(stats.accounts[1]?.invitationCooldown).not.toHaveProperty('setAt');
});
});
describe('messages route contract', () => {
it('accepts the attribution fields the learning loop adds to sent messages', async () => {
const context = {
services: {},
repos: {
messages: {
getRecent: () => [{
id: 1,
postId: 'p1',
recipient: 'someone',
subject: 'hey',
message: 'hello there',
sentAt: 1_700_000_000_000,
gotResponse: true,
gotImage: true,
responseCheckedAt: 1_700_000_100_000,
strategyId: null,
chatRoomId: '!room:reddit.com',
// Added by the conversation learning loop; must not break the dashboard.
matrixEventId: '$opener1',
imageOutcomeCredit: 'human-created-success',
imageOutcomeAt: 1_700_000_200_000,
}],
},
},
activity: { recent: () => [], totalProcessed: () => 0 },
getBotRunning: () => true,
getBotMode: () => 'running',
getMessagesThisHour: () => 0,
} as unknown as WebContext;
const app = express().use('/api', createStatsRouter(context));
const response = await request(app, '/api/messages');
const messages = MessagesResponseSchema.parse(await response.json());
expect(messages[0]?.matrixEventId).toBe('$opener1');
expect(messages[0]?.imageOutcomeCredit).toBe('human-created-success');
expect(messages[0]?.imageOutcomeAt).toBe(1_700_000_200_000);
});
});