Restructure chat service, web routes, and add test suite
Split monolithic chat service into modular structure with Matrix client support, extract web server into per-resource route modules, introduce bootstrap/main-loop separation, and add comprehensive vitest coverage across services, repositories, and utilities. Adds project docs and CLAUDE.md agent instructions.
This commit is contained in:
jjbrunton committed
1 parent
dfa0580859
commit
633bb0ec04
84 files changed
+19278
-3888
No files matched your search
@@ -0,0 +1,351 @@
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
||||
import { StrategySelector } from '../src/services/strategy-selector.js';
|
||||
import { MessagesRepository } from '../src/database/repositories.js';
|
||||
import type { StrategyStats } from '../src/database/repositories.js';
|
||||
import { createMockLogger, getLogsByLevel } from '../tests/mockLogger.js';
|
||||
import { getTestConfig } from '../tests/testUtils.js';
|
||||
|
||||
function createMockMessagesRepo(stats: StrategyStats[] = []): MessagesRepository {
|
||||
return {
|
||||
getStrategyStats: vi.fn(() => stats),
|
||||
getSubredditStats: vi.fn(() => []),
|
||||
getResponseRateTrend: vi.fn(() => []),
|
||||
insert: vi.fn(() => 1),
|
||||
getByPostId: vi.fn(),
|
||||
getRecent: vi.fn(),
|
||||
getUncheckedMessages: vi.fn(),
|
||||
markGotResponse: vi.fn(),
|
||||
markChecked: vi.fn(),
|
||||
hasSentTo: vi.fn(),
|
||||
} as unknown as MessagesRepository;
|
||||
}
|
||||
|
||||
describe('StrategySelector', () => {
|
||||
let selector: StrategySelector;
|
||||
let logger: ReturnType<typeof createMockLogger>;
|
||||
|
||||
beforeEach(() => {
|
||||
logger = createMockLogger();
|
||||
});
|
||||
|
||||
it('returns null when no variants configured', () => {
|
||||
const config = getTestConfig({ strategies: { variants: [] } });
|
||||
const repo = createMockMessagesRepo();
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
const result = selector.selectStrategy();
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it('hasStrategies returns false when no variants', () => {
|
||||
const config = getTestConfig({ strategies: { variants: [] } });
|
||||
const repo = createMockMessagesRepo();
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
expect(selector.hasStrategies()).toBe(false);
|
||||
});
|
||||
|
||||
it('hasStrategies returns true when variants exist', () => {
|
||||
const config = getTestConfig({ strategies: { variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'test' },
|
||||
] } });
|
||||
const repo = createMockMessagesRepo();
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
expect(selector.hasStrategies()).toBe(true);
|
||||
});
|
||||
|
||||
describe('explore mode', () => {
|
||||
it('selects random variant when exploration rate is 1.0', () => {
|
||||
const config = getTestConfig({ strategies: { explorationRate: 1.0, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'prompt1' },
|
||||
{ id: 'v2', weight: 1, promptOverride: 'prompt2' },
|
||||
] } });
|
||||
const repo = createMockMessagesRepo();
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const result = selector.selectStrategy();
|
||||
expect(result).not.toBeNull();
|
||||
expect(['v1', 'v2']).toContain(result!.id);
|
||||
expect(result!.reason).toBe('explore');
|
||||
});
|
||||
|
||||
it('selects random variant when exploration rate is 0.5', () => {
|
||||
const config = getTestConfig({ strategies: { explorationRate: 0.5, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'prompt1' },
|
||||
] } });
|
||||
const repo = createMockMessagesRepo();
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
// With 50% exploration rate, we can't guarantee the outcome,
|
||||
// but we can verify it returns a valid strategy
|
||||
const results = new Set<string>();
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const result = selector.selectStrategy();
|
||||
if (result) results.add(result.id);
|
||||
}
|
||||
expect(results.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('logs explore selection', () => {
|
||||
const config = getTestConfig({ strategies: { explorationRate: 1.0, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'prompt1' },
|
||||
] } });
|
||||
const repo = createMockMessagesRepo();
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
selector.selectStrategy();
|
||||
const debugLogs = getLogsByLevel(logger, 'debug');
|
||||
expect(debugLogs.some(l => l.message.includes('explore'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('exploitation mode', () => {
|
||||
it('selects best performing strategy when all have enough samples', () => {
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'v1', totalSent: 10, totalResponses: 2, responseRate: 20 },
|
||||
{ strategyId: 'v2', totalSent: 10, totalResponses: 5, responseRate: 50 },
|
||||
{ strategyId: 'v3', totalSent: 10, totalResponses: 1, responseRate: 10 },
|
||||
];
|
||||
const repo = createMockMessagesRepo(stats);
|
||||
const config = getTestConfig({ strategies: { explorationRate: 0.0, minSamples: 5, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
{ id: 'v2', weight: 1, promptOverride: 'p2' },
|
||||
{ id: 'v3', weight: 1, promptOverride: 'p3' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const result = selector.selectStrategy();
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.id).toBe('v2');
|
||||
expect(result!.reason).toBe('exploit');
|
||||
});
|
||||
|
||||
it('selects first variant when all have equal response rates', () => {
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'v1', totalSent: 10, totalResponses: 5, responseRate: 50 },
|
||||
{ strategyId: 'v2', totalSent: 10, totalResponses: 5, responseRate: 50 },
|
||||
];
|
||||
const repo = createMockMessagesRepo(stats);
|
||||
const config = getTestConfig({ strategies: { explorationRate: 0.0, minSamples: 5, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
{ id: 'v2', weight: 1, promptOverride: 'p2' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const result = selector.selectStrategy();
|
||||
expect(result).not.toBeNull();
|
||||
// When rates are equal, the first one with the best rate wins
|
||||
// Since v1 has rate 50 and v2 has rate 50, v1 wins (first encountered)
|
||||
expect(result!.id).toBe('v1');
|
||||
});
|
||||
|
||||
it('selects best strategy when some have 0 responses', () => {
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'v1', totalSent: 10, totalResponses: 0, responseRate: 0 },
|
||||
{ strategyId: 'v2', totalSent: 10, totalResponses: 3, responseRate: 30 },
|
||||
];
|
||||
const repo = createMockMessagesRepo(stats);
|
||||
const config = getTestConfig({ strategies: { explorationRate: 0.0, minSamples: 5, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
{ id: 'v2', weight: 1, promptOverride: 'p2' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const result = selector.selectStrategy();
|
||||
expect(result!.id).toBe('v2');
|
||||
});
|
||||
|
||||
it('logs exploit selection with response rate', () => {
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'v1', totalSent: 10, totalResponses: 5, responseRate: 50 },
|
||||
];
|
||||
const repo = createMockMessagesRepo(stats);
|
||||
const config = getTestConfig({ strategies: { explorationRate: 0.0, minSamples: 5, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
selector.selectStrategy();
|
||||
const debugLogs = getLogsByLevel(logger, 'debug');
|
||||
expect(debugLogs.some(l => l.message.includes('exploit'))).toBe(true);
|
||||
expect(debugLogs.some(l => l.message.includes('50'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('under-sampled strategies', () => {
|
||||
it('selects under-sampled variant even in exploit mode', () => {
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'v1', totalSent: 10, totalResponses: 2, responseRate: 20 },
|
||||
// v2 has no stats (never used)
|
||||
];
|
||||
const repo = createMockMessagesRepo(stats);
|
||||
const config = getTestConfig({ strategies: { explorationRate: 0.0, minSamples: 5, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
{ id: 'v2', weight: 1, promptOverride: 'p2' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
// v2 has 0 samples which is < minSamples(5), so it should be selected
|
||||
const result = selector.selectStrategy();
|
||||
expect(result!.id).toBe('v2');
|
||||
expect(result!.reason).toBe('explore');
|
||||
});
|
||||
|
||||
it('selects under-sampled variant with insufficient samples', () => {
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'v1', totalSent: 3, totalResponses: 1, responseRate: 33 },
|
||||
// v2 has no stats
|
||||
];
|
||||
const repo = createMockMessagesRepo(stats);
|
||||
const config = getTestConfig({ strategies: { explorationRate: 0.0, minSamples: 5, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
{ id: 'v2', weight: 1, promptOverride: 'p2' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
// Both variants are under-sampled (v1: 3 < 5, v2: 0 < 5) so selection
|
||||
// is random between them. Just assert the explore branch is taken.
|
||||
const result = selector.selectStrategy();
|
||||
expect(['v1', 'v2']).toContain(result!.id);
|
||||
expect(result!.reason).toBe('explore');
|
||||
});
|
||||
|
||||
it('skips under-sampled check when all have enough samples', () => {
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'v1', totalSent: 10, totalResponses: 2, responseRate: 20 },
|
||||
{ strategyId: 'v2', totalSent: 10, totalResponses: 8, responseRate: 80 },
|
||||
];
|
||||
const repo = createMockMessagesRepo(stats);
|
||||
const config = getTestConfig({ strategies: { explorationRate: 0.0, minSamples: 5, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
{ id: 'v2', weight: 1, promptOverride: 'p2' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const result = selector.selectStrategy();
|
||||
expect(result!.id).toBe('v2');
|
||||
expect(result!.reason).toBe('exploit');
|
||||
});
|
||||
|
||||
it('prioritizes unexplored over under-sampled', () => {
|
||||
// v1 has 10 samples (above minSamples), v2 has none — only v2 is unexplored.
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'v1', totalSent: 10, totalResponses: 5, responseRate: 50 },
|
||||
];
|
||||
const repo = createMockMessagesRepo(stats);
|
||||
const config = getTestConfig({ strategies: { explorationRate: 0.0, minSamples: 5, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
{ id: 'v2', weight: 1, promptOverride: 'p2' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const result = selector.selectStrategy();
|
||||
expect(result!.id).toBe('v2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getStats', () => {
|
||||
it('returns all configured variants', () => {
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'v1', totalSent: 5, totalResponses: 2, responseRate: 40 },
|
||||
];
|
||||
const repo = createMockMessagesRepo(stats);
|
||||
const config = getTestConfig({ strategies: { variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
{ id: 'v2', weight: 1, promptOverride: 'p2' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const result = selector.getStats();
|
||||
expect(result.length).toBe(2);
|
||||
expect(result[0].configured).toBe(true);
|
||||
expect(result[1].configured).toBe(true);
|
||||
});
|
||||
|
||||
it('includes zero counts for variants with no data', () => {
|
||||
const repo = createMockMessagesRepo([]);
|
||||
const config = getTestConfig({ strategies: { variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const result = selector.getStats();
|
||||
expect(result[0].totalSent).toBe(0);
|
||||
expect(result[0].totalResponses).toBe(0);
|
||||
expect(result[0].responseRate).toBe(0);
|
||||
});
|
||||
|
||||
it('includes historical strategies not in config', () => {
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'old-strategy', totalSent: 10, totalResponses: 3, responseRate: 30 },
|
||||
];
|
||||
const repo = createMockMessagesRepo(stats);
|
||||
const config = getTestConfig({ strategies: { variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const result = selector.getStats();
|
||||
expect(result.length).toBe(2);
|
||||
const historical = result.find(r => r.strategyId === 'old-strategy');
|
||||
expect(historical).toBeDefined();
|
||||
expect(historical!.configured).toBe(false);
|
||||
});
|
||||
|
||||
it('maps response rates correctly', () => {
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'v1', totalSent: 10, totalResponses: 3, responseRate: 30 },
|
||||
];
|
||||
const repo = createMockMessagesRepo(stats);
|
||||
const config = getTestConfig({ strategies: { variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const result = selector.getStats();
|
||||
expect(result[0].totalSent).toBe(10);
|
||||
expect(result[0].totalResponses).toBe(3);
|
||||
expect(result[0].responseRate).toBe(30);
|
||||
});
|
||||
});
|
||||
|
||||
describe('edge cases', () => {
|
||||
it('handles single variant', () => {
|
||||
const config = getTestConfig({ strategies: { explorationRate: 0.0, variants: [
|
||||
{ id: 'only', weight: 1, promptOverride: 'only one' },
|
||||
] } });
|
||||
const repo = createMockMessagesRepo();
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const result = selector.selectStrategy();
|
||||
expect(result!.id).toBe('only');
|
||||
});
|
||||
|
||||
it('handles many variants', () => {
|
||||
const variants = Array.from({ length: 10 }, (_, i) => ({
|
||||
id: `v${i}`,
|
||||
weight: 1,
|
||||
promptOverride: `prompt ${i}`,
|
||||
}));
|
||||
const config = getTestConfig({ strategies: { explorationRate: 1.0, variants } });
|
||||
const repo = createMockMessagesRepo();
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const results = new Set<string>();
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const result = selector.selectStrategy();
|
||||
if (result) results.add(result.id);
|
||||
}
|
||||
expect(results.size).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it('returns promptOverride from the selected variant', () => {
|
||||
const config = getTestConfig({ strategies: { explorationRate: 1.0, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'My custom prompt' },
|
||||
] } });
|
||||
const repo = createMockMessagesRepo();
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
const result = selector.selectStrategy();
|
||||
expect(result!.promptOverride).toBe('My custom prompt');
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user