Add varied messaging experiments

This commit is contained in:
jjbrunton committed 2026-08-09 10:28:55 +01:00
1 parent 43f459a024
commit 23d4c3a320
12 files changed
+217 -72

No files matched your search

+1 -1
View File
@@ -227,7 +227,7 @@ describe('ConfigSchema', () => {
expect(result.success).toBe(true);
if (result.success) {
expect(result.data.strategies.explorationRate).toBe(0.2);
expect(result.data.strategies.minSamples).toBe(30);
expect(result.data.strategies.minSamples).toBe(20);
expect(result.data.strategies.responseWindowHours).toBe(12);
expect(result.data.strategies.variants).toEqual([]);
expect(result.data.chatStrategies.responseWindowHours).toBe(12);
+25 -1
View File
@@ -470,7 +470,31 @@ describe('OpenRouterClient', () => {
expect(prompt).toContain('Example 1:');
expect(prompt).toContain('Example 2:');
expect(prompt).toContain('Hey you look great');
expect(prompt).toContain('positive responses');
expect(prompt).toContain('PERFORMANCE EXAMPLES');
expect(prompt).toContain('do not copy their opening words');
});
it('includes recent messages as anti-repetition context', async () => {
mockCreate.mockResolvedValueOnce({
choices: [{ message: { content: 'A different shape' } }],
});
await client.generateMessage(
'Analysis',
'Title',
'Author',
[],
undefined,
null,
null,
undefined,
['Damn, you look amazing. Got any more pics?'],
);
const prompt = mockCreate.mock.calls[0][0].messages[0].content as string;
expect(prompt).toContain('RECENT MESSAGES TO AVOID IMITATING');
expect(prompt).toContain('Damn, you look amazing');
expect(prompt).toContain('different opening words');
});
it('substitutes {rating} placeholder when score provided', async () => {
+35 -3
View File
@@ -196,7 +196,8 @@ describe('ProcessorService', () => {
['Example 1: Hey', 'Example 2: Nice'],
undefined,
undefined,
undefined
undefined,
[],
);
});
@@ -218,7 +219,35 @@ describe('ProcessorService', () => {
[],
'Be teasing',
undefined,
undefined
undefined,
[],
);
});
it('passes recent image messages to generation for repetition avoidance', async () => {
const recentMessages = ['Earlier opener one', 'Earlier opener two'];
const config = getTestConfig({ reddit: { targetGenders: ['female'] } });
processor = new ProcessorService(
redditApi,
aiClient,
config,
logger,
fakeFetchImage,
(sourceType) => sourceType === 'image' ? recentMessages : [],
);
await processor.processPost(mockPost);
expect(aiClient.analyzeAndGenerateMessage).toHaveBeenCalledWith(
expect.any(String),
'image/jpeg',
'Test Post',
'testuser',
[],
undefined,
undefined,
undefined,
recentMessages,
);
});
@@ -260,7 +289,8 @@ describe('ProcessorService', () => {
['Example A'],
undefined,
undefined,
undefined
undefined,
[],
);
});
@@ -316,6 +346,7 @@ describe('ProcessorService', () => {
undefined,
undefined,
undefined,
[],
);
});
@@ -347,6 +378,7 @@ describe('ProcessorService', () => {
'Be playful',
undefined,
undefined,
[],
);
});
});
+6 -5
View File
@@ -60,11 +60,11 @@ describe('Prompt Testing', () => {
expect(prompts.messageGenerationPrompt).toContain('radiant');
});
it('messageGenerationPrompt specifies GOOD HOOKS', () => {
it('messageGenerationPrompt allows structurally different hooks', () => {
const content = fs.readFileSync(configPath, 'utf-8');
const prompts = parseYamlPrompts(content);
expect(prompts.messageGenerationPrompt).toContain('GOOD HOOKS');
expect(prompts.messageGenerationPrompt).toContain('POSSIBLE HOOKS');
expect(prompts.messageGenerationPrompt).toContain('BAD HOOKS');
});
@@ -184,13 +184,14 @@ describe('Prompt Testing', () => {
expect(config.openrouter.imageAnalysisPrompt.trim().length).toBeGreaterThan(50);
});
it('messageGenerationPrompt has examples', () => {
it('messageGenerationPrompt avoids anchoring on one example shape', () => {
const content = fs.readFileSync(configPath, 'utf-8');
const yaml = require('yaml');
const config = yaml.parse(content);
expect(config.openrouter.messageGenerationPrompt).toContain('Examples of good messages');
expect(config.openrouter.messageGenerationPrompt).toContain('"Damn those tits');
expect(config.openrouter.messageGenerationPrompt).not.toContain('Examples of good messages');
expect(config.openrouter.messageGenerationPrompt).toContain('Default to opening with "Damn"');
expect(config.openrouter.messageGenerationPrompt).toContain('Follow the selected STYLE exactly');
});
it('imageAnalysisPrompt requests a rating', () => {
+15
View File
@@ -187,6 +187,21 @@ describe('StrategySelector', () => {
});
describe('under-sampled strategies', () => {
it('balances selection toward the variant with the fewest mature and pending sends', () => {
const stats: StrategyStats[] = [
{ strategyId: 'v1', totalSent: 2, pendingSent: 5, totalResponses: 0, responseRate: 0 },
{ strategyId: 'v2', totalSent: 2, pendingSent: 1, totalResponses: 0, responseRate: 0 },
];
const repo = createMockMessagesRepo(stats);
const config = getTestConfig({ strategies: { explorationRate: 0, minSamples: 20, variants: [
{ id: 'v1', weight: 1, promptOverride: 'p1' },
{ id: 'v2', weight: 1, promptOverride: 'p2' },
] } });
selector = new StrategySelector(config, repo, logger);
expect(selector.selectStrategy()?.id).toBe('v2');
});
it('selects under-sampled variant even in exploit mode', () => {
const stats: StrategyStats[] = [
{ strategyId: 'v1', totalSent: 10, totalResponses: 2, responseRate: 20 },