Add varied messaging experiments
This commit is contained in:
1 parent
43f459a024
commit
23d4c3a320
12 files changed
+217
-72
No files matched your search
@@ -227,7 +227,7 @@ describe('ConfigSchema', () => {
|
||||
expect(result.success).toBe(true);
|
||||
if (result.success) {
|
||||
expect(result.data.strategies.explorationRate).toBe(0.2);
|
||||
expect(result.data.strategies.minSamples).toBe(30);
|
||||
expect(result.data.strategies.minSamples).toBe(20);
|
||||
expect(result.data.strategies.responseWindowHours).toBe(12);
|
||||
expect(result.data.strategies.variants).toEqual([]);
|
||||
expect(result.data.chatStrategies.responseWindowHours).toBe(12);
|
||||
|
||||
@@ -470,7 +470,31 @@ describe('OpenRouterClient', () => {
|
||||
expect(prompt).toContain('Example 1:');
|
||||
expect(prompt).toContain('Example 2:');
|
||||
expect(prompt).toContain('Hey you look great');
|
||||
expect(prompt).toContain('positive responses');
|
||||
expect(prompt).toContain('PERFORMANCE EXAMPLES');
|
||||
expect(prompt).toContain('do not copy their opening words');
|
||||
});
|
||||
|
||||
it('includes recent messages as anti-repetition context', async () => {
|
||||
mockCreate.mockResolvedValueOnce({
|
||||
choices: [{ message: { content: 'A different shape' } }],
|
||||
});
|
||||
|
||||
await client.generateMessage(
|
||||
'Analysis',
|
||||
'Title',
|
||||
'Author',
|
||||
[],
|
||||
undefined,
|
||||
null,
|
||||
null,
|
||||
undefined,
|
||||
['Damn, you look amazing. Got any more pics?'],
|
||||
);
|
||||
|
||||
const prompt = mockCreate.mock.calls[0][0].messages[0].content as string;
|
||||
expect(prompt).toContain('RECENT MESSAGES TO AVOID IMITATING');
|
||||
expect(prompt).toContain('Damn, you look amazing');
|
||||
expect(prompt).toContain('different opening words');
|
||||
});
|
||||
|
||||
it('substitutes {rating} placeholder when score provided', async () => {
|
||||
|
||||
+35
-3
@@ -196,7 +196,8 @@ describe('ProcessorService', () => {
|
||||
['Example 1: Hey', 'Example 2: Nice'],
|
||||
undefined,
|
||||
undefined,
|
||||
undefined
|
||||
undefined,
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
@@ -218,7 +219,35 @@ describe('ProcessorService', () => {
|
||||
[],
|
||||
'Be teasing',
|
||||
undefined,
|
||||
undefined
|
||||
undefined,
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
it('passes recent image messages to generation for repetition avoidance', async () => {
|
||||
const recentMessages = ['Earlier opener one', 'Earlier opener two'];
|
||||
const config = getTestConfig({ reddit: { targetGenders: ['female'] } });
|
||||
processor = new ProcessorService(
|
||||
redditApi,
|
||||
aiClient,
|
||||
config,
|
||||
logger,
|
||||
fakeFetchImage,
|
||||
(sourceType) => sourceType === 'image' ? recentMessages : [],
|
||||
);
|
||||
|
||||
await processor.processPost(mockPost);
|
||||
|
||||
expect(aiClient.analyzeAndGenerateMessage).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
'image/jpeg',
|
||||
'Test Post',
|
||||
'testuser',
|
||||
[],
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
recentMessages,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -260,7 +289,8 @@ describe('ProcessorService', () => {
|
||||
['Example A'],
|
||||
undefined,
|
||||
undefined,
|
||||
undefined
|
||||
undefined,
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
@@ -316,6 +346,7 @@ describe('ProcessorService', () => {
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
@@ -347,6 +378,7 @@ describe('ProcessorService', () => {
|
||||
'Be playful',
|
||||
undefined,
|
||||
undefined,
|
||||
[],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -60,11 +60,11 @@ describe('Prompt Testing', () => {
|
||||
expect(prompts.messageGenerationPrompt).toContain('radiant');
|
||||
});
|
||||
|
||||
it('messageGenerationPrompt specifies GOOD HOOKS', () => {
|
||||
it('messageGenerationPrompt allows structurally different hooks', () => {
|
||||
const content = fs.readFileSync(configPath, 'utf-8');
|
||||
const prompts = parseYamlPrompts(content);
|
||||
|
||||
expect(prompts.messageGenerationPrompt).toContain('GOOD HOOKS');
|
||||
expect(prompts.messageGenerationPrompt).toContain('POSSIBLE HOOKS');
|
||||
expect(prompts.messageGenerationPrompt).toContain('BAD HOOKS');
|
||||
});
|
||||
|
||||
@@ -184,13 +184,14 @@ describe('Prompt Testing', () => {
|
||||
expect(config.openrouter.imageAnalysisPrompt.trim().length).toBeGreaterThan(50);
|
||||
});
|
||||
|
||||
it('messageGenerationPrompt has examples', () => {
|
||||
it('messageGenerationPrompt avoids anchoring on one example shape', () => {
|
||||
const content = fs.readFileSync(configPath, 'utf-8');
|
||||
const yaml = require('yaml');
|
||||
const config = yaml.parse(content);
|
||||
|
||||
expect(config.openrouter.messageGenerationPrompt).toContain('Examples of good messages');
|
||||
expect(config.openrouter.messageGenerationPrompt).toContain('"Damn those tits');
|
||||
expect(config.openrouter.messageGenerationPrompt).not.toContain('Examples of good messages');
|
||||
expect(config.openrouter.messageGenerationPrompt).toContain('Default to opening with "Damn"');
|
||||
expect(config.openrouter.messageGenerationPrompt).toContain('Follow the selected STYLE exactly');
|
||||
});
|
||||
|
||||
it('imageAnalysisPrompt requests a rating', () => {
|
||||
|
||||
@@ -187,6 +187,21 @@ describe('StrategySelector', () => {
|
||||
});
|
||||
|
||||
describe('under-sampled strategies', () => {
|
||||
it('balances selection toward the variant with the fewest mature and pending sends', () => {
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'v1', totalSent: 2, pendingSent: 5, totalResponses: 0, responseRate: 0 },
|
||||
{ strategyId: 'v2', totalSent: 2, pendingSent: 1, totalResponses: 0, responseRate: 0 },
|
||||
];
|
||||
const repo = createMockMessagesRepo(stats);
|
||||
const config = getTestConfig({ strategies: { explorationRate: 0, minSamples: 20, variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'p1' },
|
||||
{ id: 'v2', weight: 1, promptOverride: 'p2' },
|
||||
] } });
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
expect(selector.selectStrategy()?.id).toBe('v2');
|
||||
});
|
||||
|
||||
it('selects under-sampled variant even in exploit mode', () => {
|
||||
const stats: StrategyStats[] = [
|
||||
{ strategyId: 'v1', totalSent: 10, totalResponses: 2, responseRate: 20 },
|
||||
|
||||
Reference in new issue
Block a user