Show pending performance groups before maturity

This commit is contained in:
jjbrunton committed 2026-08-09 08:29:22 +01:00
1 parent 55a6774daf
commit ca59ba0f1f
5 files changed
+90 -9

No files matched your search

+24 -2
View File
@@ -733,7 +733,7 @@ export class MessagesRepository {
.map((row) => [row.strategy_id, row.total]), .map((row) => [row.strategy_id, row.total]),
); );
return rows.map(row => performanceStats( const results = rows.map(row => performanceStats(
{ {
strategyId: row.strategy_id as string, strategyId: row.strategy_id as string,
totalSent: row.total_sent as number, totalSent: row.total_sent as number,
@@ -742,6 +742,17 @@ export class MessagesRepository {
}, },
matureBefore === undefined ? undefined : pending.get(row.strategy_id as string) ?? 0, matureBefore === undefined ? undefined : pending.get(row.strategy_id as string) ?? 0,
)); ));
const matureStrategies = new Set(rows.map((row) => row.strategy_id as string));
for (const [strategyId, pendingSent] of pending) {
if (matureStrategies.has(strategyId)) continue;
results.push(performanceStats({
strategyId,
totalSent: 0,
totalResponses: 0,
totalImages: 0,
}, pendingSent));
}
return results;
} }
getSubredditStats(matureBefore?: number): SubredditStats[] { getSubredditStats(matureBefore?: number): SubredditStats[] {
@@ -770,7 +781,7 @@ export class MessagesRepository {
.map((row) => [row.subreddit, row.total]), .map((row) => [row.subreddit, row.total]),
); );
return rows.map(row => performanceStats( const results = rows.map(row => performanceStats(
{ {
subreddit: row.subreddit as string, subreddit: row.subreddit as string,
totalSent: row.total_sent as number, totalSent: row.total_sent as number,
@@ -779,6 +790,17 @@ export class MessagesRepository {
}, },
matureBefore === undefined ? undefined : pending.get(row.subreddit as string) ?? 0, matureBefore === undefined ? undefined : pending.get(row.subreddit as string) ?? 0,
)); ));
const matureSubreddits = new Set(rows.map((row) => row.subreddit as string));
for (const [subreddit, pendingSent] of pending) {
if (matureSubreddits.has(subreddit)) continue;
results.push(performanceStats({
subreddit,
totalSent: 0,
totalResponses: 0,
totalImages: 0,
}, pendingSent));
}
return results;
} }
getResponseRateTrend(days: number = 30): DailyTrend[] { getResponseRateTrend(days: number = 30): DailyTrend[] {
+8 -5
View File
@@ -85,12 +85,15 @@ export class StrategySelector {
for (const variant of this.variants) { for (const variant of this.variants) {
const s = statsMap.get(variant.id); const s = statsMap.get(variant.id);
results.push({ results.push({
...(s ?? {
strategyId: variant.id,
totalSent: 0,
totalResponses: 0,
responseRate: 0,
totalImages: 0,
imageRate: 0,
}),
strategyId: variant.id, strategyId: variant.id,
totalSent: s?.totalSent || 0,
totalResponses: s?.totalResponses || 0,
responseRate: s?.responseRate || 0,
totalImages: s?.totalImages || 0,
imageRate: s?.imageRate || 0,
configured: true, configured: true,
}); });
} }
@@ -38,12 +38,14 @@ describe('PerformancePage', () => {
HttpResponse.json([ HttpResponse.json([
{ subreddit: 'askuk', totalSent: 40, totalResponses: 12, responseRate: 30, totalImages: 6, imageRate: 15 }, { subreddit: 'askuk', totalSent: 40, totalResponses: 12, responseRate: 30, totalImages: 6, imageRate: 15 },
{ subreddit: 'tiny', totalSent: 3, totalResponses: 2, responseRate: 67, totalImages: 1, imageRate: 33 }, { subreddit: 'tiny', totalSent: 3, totalResponses: 2, responseRate: 67, totalImages: 1, imageRate: 33 },
{ subreddit: 'pending-only', totalSent: 0, totalResponses: 0, responseRate: 0, totalImages: 0, imageRate: 0, pendingSent: 9, responseRateLow: 0, responseRateHigh: 100 },
]), ]),
), ),
http.get('/api/analytics/by-strategy', () => http.get('/api/analytics/by-strategy', () =>
HttpResponse.json([ HttpResponse.json([
{ strategyId: 'friendly', totalSent: 25, totalResponses: 10, responseRate: 40, totalImages: 5, imageRate: 20, configured: true }, { strategyId: 'friendly', totalSent: 25, totalResponses: 10, responseRate: 40, totalImages: 5, imageRate: 20, configured: true },
{ strategyId: 'new-test', totalSent: 2, totalResponses: 1, responseRate: 50, totalImages: 1, imageRate: 50 }, { strategyId: 'new-test', totalSent: 2, totalResponses: 1, responseRate: 50, totalImages: 1, imageRate: 50 },
{ strategyId: 'pending-test', totalSent: 0, totalResponses: 0, responseRate: 0, totalImages: 0, imageRate: 0, pendingSent: 7, responseRateLow: 0, responseRateHigh: 100, configured: true },
]), ]),
), ),
http.get('/api/analytics/trend', ({ request }) => { http.get('/api/analytics/trend', ({ request }) => {
@@ -91,12 +93,14 @@ describe('PerformancePage', () => {
const subredditTable = screen.getByRole('table', { name: 'Subreddit performance' }); const subredditTable = screen.getByRole('table', { name: 'Subreddit performance' });
expect(within(subredditTable).getByText('n = 40')).toBeInTheDocument(); expect(within(subredditTable).getByText('n = 40')).toBeInTheDocument();
expect(within(subredditTable).getByText('n = 3')).toBeInTheDocument(); expect(within(subredditTable).getByText('n = 3')).toBeInTheDocument();
expect(within(subredditTable).getByText('Low confidence')).toBeInTheDocument(); expect(within(subredditTable).getAllByText('Low confidence')).toHaveLength(2);
expect(within(subredditTable).getByRole('columnheader', { name: 'Images' })).toBeInTheDocument(); expect(within(subredditTable).getByRole('columnheader', { name: 'Images' })).toBeInTheDocument();
expect(within(subredditTable).getByText('15%')).toBeInTheDocument(); expect(within(subredditTable).getByText('15%')).toBeInTheDocument();
expect(within(subredditTable).getByText('9 pending')).toBeInTheDocument();
const strategyTable = screen.getByRole('table', { name: 'Strategy performance' }); const strategyTable = screen.getByRole('table', { name: 'Strategy performance' });
expect(within(strategyTable).getByText('n = 25')).toBeInTheDocument(); expect(within(strategyTable).getByText('n = 25')).toBeInTheDocument();
expect(within(strategyTable).getByText('Configured')).toBeInTheDocument(); expect(within(strategyTable).getAllByText('Configured')).toHaveLength(2);
expect(within(strategyTable).getByText('7 pending')).toBeInTheDocument();
expect(screen.getByRole('list', { name: '30 day response trend' })).toHaveTextContent('50%'); expect(screen.getByRole('list', { name: '30 day response trend' })).toHaveTextContent('50%');
expect(screen.getByRole('article', { name: 'Successful example from askuk' })).toHaveTextContent('A thoughtful opener'); expect(screen.getByRole('article', { name: 'Successful example from askuk' })).toHaveTextContent('A thoughtful opener');
expect(screen.getByRole('article', { name: 'Successful example from askuk' })).toHaveTextContent('That sounds lovely!'); expect(screen.getByRole('article', { name: 'Successful example from askuk' })).toHaveTextContent('That sounds lovely!');
+36
View File
@@ -610,6 +610,42 @@ describe('ConversationStateRepository', () => {
}); });
describe('Mature performance and guardrails', () => { describe('Mature performance and guardrails', () => {
it('returns pending-only strategy and subreddit groups before any samples mature', () => {
const { postsRepo, messagesRepo } = createTestRepositories();
const now = Date.now();
postsRepo.insert({
id: 'fresh', subreddit: 'new-community', author: 'fresh-user', title: 'fresh',
imageUrl: null, processedAt: now,
});
messagesRepo.insert({
postId: 'fresh', recipient: 'fresh-user', subject: 's', message: 'm',
sentAt: now, strategyId: 'new-strategy',
});
expect(messagesRepo.getStrategyStats(now - 12 * 3600000)).toEqual([{
strategyId: 'new-strategy',
totalSent: 0,
totalResponses: 0,
responseRate: 0,
totalImages: 0,
imageRate: 0,
pendingSent: 1,
responseRateLow: 0,
responseRateHigh: 100,
}]);
expect(messagesRepo.getSubredditStats(now - 12 * 3600000)).toEqual([{
subreddit: 'new-community',
totalSent: 0,
totalResponses: 0,
responseRate: 0,
totalImages: 0,
imageRate: 0,
pendingSent: 1,
responseRateLow: 0,
responseRateHigh: 100,
}]);
});
it('excludes pending sends and reports a confidence interval', () => { it('excludes pending sends and reports a confidence interval', () => {
const { postsRepo, messagesRepo } = createTestRepositories(); const { postsRepo, messagesRepo } = createTestRepositories();
postsRepo.insert({ id: 'old', subreddit: 'test', author: 'old-user', title: 'old', imageUrl: null, processedAt: 1 }); postsRepo.insert({ id: 'old', subreddit: 'test', author: 'old-user', title: 'old', imageUrl: null, processedAt: 1 });
+16
View File
@@ -52,6 +52,22 @@ describe('StrategySelector', () => {
expect(selector.hasStrategies()).toBe(true); expect(selector.hasStrategies()).toBe(true);
}); });
it('preserves pending-only sample counts for configured strategies', () => {
const config = getTestConfig({ strategies: { variants: [
{ id: 'v1', weight: 1, promptOverride: 'test' },
] } });
const repo = createMockMessagesRepo([{
strategyId: 'v1', totalSent: 0, totalResponses: 0, responseRate: 0,
totalImages: 0, imageRate: 0, pendingSent: 7,
responseRateLow: 0, responseRateHigh: 100,
}]);
selector = new StrategySelector(config, repo, logger);
expect(selector.getStats()).toEqual([expect.objectContaining({
strategyId: 'v1', totalSent: 0, pendingSent: 7, configured: true,
})]);
});
describe('explore mode', () => { describe('explore mode', () => {
it('selects random variant when exploration rate is 1.0', () => { it('selects random variant when exploration rate is 1.0', () => {
const config = getTestConfig({ strategies: { explorationRate: 1.0, variants: [ const config = getTestConfig({ strategies: { explorationRate: 1.0, variants: [