Show pending performance groups before maturity
This commit is contained in:
1 parent
55a6774daf
commit
ca59ba0f1f
5 files changed
+90
-9
No files matched your search
@@ -733,7 +733,7 @@ export class MessagesRepository {
|
||||
.map((row) => [row.strategy_id, row.total]),
|
||||
);
|
||||
|
||||
return rows.map(row => performanceStats(
|
||||
const results = rows.map(row => performanceStats(
|
||||
{
|
||||
strategyId: row.strategy_id as string,
|
||||
totalSent: row.total_sent as number,
|
||||
@@ -742,6 +742,17 @@ export class MessagesRepository {
|
||||
},
|
||||
matureBefore === undefined ? undefined : pending.get(row.strategy_id as string) ?? 0,
|
||||
));
|
||||
const matureStrategies = new Set(rows.map((row) => row.strategy_id as string));
|
||||
for (const [strategyId, pendingSent] of pending) {
|
||||
if (matureStrategies.has(strategyId)) continue;
|
||||
results.push(performanceStats({
|
||||
strategyId,
|
||||
totalSent: 0,
|
||||
totalResponses: 0,
|
||||
totalImages: 0,
|
||||
}, pendingSent));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
getSubredditStats(matureBefore?: number): SubredditStats[] {
|
||||
@@ -770,7 +781,7 @@ export class MessagesRepository {
|
||||
.map((row) => [row.subreddit, row.total]),
|
||||
);
|
||||
|
||||
return rows.map(row => performanceStats(
|
||||
const results = rows.map(row => performanceStats(
|
||||
{
|
||||
subreddit: row.subreddit as string,
|
||||
totalSent: row.total_sent as number,
|
||||
@@ -779,6 +790,17 @@ export class MessagesRepository {
|
||||
},
|
||||
matureBefore === undefined ? undefined : pending.get(row.subreddit as string) ?? 0,
|
||||
));
|
||||
const matureSubreddits = new Set(rows.map((row) => row.subreddit as string));
|
||||
for (const [subreddit, pendingSent] of pending) {
|
||||
if (matureSubreddits.has(subreddit)) continue;
|
||||
results.push(performanceStats({
|
||||
subreddit,
|
||||
totalSent: 0,
|
||||
totalResponses: 0,
|
||||
totalImages: 0,
|
||||
}, pendingSent));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
getResponseRateTrend(days: number = 30): DailyTrend[] {
|
||||
|
||||
@@ -85,12 +85,15 @@ export class StrategySelector {
|
||||
for (const variant of this.variants) {
|
||||
const s = statsMap.get(variant.id);
|
||||
results.push({
|
||||
...(s ?? {
|
||||
strategyId: variant.id,
|
||||
totalSent: 0,
|
||||
totalResponses: 0,
|
||||
responseRate: 0,
|
||||
totalImages: 0,
|
||||
imageRate: 0,
|
||||
}),
|
||||
strategyId: variant.id,
|
||||
totalSent: s?.totalSent || 0,
|
||||
totalResponses: s?.totalResponses || 0,
|
||||
responseRate: s?.responseRate || 0,
|
||||
totalImages: s?.totalImages || 0,
|
||||
imageRate: s?.imageRate || 0,
|
||||
configured: true,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -38,12 +38,14 @@ describe('PerformancePage', () => {
|
||||
HttpResponse.json([
|
||||
{ subreddit: 'askuk', totalSent: 40, totalResponses: 12, responseRate: 30, totalImages: 6, imageRate: 15 },
|
||||
{ subreddit: 'tiny', totalSent: 3, totalResponses: 2, responseRate: 67, totalImages: 1, imageRate: 33 },
|
||||
{ subreddit: 'pending-only', totalSent: 0, totalResponses: 0, responseRate: 0, totalImages: 0, imageRate: 0, pendingSent: 9, responseRateLow: 0, responseRateHigh: 100 },
|
||||
]),
|
||||
),
|
||||
http.get('/api/analytics/by-strategy', () =>
|
||||
HttpResponse.json([
|
||||
{ strategyId: 'friendly', totalSent: 25, totalResponses: 10, responseRate: 40, totalImages: 5, imageRate: 20, configured: true },
|
||||
{ strategyId: 'new-test', totalSent: 2, totalResponses: 1, responseRate: 50, totalImages: 1, imageRate: 50 },
|
||||
{ strategyId: 'pending-test', totalSent: 0, totalResponses: 0, responseRate: 0, totalImages: 0, imageRate: 0, pendingSent: 7, responseRateLow: 0, responseRateHigh: 100, configured: true },
|
||||
]),
|
||||
),
|
||||
http.get('/api/analytics/trend', ({ request }) => {
|
||||
@@ -91,12 +93,14 @@ describe('PerformancePage', () => {
|
||||
const subredditTable = screen.getByRole('table', { name: 'Subreddit performance' });
|
||||
expect(within(subredditTable).getByText('n = 40')).toBeInTheDocument();
|
||||
expect(within(subredditTable).getByText('n = 3')).toBeInTheDocument();
|
||||
expect(within(subredditTable).getByText('Low confidence')).toBeInTheDocument();
|
||||
expect(within(subredditTable).getAllByText('Low confidence')).toHaveLength(2);
|
||||
expect(within(subredditTable).getByRole('columnheader', { name: 'Images' })).toBeInTheDocument();
|
||||
expect(within(subredditTable).getByText('15%')).toBeInTheDocument();
|
||||
expect(within(subredditTable).getByText('9 pending')).toBeInTheDocument();
|
||||
const strategyTable = screen.getByRole('table', { name: 'Strategy performance' });
|
||||
expect(within(strategyTable).getByText('n = 25')).toBeInTheDocument();
|
||||
expect(within(strategyTable).getByText('Configured')).toBeInTheDocument();
|
||||
expect(within(strategyTable).getAllByText('Configured')).toHaveLength(2);
|
||||
expect(within(strategyTable).getByText('7 pending')).toBeInTheDocument();
|
||||
expect(screen.getByRole('list', { name: '30 day response trend' })).toHaveTextContent('50%');
|
||||
expect(screen.getByRole('article', { name: 'Successful example from askuk' })).toHaveTextContent('A thoughtful opener');
|
||||
expect(screen.getByRole('article', { name: 'Successful example from askuk' })).toHaveTextContent('That sounds lovely!');
|
||||
|
||||
@@ -610,6 +610,42 @@ describe('ConversationStateRepository', () => {
|
||||
});
|
||||
|
||||
describe('Mature performance and guardrails', () => {
|
||||
it('returns pending-only strategy and subreddit groups before any samples mature', () => {
|
||||
const { postsRepo, messagesRepo } = createTestRepositories();
|
||||
const now = Date.now();
|
||||
postsRepo.insert({
|
||||
id: 'fresh', subreddit: 'new-community', author: 'fresh-user', title: 'fresh',
|
||||
imageUrl: null, processedAt: now,
|
||||
});
|
||||
messagesRepo.insert({
|
||||
postId: 'fresh', recipient: 'fresh-user', subject: 's', message: 'm',
|
||||
sentAt: now, strategyId: 'new-strategy',
|
||||
});
|
||||
|
||||
expect(messagesRepo.getStrategyStats(now - 12 * 3600000)).toEqual([{
|
||||
strategyId: 'new-strategy',
|
||||
totalSent: 0,
|
||||
totalResponses: 0,
|
||||
responseRate: 0,
|
||||
totalImages: 0,
|
||||
imageRate: 0,
|
||||
pendingSent: 1,
|
||||
responseRateLow: 0,
|
||||
responseRateHigh: 100,
|
||||
}]);
|
||||
expect(messagesRepo.getSubredditStats(now - 12 * 3600000)).toEqual([{
|
||||
subreddit: 'new-community',
|
||||
totalSent: 0,
|
||||
totalResponses: 0,
|
||||
responseRate: 0,
|
||||
totalImages: 0,
|
||||
imageRate: 0,
|
||||
pendingSent: 1,
|
||||
responseRateLow: 0,
|
||||
responseRateHigh: 100,
|
||||
}]);
|
||||
});
|
||||
|
||||
it('excludes pending sends and reports a confidence interval', () => {
|
||||
const { postsRepo, messagesRepo } = createTestRepositories();
|
||||
postsRepo.insert({ id: 'old', subreddit: 'test', author: 'old-user', title: 'old', imageUrl: null, processedAt: 1 });
|
||||
|
||||
@@ -52,6 +52,22 @@ describe('StrategySelector', () => {
|
||||
expect(selector.hasStrategies()).toBe(true);
|
||||
});
|
||||
|
||||
it('preserves pending-only sample counts for configured strategies', () => {
|
||||
const config = getTestConfig({ strategies: { variants: [
|
||||
{ id: 'v1', weight: 1, promptOverride: 'test' },
|
||||
] } });
|
||||
const repo = createMockMessagesRepo([{
|
||||
strategyId: 'v1', totalSent: 0, totalResponses: 0, responseRate: 0,
|
||||
totalImages: 0, imageRate: 0, pendingSent: 7,
|
||||
responseRateLow: 0, responseRateHigh: 100,
|
||||
}]);
|
||||
selector = new StrategySelector(config, repo, logger);
|
||||
|
||||
expect(selector.getStats()).toEqual([expect.objectContaining({
|
||||
strategyId: 'v1', totalSent: 0, pendingSent: 7, configured: true,
|
||||
})]);
|
||||
});
|
||||
|
||||
describe('explore mode', () => {
|
||||
it('selects random variant when exploration rate is 1.0', () => {
|
||||
const config = getTestConfig({ strategies: { explorationRate: 1.0, variants: [
|
||||
|
||||
Reference in new issue
Block a user