Show pending performance groups before maturity
This commit is contained in:
1 parent
55a6774daf
commit
ca59ba0f1f
5 files changed
+90
-9
No files matched your search
@@ -733,7 +733,7 @@ export class MessagesRepository {
|
|||||||
.map((row) => [row.strategy_id, row.total]),
|
.map((row) => [row.strategy_id, row.total]),
|
||||||
);
|
);
|
||||||
|
|
||||||
return rows.map(row => performanceStats(
|
const results = rows.map(row => performanceStats(
|
||||||
{
|
{
|
||||||
strategyId: row.strategy_id as string,
|
strategyId: row.strategy_id as string,
|
||||||
totalSent: row.total_sent as number,
|
totalSent: row.total_sent as number,
|
||||||
@@ -742,6 +742,17 @@ export class MessagesRepository {
|
|||||||
},
|
},
|
||||||
matureBefore === undefined ? undefined : pending.get(row.strategy_id as string) ?? 0,
|
matureBefore === undefined ? undefined : pending.get(row.strategy_id as string) ?? 0,
|
||||||
));
|
));
|
||||||
|
const matureStrategies = new Set(rows.map((row) => row.strategy_id as string));
|
||||||
|
for (const [strategyId, pendingSent] of pending) {
|
||||||
|
if (matureStrategies.has(strategyId)) continue;
|
||||||
|
results.push(performanceStats({
|
||||||
|
strategyId,
|
||||||
|
totalSent: 0,
|
||||||
|
totalResponses: 0,
|
||||||
|
totalImages: 0,
|
||||||
|
}, pendingSent));
|
||||||
|
}
|
||||||
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
getSubredditStats(matureBefore?: number): SubredditStats[] {
|
getSubredditStats(matureBefore?: number): SubredditStats[] {
|
||||||
@@ -770,7 +781,7 @@ export class MessagesRepository {
|
|||||||
.map((row) => [row.subreddit, row.total]),
|
.map((row) => [row.subreddit, row.total]),
|
||||||
);
|
);
|
||||||
|
|
||||||
return rows.map(row => performanceStats(
|
const results = rows.map(row => performanceStats(
|
||||||
{
|
{
|
||||||
subreddit: row.subreddit as string,
|
subreddit: row.subreddit as string,
|
||||||
totalSent: row.total_sent as number,
|
totalSent: row.total_sent as number,
|
||||||
@@ -779,6 +790,17 @@ export class MessagesRepository {
|
|||||||
},
|
},
|
||||||
matureBefore === undefined ? undefined : pending.get(row.subreddit as string) ?? 0,
|
matureBefore === undefined ? undefined : pending.get(row.subreddit as string) ?? 0,
|
||||||
));
|
));
|
||||||
|
const matureSubreddits = new Set(rows.map((row) => row.subreddit as string));
|
||||||
|
for (const [subreddit, pendingSent] of pending) {
|
||||||
|
if (matureSubreddits.has(subreddit)) continue;
|
||||||
|
results.push(performanceStats({
|
||||||
|
subreddit,
|
||||||
|
totalSent: 0,
|
||||||
|
totalResponses: 0,
|
||||||
|
totalImages: 0,
|
||||||
|
}, pendingSent));
|
||||||
|
}
|
||||||
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
getResponseRateTrend(days: number = 30): DailyTrend[] {
|
getResponseRateTrend(days: number = 30): DailyTrend[] {
|
||||||
|
|||||||
@@ -85,12 +85,15 @@ export class StrategySelector {
|
|||||||
for (const variant of this.variants) {
|
for (const variant of this.variants) {
|
||||||
const s = statsMap.get(variant.id);
|
const s = statsMap.get(variant.id);
|
||||||
results.push({
|
results.push({
|
||||||
|
...(s ?? {
|
||||||
|
strategyId: variant.id,
|
||||||
|
totalSent: 0,
|
||||||
|
totalResponses: 0,
|
||||||
|
responseRate: 0,
|
||||||
|
totalImages: 0,
|
||||||
|
imageRate: 0,
|
||||||
|
}),
|
||||||
strategyId: variant.id,
|
strategyId: variant.id,
|
||||||
totalSent: s?.totalSent || 0,
|
|
||||||
totalResponses: s?.totalResponses || 0,
|
|
||||||
responseRate: s?.responseRate || 0,
|
|
||||||
totalImages: s?.totalImages || 0,
|
|
||||||
imageRate: s?.imageRate || 0,
|
|
||||||
configured: true,
|
configured: true,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -38,12 +38,14 @@ describe('PerformancePage', () => {
|
|||||||
HttpResponse.json([
|
HttpResponse.json([
|
||||||
{ subreddit: 'askuk', totalSent: 40, totalResponses: 12, responseRate: 30, totalImages: 6, imageRate: 15 },
|
{ subreddit: 'askuk', totalSent: 40, totalResponses: 12, responseRate: 30, totalImages: 6, imageRate: 15 },
|
||||||
{ subreddit: 'tiny', totalSent: 3, totalResponses: 2, responseRate: 67, totalImages: 1, imageRate: 33 },
|
{ subreddit: 'tiny', totalSent: 3, totalResponses: 2, responseRate: 67, totalImages: 1, imageRate: 33 },
|
||||||
|
{ subreddit: 'pending-only', totalSent: 0, totalResponses: 0, responseRate: 0, totalImages: 0, imageRate: 0, pendingSent: 9, responseRateLow: 0, responseRateHigh: 100 },
|
||||||
]),
|
]),
|
||||||
),
|
),
|
||||||
http.get('/api/analytics/by-strategy', () =>
|
http.get('/api/analytics/by-strategy', () =>
|
||||||
HttpResponse.json([
|
HttpResponse.json([
|
||||||
{ strategyId: 'friendly', totalSent: 25, totalResponses: 10, responseRate: 40, totalImages: 5, imageRate: 20, configured: true },
|
{ strategyId: 'friendly', totalSent: 25, totalResponses: 10, responseRate: 40, totalImages: 5, imageRate: 20, configured: true },
|
||||||
{ strategyId: 'new-test', totalSent: 2, totalResponses: 1, responseRate: 50, totalImages: 1, imageRate: 50 },
|
{ strategyId: 'new-test', totalSent: 2, totalResponses: 1, responseRate: 50, totalImages: 1, imageRate: 50 },
|
||||||
|
{ strategyId: 'pending-test', totalSent: 0, totalResponses: 0, responseRate: 0, totalImages: 0, imageRate: 0, pendingSent: 7, responseRateLow: 0, responseRateHigh: 100, configured: true },
|
||||||
]),
|
]),
|
||||||
),
|
),
|
||||||
http.get('/api/analytics/trend', ({ request }) => {
|
http.get('/api/analytics/trend', ({ request }) => {
|
||||||
@@ -91,12 +93,14 @@ describe('PerformancePage', () => {
|
|||||||
const subredditTable = screen.getByRole('table', { name: 'Subreddit performance' });
|
const subredditTable = screen.getByRole('table', { name: 'Subreddit performance' });
|
||||||
expect(within(subredditTable).getByText('n = 40')).toBeInTheDocument();
|
expect(within(subredditTable).getByText('n = 40')).toBeInTheDocument();
|
||||||
expect(within(subredditTable).getByText('n = 3')).toBeInTheDocument();
|
expect(within(subredditTable).getByText('n = 3')).toBeInTheDocument();
|
||||||
expect(within(subredditTable).getByText('Low confidence')).toBeInTheDocument();
|
expect(within(subredditTable).getAllByText('Low confidence')).toHaveLength(2);
|
||||||
expect(within(subredditTable).getByRole('columnheader', { name: 'Images' })).toBeInTheDocument();
|
expect(within(subredditTable).getByRole('columnheader', { name: 'Images' })).toBeInTheDocument();
|
||||||
expect(within(subredditTable).getByText('15%')).toBeInTheDocument();
|
expect(within(subredditTable).getByText('15%')).toBeInTheDocument();
|
||||||
|
expect(within(subredditTable).getByText('9 pending')).toBeInTheDocument();
|
||||||
const strategyTable = screen.getByRole('table', { name: 'Strategy performance' });
|
const strategyTable = screen.getByRole('table', { name: 'Strategy performance' });
|
||||||
expect(within(strategyTable).getByText('n = 25')).toBeInTheDocument();
|
expect(within(strategyTable).getByText('n = 25')).toBeInTheDocument();
|
||||||
expect(within(strategyTable).getByText('Configured')).toBeInTheDocument();
|
expect(within(strategyTable).getAllByText('Configured')).toHaveLength(2);
|
||||||
|
expect(within(strategyTable).getByText('7 pending')).toBeInTheDocument();
|
||||||
expect(screen.getByRole('list', { name: '30 day response trend' })).toHaveTextContent('50%');
|
expect(screen.getByRole('list', { name: '30 day response trend' })).toHaveTextContent('50%');
|
||||||
expect(screen.getByRole('article', { name: 'Successful example from askuk' })).toHaveTextContent('A thoughtful opener');
|
expect(screen.getByRole('article', { name: 'Successful example from askuk' })).toHaveTextContent('A thoughtful opener');
|
||||||
expect(screen.getByRole('article', { name: 'Successful example from askuk' })).toHaveTextContent('That sounds lovely!');
|
expect(screen.getByRole('article', { name: 'Successful example from askuk' })).toHaveTextContent('That sounds lovely!');
|
||||||
|
|||||||
@@ -610,6 +610,42 @@ describe('ConversationStateRepository', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe('Mature performance and guardrails', () => {
|
describe('Mature performance and guardrails', () => {
|
||||||
|
it('returns pending-only strategy and subreddit groups before any samples mature', () => {
|
||||||
|
const { postsRepo, messagesRepo } = createTestRepositories();
|
||||||
|
const now = Date.now();
|
||||||
|
postsRepo.insert({
|
||||||
|
id: 'fresh', subreddit: 'new-community', author: 'fresh-user', title: 'fresh',
|
||||||
|
imageUrl: null, processedAt: now,
|
||||||
|
});
|
||||||
|
messagesRepo.insert({
|
||||||
|
postId: 'fresh', recipient: 'fresh-user', subject: 's', message: 'm',
|
||||||
|
sentAt: now, strategyId: 'new-strategy',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(messagesRepo.getStrategyStats(now - 12 * 3600000)).toEqual([{
|
||||||
|
strategyId: 'new-strategy',
|
||||||
|
totalSent: 0,
|
||||||
|
totalResponses: 0,
|
||||||
|
responseRate: 0,
|
||||||
|
totalImages: 0,
|
||||||
|
imageRate: 0,
|
||||||
|
pendingSent: 1,
|
||||||
|
responseRateLow: 0,
|
||||||
|
responseRateHigh: 100,
|
||||||
|
}]);
|
||||||
|
expect(messagesRepo.getSubredditStats(now - 12 * 3600000)).toEqual([{
|
||||||
|
subreddit: 'new-community',
|
||||||
|
totalSent: 0,
|
||||||
|
totalResponses: 0,
|
||||||
|
responseRate: 0,
|
||||||
|
totalImages: 0,
|
||||||
|
imageRate: 0,
|
||||||
|
pendingSent: 1,
|
||||||
|
responseRateLow: 0,
|
||||||
|
responseRateHigh: 100,
|
||||||
|
}]);
|
||||||
|
});
|
||||||
|
|
||||||
it('excludes pending sends and reports a confidence interval', () => {
|
it('excludes pending sends and reports a confidence interval', () => {
|
||||||
const { postsRepo, messagesRepo } = createTestRepositories();
|
const { postsRepo, messagesRepo } = createTestRepositories();
|
||||||
postsRepo.insert({ id: 'old', subreddit: 'test', author: 'old-user', title: 'old', imageUrl: null, processedAt: 1 });
|
postsRepo.insert({ id: 'old', subreddit: 'test', author: 'old-user', title: 'old', imageUrl: null, processedAt: 1 });
|
||||||
|
|||||||
@@ -52,6 +52,22 @@ describe('StrategySelector', () => {
|
|||||||
expect(selector.hasStrategies()).toBe(true);
|
expect(selector.hasStrategies()).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('preserves pending-only sample counts for configured strategies', () => {
|
||||||
|
const config = getTestConfig({ strategies: { variants: [
|
||||||
|
{ id: 'v1', weight: 1, promptOverride: 'test' },
|
||||||
|
] } });
|
||||||
|
const repo = createMockMessagesRepo([{
|
||||||
|
strategyId: 'v1', totalSent: 0, totalResponses: 0, responseRate: 0,
|
||||||
|
totalImages: 0, imageRate: 0, pendingSent: 7,
|
||||||
|
responseRateLow: 0, responseRateHigh: 100,
|
||||||
|
}]);
|
||||||
|
selector = new StrategySelector(config, repo, logger);
|
||||||
|
|
||||||
|
expect(selector.getStats()).toEqual([expect.objectContaining({
|
||||||
|
strategyId: 'v1', totalSent: 0, pendingSent: 7, configured: true,
|
||||||
|
})]);
|
||||||
|
});
|
||||||
|
|
||||||
describe('explore mode', () => {
|
describe('explore mode', () => {
|
||||||
it('selects random variant when exploration rate is 1.0', () => {
|
it('selects random variant when exploration rate is 1.0', () => {
|
||||||
const config = getTestConfig({ strategies: { explorationRate: 1.0, variants: [
|
const config = getTestConfig({ strategies: { explorationRate: 1.0, variants: [
|
||||||
|
|||||||
Reference in new issue
Block a user