Add attention inbox and AI usage observability
This commit is contained in:
1 parent
ca59ba0f1f
commit
83b5f1f5b6
30 files changed
+702
-28
No files matched your search
@@ -6,6 +6,7 @@ export interface NavigationItem {
|
|||||||
|
|
||||||
export const navigationItems: readonly NavigationItem[] = [
|
export const navigationItems: readonly NavigationItem[] = [
|
||||||
{ label: 'Overview', path: '/', description: 'System health and current activity' },
|
{ label: 'Overview', path: '/', description: 'System health and current activity' },
|
||||||
|
{ label: 'Needs attention', path: '/attention', description: 'Handoffs, failures, and account risks' },
|
||||||
{ label: 'Conversations', path: '/conversations', description: 'Inbox and message history' },
|
{ label: 'Conversations', path: '/conversations', description: 'Inbox and message history' },
|
||||||
{ label: 'Media', path: '/media', description: 'Private media and sync controls' },
|
{ label: 'Media', path: '/media', description: 'Private media and sync controls' },
|
||||||
{ label: 'Performance', path: '/performance', description: 'Response and strategy trends' },
|
{ label: 'Performance', path: '/performance', description: 'Response and strategy trends' },
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import { OperationsPage } from '../features/operations/operations-page.js';
|
|||||||
import { PerformancePage } from '../features/performance/performance-page.js';
|
import { PerformancePage } from '../features/performance/performance-page.js';
|
||||||
import { SettingsRoute } from '../features/settings/settings-route.js';
|
import { SettingsRoute } from '../features/settings/settings-route.js';
|
||||||
import { RouteErrorPage } from './route-error-page.js';
|
import { RouteErrorPage } from './route-error-page.js';
|
||||||
|
import { AttentionPage } from '../features/attention/attention-page.js';
|
||||||
|
|
||||||
export function createDashboardRoutes(): RouteObject[] {
|
export function createDashboardRoutes(): RouteObject[] {
|
||||||
return [
|
return [
|
||||||
@@ -16,6 +17,7 @@ export function createDashboardRoutes(): RouteObject[] {
|
|||||||
errorElement: <RouteErrorPage />,
|
errorElement: <RouteErrorPage />,
|
||||||
children: [
|
children: [
|
||||||
{ index: true, element: <OverviewPage /> },
|
{ index: true, element: <OverviewPage /> },
|
||||||
|
{ path: 'attention', element: <AttentionPage /> },
|
||||||
{ path: 'conversations', element: <ConversationRoute /> },
|
{ path: 'conversations', element: <ConversationRoute /> },
|
||||||
{ path: 'conversations/:conversationId', element: <ConversationRoute /> },
|
{ path: 'conversations/:conversationId', element: <ConversationRoute /> },
|
||||||
{ path: 'media', element: <MediaRoute /> },
|
{ path: 'media', element: <MediaRoute /> },
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { AttentionResponseSchema } from '../../../../../packages/api-contracts/src/index.js';
|
||||||
|
import { createDashboardRequest } from '../../api/request.js';
|
||||||
|
|
||||||
|
export function createAttentionApi(fetchImplementation?: typeof fetch) {
|
||||||
|
const request = createDashboardRequest(fetchImplementation);
|
||||||
|
return {
|
||||||
|
async getAttention() {
|
||||||
|
const response = await request('/api/attention');
|
||||||
|
const payload: unknown = await response.json();
|
||||||
|
if (!response.ok) throw new Error(`Attention inbox unavailable (${response.status})`);
|
||||||
|
return AttentionResponseSchema.parse(payload);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export const attentionApi = createAttentionApi();
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import { useQuery } from '@tanstack/react-query';
|
||||||
|
import { Link } from 'react-router-dom';
|
||||||
|
import type { AttentionItem } from '../../../../../packages/api-contracts/src/index.js';
|
||||||
|
import { AccountOwnerBadge } from '../../components/account-owner-badge.js';
|
||||||
|
import { attentionApi } from './attention-api.js';
|
||||||
|
import './attention.css';
|
||||||
|
|
||||||
|
function formatTime(timestamp: number): string {
|
||||||
|
return new Intl.DateTimeFormat(undefined, { dateStyle: 'medium', timeStyle: 'short' }).format(timestamp);
|
||||||
|
}
|
||||||
|
|
||||||
|
function itemAction(item: AttentionItem) {
|
||||||
|
if (!item.conversationId) return null;
|
||||||
|
return <Link className="button secondary" to={`/conversations/${encodeURIComponent(item.conversationId)}`}>Open conversation</Link>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AttentionPage() {
|
||||||
|
const query = useQuery({ queryKey: ['attention'], queryFn: attentionApi.getAttention, refetchInterval: 30_000 });
|
||||||
|
if (query.isPending) return <div className="page-state" role="status">Loading attention inbox…</div>;
|
||||||
|
if (query.isError) return <div className="page-state error-state" role="alert"><strong>Attention inbox unavailable</strong><span>{query.error.message}</span><button className="button secondary" onClick={() => void query.refetch()} type="button">Retry</button></div>;
|
||||||
|
const critical = query.data.items.filter((item) => item.severity === 'critical').length;
|
||||||
|
return (
|
||||||
|
<div className="page attention-page">
|
||||||
|
<header className="page-header"><div><span className="eyebrow">Operator queue</span><h1>Needs attention</h1><p>Boundaries, human handoffs, delivery failures, account problems, and Reddit cooldowns in one actionable inbox.</p></div><button className="button secondary" onClick={() => void query.refetch()} type="button">Refresh</button></header>
|
||||||
|
<section aria-label="Attention totals" className="status-strip"><div><span className={`status-dot ${query.data.items.length === 0 ? 'running' : 'stopped'}`} aria-hidden="true" /><div><small>Open items</small><strong>{query.data.items.length}</strong></div></div><span className={critical > 0 ? 'pill warning' : 'pill success'}>{critical} critical</span><span className="status-note">Updated {formatTime(query.data.generatedAt)}</span></section>
|
||||||
|
{query.data.items.length === 0 ? <div className="empty-state panel"><strong>Nothing needs attention.</strong><span>Automation and account health have no outstanding operator items.</span></div> : (
|
||||||
|
<ol aria-label="Attention items" className="attention-list">{query.data.items.map((item) => (
|
||||||
|
<li className={`attention-item is-${item.severity}`} key={item.id}><div className="attention-item-heading"><div><span className={`pill ${item.severity === 'critical' ? 'warning' : 'neutral'}`}>{item.kind.replaceAll('-', ' ')}</span><h2>{item.title}</h2></div><time dateTime={new Date(item.occurredAt).toISOString()}>{formatTime(item.occurredAt)}</time></div><p>{item.detail}</p>{item.owner ? <AccountOwnerBadge owner={item.owner} /> : null}{itemAction(item)}</li>
|
||||||
|
))}</ol>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
.attention-list { display: grid; gap: .8rem; padding: 0; margin: 0; list-style: none; }
|
||||||
|
.attention-item { display: grid; gap: .8rem; padding: 1rem; background: var(--surface); border: 1px solid var(--border); border-left: .3rem solid var(--warning); border-radius: .9rem; }
|
||||||
|
.attention-item.is-critical { border-left-color: var(--danger); }
|
||||||
|
.attention-item-heading { display: flex; align-items: flex-start; justify-content: space-between; gap: 1rem; }
|
||||||
|
.attention-item-heading > div { display: grid; gap: .55rem; }
|
||||||
|
.attention-item-heading time { color: var(--muted); font-size: .8rem; text-align: right; }
|
||||||
|
.attention-item p { margin: 0; color: var(--muted); }
|
||||||
|
.attention-item .button { justify-self: start; text-decoration: none; }
|
||||||
@@ -4,6 +4,7 @@ import {
|
|||||||
SubredditPerformanceResponseSchema,
|
SubredditPerformanceResponseSchema,
|
||||||
SuccessfulExamplesResponseSchema,
|
SuccessfulExamplesResponseSchema,
|
||||||
FunnelResponseSchema,
|
FunnelResponseSchema,
|
||||||
|
AiUsageSummaryResponseSchema,
|
||||||
} from '../../../../../packages/api-contracts/src/index.js';
|
} from '../../../../../packages/api-contracts/src/index.js';
|
||||||
import type { z } from 'zod';
|
import type { z } from 'zod';
|
||||||
import { createDashboardRequest } from '../../api/request.js';
|
import { createDashboardRequest } from '../../api/request.js';
|
||||||
@@ -13,6 +14,7 @@ type StrategyPerformanceResponse = z.infer<typeof StrategyPerformanceResponseSch
|
|||||||
type DailyTrendResponse = z.infer<typeof DailyTrendResponseSchema>;
|
type DailyTrendResponse = z.infer<typeof DailyTrendResponseSchema>;
|
||||||
type SuccessfulExamplesResponse = z.infer<typeof SuccessfulExamplesResponseSchema>;
|
type SuccessfulExamplesResponse = z.infer<typeof SuccessfulExamplesResponseSchema>;
|
||||||
type FunnelResponse = z.infer<typeof FunnelResponseSchema>;
|
type FunnelResponse = z.infer<typeof FunnelResponseSchema>;
|
||||||
|
type AiUsageSummaryResponse = z.infer<typeof AiUsageSummaryResponseSchema>;
|
||||||
|
|
||||||
async function parseResponse<T>(
|
async function parseResponse<T>(
|
||||||
response: Response,
|
response: Response,
|
||||||
@@ -38,6 +40,7 @@ export interface PerformanceApi {
|
|||||||
getDailyTrend(days: number): Promise<DailyTrendResponse>;
|
getDailyTrend(days: number): Promise<DailyTrendResponse>;
|
||||||
getSuccessfulExamples(): Promise<SuccessfulExamplesResponse>;
|
getSuccessfulExamples(): Promise<SuccessfulExamplesResponse>;
|
||||||
getFunnel(): Promise<FunnelResponse>;
|
getFunnel(): Promise<FunnelResponse>;
|
||||||
|
getAiUsage(days: number): Promise<AiUsageSummaryResponse>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createPerformanceApi(
|
export function createPerformanceApi(
|
||||||
@@ -75,6 +78,12 @@ export function createPerformanceApi(
|
|||||||
FunnelResponseSchema,
|
FunnelResponseSchema,
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
|
async getAiUsage(days: number) {
|
||||||
|
return parseResponse(
|
||||||
|
await request(`/api/analytics/ai-usage?days=${days}`),
|
||||||
|
AiUsageSummaryResponseSchema,
|
||||||
|
);
|
||||||
|
},
|
||||||
} satisfies PerformanceApi;
|
} satisfies PerformanceApi;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import type {
|
|||||||
SubredditPerformanceResponseSchema,
|
SubredditPerformanceResponseSchema,
|
||||||
SuccessfulExample,
|
SuccessfulExample,
|
||||||
FunnelResponseSchema,
|
FunnelResponseSchema,
|
||||||
|
AiUsageSummaryResponseSchema,
|
||||||
} from '../../../../../packages/api-contracts/src/index.js';
|
} from '../../../../../packages/api-contracts/src/index.js';
|
||||||
import {
|
import {
|
||||||
useDailyTrendQuery,
|
useDailyTrendQuery,
|
||||||
@@ -12,6 +13,7 @@ import {
|
|||||||
useSubredditPerformanceQuery,
|
useSubredditPerformanceQuery,
|
||||||
useSuccessfulExamplesQuery,
|
useSuccessfulExamplesQuery,
|
||||||
useFunnelQuery,
|
useFunnelQuery,
|
||||||
|
useAiUsageQuery,
|
||||||
} from './performance-queries.js';
|
} from './performance-queries.js';
|
||||||
import './performance.css';
|
import './performance.css';
|
||||||
|
|
||||||
@@ -20,6 +22,33 @@ type SubredditPerformance = z.infer<typeof SubredditPerformanceResponseSchema>[n
|
|||||||
type StrategyPerformance = z.infer<typeof StrategyPerformanceResponseSchema>[number];
|
type StrategyPerformance = z.infer<typeof StrategyPerformanceResponseSchema>[number];
|
||||||
type DailyTrend = z.infer<typeof DailyTrendResponseSchema>[number];
|
type DailyTrend = z.infer<typeof DailyTrendResponseSchema>[number];
|
||||||
type Funnel = z.infer<typeof FunnelResponseSchema>;
|
type Funnel = z.infer<typeof FunnelResponseSchema>;
|
||||||
|
type AiUsageSummary = z.infer<typeof AiUsageSummaryResponseSchema>;
|
||||||
|
|
||||||
|
function formatUsd(value: number): string {
|
||||||
|
return new Intl.NumberFormat(undefined, {
|
||||||
|
style: 'currency', currency: 'USD', minimumFractionDigits: 2, maximumFractionDigits: 4,
|
||||||
|
}).format(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function AiUsagePanel({ usage }: { usage: AiUsageSummary }) {
|
||||||
|
const successRate = usage.requests > 0 ? Math.round((usage.successes / usage.requests) * 100) : 0;
|
||||||
|
return (
|
||||||
|
<section aria-labelledby="ai-usage-heading" className="panel feature-panel">
|
||||||
|
<div className="panel-heading"><div><span className="eyebrow">OpenRouter · last {usage.days} days</span><h2 id="ai-usage-heading">AI cost and reliability</h2></div><span className="sample-size">{usage.requests} requests</span></div>
|
||||||
|
<div aria-label="AI usage totals" className="metrics-grid performance-metrics">
|
||||||
|
<article className="metric-card"><span>Reported cost</span><strong>{formatUsd(usage.costUsd)}</strong><small>OpenRouter usage accounting</small></article>
|
||||||
|
<article className="metric-card"><span>Tokens</span><strong>{usage.totalTokens.toLocaleString()}</strong><small>{usage.promptTokens.toLocaleString()} prompt · {usage.completionTokens.toLocaleString()} completion</small></article>
|
||||||
|
<article className="metric-card"><span>Success rate</span><strong>{successRate}%</strong><small>{usage.refusals} refusals · {usage.errors} errors/empty</small></article>
|
||||||
|
<article className="metric-card"><span>Average latency</span><strong>{usage.averageLatencyMs.toLocaleString()} ms</strong><small>All recorded attempts</small></article>
|
||||||
|
</div>
|
||||||
|
{usage.models.length === 0 ? <p className="feature-empty">No AI usage has been recorded yet. New requests will appear here.</p> : (
|
||||||
|
<div className="table-scroll"><table aria-label="AI usage by model" className="performance-table"><thead><tr><th scope="col">Model</th><th scope="col">Requests</th><th scope="col">Success</th><th scope="col">Refusals</th><th scope="col">Errors</th><th scope="col">Tokens</th><th scope="col">Cost</th><th scope="col">Latency</th></tr></thead><tbody>{usage.models.map((model) => (
|
||||||
|
<tr key={model.model}><th scope="row">{model.model}</th><td>{model.requests}</td><td>{model.successes}</td><td>{model.refusals}</td><td>{model.errors}</td><td>{model.totalTokens.toLocaleString()}</td><td>{formatUsd(model.costUsd)}</td><td>{model.averageLatencyMs.toLocaleString()} ms</td></tr>
|
||||||
|
))}</tbody></table></div>
|
||||||
|
)}
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function SampleSize({ totalSent }: { totalSent: number }) {
|
function SampleSize({ totalSent }: { totalSent: number }) {
|
||||||
const isLowConfidence = totalSent < LOW_CONFIDENCE_SAMPLE_SIZE;
|
const isLowConfidence = totalSent < LOW_CONFIDENCE_SAMPLE_SIZE;
|
||||||
@@ -129,7 +158,8 @@ export function PerformancePage() {
|
|||||||
const trend = useDailyTrendQuery(30);
|
const trend = useDailyTrendQuery(30);
|
||||||
const examples = useSuccessfulExamplesQuery();
|
const examples = useSuccessfulExamplesQuery();
|
||||||
const funnel = useFunnelQuery();
|
const funnel = useFunnelQuery();
|
||||||
const queries = [subreddits, strategies, trend, examples, funnel];
|
const aiUsage = useAiUsageQuery(30);
|
||||||
|
const queries = [subreddits, strategies, trend, examples, funnel, aiUsage];
|
||||||
if (queries.some((query) => query.isPending)) {
|
if (queries.some((query) => query.isPending)) {
|
||||||
return <div className="page-state" role="status">Loading performance…</div>;
|
return <div className="page-state" role="status">Loading performance…</div>;
|
||||||
}
|
}
|
||||||
@@ -142,6 +172,7 @@ export function PerformancePage() {
|
|||||||
<div className="page performance-page">
|
<div className="page performance-page">
|
||||||
<header className="page-header"><div><span className="eyebrow">Decision support</span><h1>Performance</h1><p>Compare response and received-image outcomes without hiding the sample sizes behind the percentages.</p></div></header>
|
<header className="page-header"><div><span className="eyebrow">Decision support</span><h1>Performance</h1><p>Compare response and received-image outcomes without hiding the sample sizes behind the percentages.</p></div></header>
|
||||||
{funnel.data ? <FunnelPanel funnel={funnel.data} /> : null}
|
{funnel.data ? <FunnelPanel funnel={funnel.data} /> : null}
|
||||||
|
{aiUsage.data ? <AiUsagePanel usage={aiUsage.data} /> : null}
|
||||||
<section aria-labelledby="trend-heading" className="panel feature-panel"><div className="panel-heading"><div><span className="eyebrow">Last 30 days</span><h2 id="trend-heading">Response trend</h2></div></div><Trend rows={trend.data ?? []} /></section>
|
<section aria-labelledby="trend-heading" className="panel feature-panel"><div className="panel-heading"><div><span className="eyebrow">Last 30 days</span><h2 id="trend-heading">Response trend</h2></div></div><Trend rows={trend.data ?? []} /></section>
|
||||||
<div className="performance-grid">
|
<div className="performance-grid">
|
||||||
<section aria-labelledby="subreddit-heading" className="panel feature-panel"><div className="panel-heading"><h2 id="subreddit-heading">By subreddit</h2></div><PerformanceTable label="Subreddit performance" rows={subreddits.data ?? []} rowLabel={(row) => 'subreddit' in row ? row.subreddit : row.strategyId} /></section>
|
<section aria-labelledby="subreddit-heading" className="panel feature-panel"><div className="panel-heading"><h2 id="subreddit-heading">By subreddit</h2></div><PerformanceTable label="Subreddit performance" rows={subreddits.data ?? []} rowLabel={(row) => 'subreddit' in row ? row.subreddit : row.strategyId} /></section>
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ export const performanceQueryKeys = {
|
|||||||
trend: (days: number) => ['performance', 'trend', days] as const,
|
trend: (days: number) => ['performance', 'trend', days] as const,
|
||||||
successfulExamples: ['performance', 'successful-examples'] as const,
|
successfulExamples: ['performance', 'successful-examples'] as const,
|
||||||
funnel: ['performance', 'funnel'] as const,
|
funnel: ['performance', 'funnel'] as const,
|
||||||
|
aiUsage: (days: number) => ['performance', 'ai-usage', days] as const,
|
||||||
};
|
};
|
||||||
|
|
||||||
export function useSubredditPerformanceQuery() {
|
export function useSubredditPerformanceQuery() {
|
||||||
@@ -43,3 +44,10 @@ export function useFunnelQuery() {
|
|||||||
queryFn: performanceApi.getFunnel,
|
queryFn: performanceApi.getFunnel,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function useAiUsageQuery(days: number) {
|
||||||
|
return useQuery({
|
||||||
|
queryKey: performanceQueryKeys.aiUsage(days),
|
||||||
|
queryFn: () => performanceApi.getAiUsage(days),
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -86,6 +86,8 @@ an administrator credential.
|
|||||||
| `GET` | `/api/analytics/by-strategy` | Get performance by strategy | read | no |
|
| `GET` | `/api/analytics/by-strategy` | Get performance by strategy | read | no |
|
||||||
| `GET` | `/api/analytics/funnel` | Get mature message outcome funnel | read | no |
|
| `GET` | `/api/analytics/funnel` | Get mature message outcome funnel | read | no |
|
||||||
| `GET` | `/api/analytics/trend` | Get daily performance trend | read | no |
|
| `GET` | `/api/analytics/trend` | Get daily performance trend | read | no |
|
||||||
|
| `GET` | `/api/analytics/ai-usage` | Get OpenRouter cost and model reliability | read | no |
|
||||||
|
| `GET` | `/api/attention` | Get operator attention items | read | no |
|
||||||
| `GET` | `/api/config` | Get non-secret application configuration | read | no |
|
| `GET` | `/api/config` | Get non-secret application configuration | read | no |
|
||||||
| `PUT` | `/api/config` | Update application configuration | admin | yes |
|
| `PUT` | `/api/config` | Update application configuration | admin | yes |
|
||||||
| `GET` | `/api/login/status` | Get Reddit session state | read | no |
|
| `GET` | `/api/login/status` | Get Reddit session state | read | no |
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
import { z } from 'zod';
|
||||||
|
import { RedditAccountOwnerSchema } from './session.js';
|
||||||
|
|
||||||
|
export const AttentionKindSchema = z.enum([
|
||||||
|
'human-required',
|
||||||
|
'boundary',
|
||||||
|
'failed-delivery',
|
||||||
|
'account-login-required',
|
||||||
|
'runtime-failed',
|
||||||
|
'invitation-cooldown',
|
||||||
|
]);
|
||||||
|
|
||||||
|
export const AttentionItemSchema = z.strictObject({
|
||||||
|
id: z.string(),
|
||||||
|
kind: AttentionKindSchema,
|
||||||
|
severity: z.enum(['warning', 'critical']),
|
||||||
|
title: z.string(),
|
||||||
|
detail: z.string(),
|
||||||
|
occurredAt: z.number(),
|
||||||
|
owner: RedditAccountOwnerSchema.optional(),
|
||||||
|
conversationId: z.string().optional(),
|
||||||
|
username: z.string().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const AttentionResponseSchema = z.strictObject({
|
||||||
|
generatedAt: z.number(),
|
||||||
|
items: z.array(AttentionItemSchema),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type AttentionItem = z.infer<typeof AttentionItemSchema>;
|
||||||
|
export type AttentionResponse = z.infer<typeof AttentionResponseSchema>;
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
export * from './activity.js';
|
export * from './activity.js';
|
||||||
|
export * from './attention.js';
|
||||||
export * from './chats.js';
|
export * from './chats.js';
|
||||||
export * from './common.js';
|
export * from './common.js';
|
||||||
export * from './config.js';
|
export * from './config.js';
|
||||||
|
|||||||
@@ -74,5 +74,24 @@ export const FunnelResponseSchema = z.strictObject({
|
|||||||
images: z.number(),
|
images: z.number(),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const AiUsageFields = {
|
||||||
|
requests: z.number().int().nonnegative(),
|
||||||
|
successes: z.number().int().nonnegative(),
|
||||||
|
refusals: z.number().int().nonnegative(),
|
||||||
|
errors: z.number().int().nonnegative(),
|
||||||
|
promptTokens: z.number().int().nonnegative(),
|
||||||
|
completionTokens: z.number().int().nonnegative(),
|
||||||
|
totalTokens: z.number().int().nonnegative(),
|
||||||
|
costUsd: z.number().nonnegative(),
|
||||||
|
averageLatencyMs: z.number().nonnegative(),
|
||||||
|
};
|
||||||
|
|
||||||
|
export const AiUsageSummaryResponseSchema = z.strictObject({
|
||||||
|
days: z.number().int().positive(),
|
||||||
|
...AiUsageFields,
|
||||||
|
models: z.array(z.strictObject({ model: z.string(), ...AiUsageFields })),
|
||||||
|
});
|
||||||
|
|
||||||
export type StatsResponse = z.infer<typeof StatsResponseSchema>;
|
export type StatsResponse = z.infer<typeof StatsResponseSchema>;
|
||||||
export type TimersResponse = z.infer<typeof TimersResponseSchema>;
|
export type TimersResponse = z.infer<typeof TimersResponseSchema>;
|
||||||
|
export type AiUsageSummaryResponse = z.infer<typeof AiUsageSummaryResponseSchema>;
|
||||||
+96
-22
@@ -1,6 +1,7 @@
|
|||||||
import OpenAI from 'openai';
|
import OpenAI from 'openai';
|
||||||
import type { Config } from '../config/index.js';
|
import type { Config } from '../config/index.js';
|
||||||
import type { Logger } from '../utils/logger.js';
|
import type { Logger } from '../utils/logger.js';
|
||||||
|
import type { AiUsageRecordInput, AiUsageOutcome } from '../database/ai-usage-repository.js';
|
||||||
import {
|
import {
|
||||||
formatConversationAsUntrustedData,
|
formatConversationAsUntrustedData,
|
||||||
isSafeConversationReply,
|
isSafeConversationReply,
|
||||||
@@ -29,6 +30,8 @@ interface ProtectedTextPrompt {
|
|||||||
user: string;
|
user: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type AiUsageRecorder = (usage: AiUsageRecordInput) => void;
|
||||||
|
|
||||||
export class OpenRouterClient {
|
export class OpenRouterClient {
|
||||||
private client: OpenAI;
|
private client: OpenAI;
|
||||||
|
|
||||||
@@ -36,6 +39,7 @@ export class OpenRouterClient {
|
|||||||
private config: Config,
|
private config: Config,
|
||||||
private logger: Logger,
|
private logger: Logger,
|
||||||
client?: OpenAI,
|
client?: OpenAI,
|
||||||
|
private readonly usageRecorder?: AiUsageRecorder,
|
||||||
) {
|
) {
|
||||||
this.client = client ?? new OpenAI({
|
this.client = client ?? new OpenAI({
|
||||||
baseURL: 'https://openrouter.ai/api/v1',
|
baseURL: 'https://openrouter.ai/api/v1',
|
||||||
@@ -48,6 +52,7 @@ export class OpenRouterClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async detectGenderOnly(imageBase64: string, mimeType: string): Promise<DetectedGender> {
|
async detectGenderOnly(imageBase64: string, mimeType: string): Promise<DetectedGender> {
|
||||||
|
const startedAt = Date.now();
|
||||||
try {
|
try {
|
||||||
this.logger.debug('Quick gender detection...');
|
this.logger.debug('Quick gender detection...');
|
||||||
|
|
||||||
@@ -74,12 +79,14 @@ export class OpenRouterClient {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const result = response.choices[0]?.message?.content?.toLowerCase().trim() || 'unknown';
|
const result = response.choices[0]?.message?.content?.toLowerCase().trim() || 'unknown';
|
||||||
|
this.recordUsage('detectGender', this.config.openrouter.imageAnalysisModel, startedAt, response, result ? 'success' : 'empty');
|
||||||
|
|
||||||
if (result.includes('female')) return 'female';
|
if (result.includes('female')) return 'female';
|
||||||
if (result.includes('male') && !result.includes('female')) return 'male';
|
if (result.includes('male') && !result.includes('female')) return 'male';
|
||||||
if (result.includes('other')) return 'other';
|
if (result.includes('other')) return 'other';
|
||||||
return 'unknown';
|
return 'unknown';
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
this.recordError('detectGender', this.config.openrouter.imageAnalysisModel, startedAt, error);
|
||||||
this.logger.error('Gender detection failed:', error);
|
this.logger.error('Gender detection failed:', error);
|
||||||
return 'unknown';
|
return 'unknown';
|
||||||
}
|
}
|
||||||
@@ -134,33 +141,40 @@ export class OpenRouterClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async tryAnalyzeWithModel(model: string, imageBase64: string, mimeType: string): Promise<ImageAnalysisResult | null> {
|
private async tryAnalyzeWithModel(model: string, imageBase64: string, mimeType: string): Promise<ImageAnalysisResult | null> {
|
||||||
|
const startedAt = Date.now();
|
||||||
const genderPrompt = `First, identify the gender of the person in this image. Respond with EXACTLY one of these on the first line: GENDER:male, GENDER:female, GENDER:other, or GENDER:unknown
|
const genderPrompt = `First, identify the gender of the person in this image. Respond with EXACTLY one of these on the first line: GENDER:male, GENDER:female, GENDER:other, or GENDER:unknown
|
||||||
|
|
||||||
Then provide your analysis:
|
Then provide your analysis:
|
||||||
|
|
||||||
${this.config.openrouter.imageAnalysisPrompt}`;
|
${this.config.openrouter.imageAnalysisPrompt}`;
|
||||||
|
|
||||||
const response = await this.client.chat.completions.create({
|
let response;
|
||||||
model,
|
try {
|
||||||
messages: [
|
response = await this.client.chat.completions.create({
|
||||||
{
|
model,
|
||||||
role: 'user',
|
messages: [
|
||||||
content: [
|
{
|
||||||
{
|
role: 'user',
|
||||||
type: 'text',
|
content: [
|
||||||
text: genderPrompt,
|
{
|
||||||
},
|
type: 'text',
|
||||||
{
|
text: genderPrompt,
|
||||||
type: 'image_url',
|
|
||||||
image_url: {
|
|
||||||
url: `data:${mimeType};base64,${imageBase64}`,
|
|
||||||
},
|
},
|
||||||
},
|
{
|
||||||
],
|
type: 'image_url',
|
||||||
},
|
image_url: {
|
||||||
],
|
url: `data:${mimeType};base64,${imageBase64}`,
|
||||||
max_tokens: 600,
|
},
|
||||||
});
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
max_tokens: 600,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
this.recordError('analyzeImage', model, startedAt, error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
const fullResponse = response.choices[0]?.message?.content || '';
|
const fullResponse = response.choices[0]?.message?.content || '';
|
||||||
|
|
||||||
@@ -190,9 +204,12 @@ ${this.config.openrouter.imageAnalysisPrompt}`;
|
|||||||
|
|
||||||
// Detect AI refusal - return null to signal fallback should be tried
|
// Detect AI refusal - return null to signal fallback should be tried
|
||||||
if (this.isRefusal(description)) {
|
if (this.isRefusal(description)) {
|
||||||
|
this.recordUsage('analyzeImage', model, startedAt, response, 'refusal');
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.recordUsage('analyzeImage', model, startedAt, response, description ? 'success' : 'empty');
|
||||||
|
|
||||||
this.logger.debug(`Image analysis complete (${model}) - Gender: ${detectedGender}, Rating: ${attractivenessScore ?? 'n/a'}`);
|
this.logger.debug(`Image analysis complete (${model}) - Gender: ${detectedGender}, Rating: ${attractivenessScore ?? 'n/a'}`);
|
||||||
this.logger.debug(`Description: ${description}`);
|
this.logger.debug(`Description: ${description}`);
|
||||||
|
|
||||||
@@ -480,7 +497,10 @@ Just write the message, nothing else.`;
|
|||||||
if (isFallback) {
|
if (isFallback) {
|
||||||
this.logger.info(`[AI] ${label} trying fallback model: ${model}`);
|
this.logger.info(`[AI] ${label} trying fallback model: ${model}`);
|
||||||
}
|
}
|
||||||
const response = await this.client.chat.completions.create({
|
const startedAt = Date.now();
|
||||||
|
let response;
|
||||||
|
try {
|
||||||
|
response = await this.client.chat.completions.create({
|
||||||
model,
|
model,
|
||||||
messages: typeof prompt === 'string'
|
messages: typeof prompt === 'string'
|
||||||
? [{ role: 'user', content: prompt }]
|
? [{ role: 'user', content: prompt }]
|
||||||
@@ -490,7 +510,11 @@ Just write the message, nothing else.`;
|
|||||||
],
|
],
|
||||||
max_tokens: opts.max_tokens,
|
max_tokens: opts.max_tokens,
|
||||||
temperature: opts.temperature,
|
temperature: opts.temperature,
|
||||||
});
|
});
|
||||||
|
} catch (error) {
|
||||||
|
this.recordError(label, model, startedAt, error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
const raw = response.choices[0]?.message?.content || '';
|
const raw = response.choices[0]?.message?.content || '';
|
||||||
const choice = response.choices[0] as unknown as Record<string, unknown> | undefined;
|
const choice = response.choices[0] as unknown as Record<string, unknown> | undefined;
|
||||||
@@ -508,14 +532,18 @@ Just write the message, nothing else.`;
|
|||||||
|
|
||||||
const trimmed = raw.trim();
|
const trimmed = raw.trim();
|
||||||
if (!trimmed) {
|
if (!trimmed) {
|
||||||
|
this.recordUsage(label, model, startedAt, response, 'empty');
|
||||||
this.logger.warn(`[AI] ${label}: ${model} returned empty content${i < models.length - 1 ? ', trying fallback...' : ''}`);
|
this.logger.warn(`[AI] ${label}: ${model} returned empty content${i < models.length - 1 ? ', trying fallback...' : ''}`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (this.isRefusal(trimmed)) {
|
if (this.isRefusal(trimmed)) {
|
||||||
|
this.recordUsage(label, model, startedAt, response, 'refusal');
|
||||||
this.logger.warn(`[AI] ${label}: ${model} returned a refusal${i < models.length - 1 ? ', trying fallback...' : ''}`);
|
this.logger.warn(`[AI] ${label}: ${model} returned a refusal${i < models.length - 1 ? ', trying fallback...' : ''}`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.recordUsage(label, model, startedAt, response, 'success');
|
||||||
|
|
||||||
if (isFallback) {
|
if (isFallback) {
|
||||||
this.logger.info(`[AI] ${label}: fallback model ${model} succeeded`);
|
this.logger.info(`[AI] ${label}: fallback model ${model} succeeded`);
|
||||||
}
|
}
|
||||||
@@ -532,6 +560,48 @@ Just write the message, nothing else.`;
|
|||||||
return '';
|
return '';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private recordUsage(
|
||||||
|
operation: string,
|
||||||
|
requestedModel: string,
|
||||||
|
startedAt: number,
|
||||||
|
response: unknown,
|
||||||
|
outcome: AiUsageOutcome,
|
||||||
|
): void {
|
||||||
|
if (!this.usageRecorder) return;
|
||||||
|
const record = response as Record<string, unknown>;
|
||||||
|
const usage = (record.usage ?? {}) as Record<string, unknown>;
|
||||||
|
try {
|
||||||
|
this.usageRecorder({
|
||||||
|
operation,
|
||||||
|
requestedModel,
|
||||||
|
actualModel: typeof record.model === 'string' ? record.model : requestedModel,
|
||||||
|
outcome,
|
||||||
|
promptTokens: numericUsage(usage.prompt_tokens),
|
||||||
|
completionTokens: numericUsage(usage.completion_tokens),
|
||||||
|
totalTokens: numericUsage(usage.total_tokens),
|
||||||
|
costUsd: typeof usage.cost === 'number' && Number.isFinite(usage.cost) ? usage.cost : null,
|
||||||
|
latencyMs: Date.now() - startedAt,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn('Failed to persist AI usage accounting:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private recordError(operation: string, requestedModel: string, startedAt: number, error: unknown): void {
|
||||||
|
if (!this.usageRecorder) return;
|
||||||
|
try {
|
||||||
|
this.usageRecorder({
|
||||||
|
operation,
|
||||||
|
requestedModel,
|
||||||
|
outcome: 'error',
|
||||||
|
latencyMs: Date.now() - startedAt,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
} catch (recordingError) {
|
||||||
|
this.logger.warn('Failed to persist AI usage error:', recordingError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async analyzeAndGenerateMessage(
|
async analyzeAndGenerateMessage(
|
||||||
imageBase64: string,
|
imageBase64: string,
|
||||||
mimeType: string,
|
mimeType: string,
|
||||||
@@ -547,3 +617,7 @@ Just write the message, nothing else.`;
|
|||||||
return { analysis, message };
|
return { analysis, message };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function numericUsage(value: unknown): number {
|
||||||
|
return typeof value === 'number' && Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0;
|
||||||
|
}
|
||||||
+7
-1
@@ -47,6 +47,7 @@ import type { Cookie } from 'playwright';
|
|||||||
import { ChatCache } from './services/chat/cache.js';
|
import { ChatCache } from './services/chat/cache.js';
|
||||||
import { RedditAccountRuntimeManager } from './services/reddit-account-runtime.js';
|
import { RedditAccountRuntimeManager } from './services/reddit-account-runtime.js';
|
||||||
import { createConcreteRedditAccountRuntimeFactory } from './services/reddit-account-runtime-builder.js';
|
import { createConcreteRedditAccountRuntimeFactory } from './services/reddit-account-runtime-builder.js';
|
||||||
|
import { AiUsageRepository } from './database/ai-usage-repository.js';
|
||||||
import { importLegacyAccountContent } from './database/legacy-content-import.js';
|
import { importLegacyAccountContent } from './database/legacy-content-import.js';
|
||||||
import { RecipientDeduplicator } from './services/recipient-deduplicator.js';
|
import { RecipientDeduplicator } from './services/recipient-deduplicator.js';
|
||||||
import { createConfigBackedSubredditPollingControl } from './services/subreddit-polling-control.js';
|
import { createConfigBackedSubredditPollingControl } from './services/subreddit-polling-control.js';
|
||||||
@@ -127,6 +128,7 @@ export async function bootstrap(
|
|||||||
const recipientDeduplicator = new RecipientDeduplicator(new RecipientHistoryRepository(db));
|
const recipientDeduplicator = new RecipientDeduplicator(new RecipientHistoryRepository(db));
|
||||||
const redditAccountsRepo = bootstrapAccountsRepo;
|
const redditAccountsRepo = bootstrapAccountsRepo;
|
||||||
const accountContentRepo = new AccountContentRepository(db);
|
const accountContentRepo = new AccountContentRepository(db);
|
||||||
|
const aiUsageRepo = new AiUsageRepository(db);
|
||||||
const legacyImport = importLegacyAccountContent(db);
|
const legacyImport = importLegacyAccountContent(db);
|
||||||
if (legacyImport.conversations > 0 || legacyImport.media > 0) {
|
if (legacyImport.conversations > 0 || legacyImport.media > 0) {
|
||||||
logger.info(`Imported legacy unassigned content into SQLite (${legacyImport.conversations} conversations, ${legacyImport.media} media items)`);
|
logger.info(`Imported legacy unassigned content into SQLite (${legacyImport.conversations} conversations, ${legacyImport.media} media items)`);
|
||||||
@@ -168,7 +170,9 @@ export async function bootstrap(
|
|||||||
const chatPage = await browserClient.getContext()!.newPage();
|
const chatPage = await browserClient.getContext()!.newPage();
|
||||||
|
|
||||||
// AI + services
|
// AI + services
|
||||||
const aiClient = new OpenRouterClient(config, logger);
|
const aiClient = new OpenRouterClient(config, logger, undefined, (usage) => {
|
||||||
|
aiUsageRepo.record({ ...usage, accountId: accountAuth.getSnapshot().accountId });
|
||||||
|
});
|
||||||
const monitorService = new MonitorService(
|
const monitorService = new MonitorService(
|
||||||
redditApi, postsRepo, recipientDeduplicator, config, logger, 1000,
|
redditApi, postsRepo, recipientDeduplicator, config, logger, 1000,
|
||||||
createConfigBackedSubredditPollingControl(config, new ConfigStore(db), logger),
|
createConfigBackedSubredditPollingControl(config, new ConfigStore(db), logger),
|
||||||
@@ -344,6 +348,8 @@ export async function bootstrap(
|
|||||||
activityLogRepo,
|
activityLogRepo,
|
||||||
logger,
|
logger,
|
||||||
broadcaster,
|
broadcaster,
|
||||||
|
process.cwd(),
|
||||||
|
aiUsageRepo,
|
||||||
);
|
);
|
||||||
|
|
||||||
webServer.setChatService(chatService);
|
webServer.setChatService(chatService);
|
||||||
|
|||||||
@@ -0,0 +1,130 @@
|
|||||||
|
import type Database from 'better-sqlite3';
|
||||||
|
|
||||||
|
export type AiUsageOutcome = 'success' | 'refusal' | 'empty' | 'error';
|
||||||
|
|
||||||
|
export interface AiUsageRecordInput {
|
||||||
|
accountId?: string | null;
|
||||||
|
operation: string;
|
||||||
|
requestedModel: string;
|
||||||
|
actualModel?: string | null;
|
||||||
|
outcome: AiUsageOutcome;
|
||||||
|
promptTokens?: number;
|
||||||
|
completionTokens?: number;
|
||||||
|
totalTokens?: number;
|
||||||
|
costUsd?: number | null;
|
||||||
|
latencyMs: number;
|
||||||
|
error?: string | null;
|
||||||
|
createdAt?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AiUsageModelSummary {
|
||||||
|
model: string;
|
||||||
|
requests: number;
|
||||||
|
successes: number;
|
||||||
|
refusals: number;
|
||||||
|
errors: number;
|
||||||
|
promptTokens: number;
|
||||||
|
completionTokens: number;
|
||||||
|
totalTokens: number;
|
||||||
|
costUsd: number;
|
||||||
|
averageLatencyMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AiUsageSummary {
|
||||||
|
days: number;
|
||||||
|
requests: number;
|
||||||
|
successes: number;
|
||||||
|
refusals: number;
|
||||||
|
errors: number;
|
||||||
|
promptTokens: number;
|
||||||
|
completionTokens: number;
|
||||||
|
totalTokens: number;
|
||||||
|
costUsd: number;
|
||||||
|
averageLatencyMs: number;
|
||||||
|
models: AiUsageModelSummary[];
|
||||||
|
}
|
||||||
|
|
||||||
|
type SummaryRow = {
|
||||||
|
model?: string;
|
||||||
|
requests: number;
|
||||||
|
successes: number;
|
||||||
|
refusals: number;
|
||||||
|
errors: number;
|
||||||
|
prompt_tokens: number;
|
||||||
|
completion_tokens: number;
|
||||||
|
total_tokens: number;
|
||||||
|
cost_usd: number;
|
||||||
|
average_latency_ms: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Durable, prompt-free accounting for OpenRouter requests. */
|
||||||
|
export class AiUsageRepository {
|
||||||
|
constructor(private readonly db: Database.Database) {}
|
||||||
|
|
||||||
|
record(input: AiUsageRecordInput): void {
|
||||||
|
this.db.prepare(`
|
||||||
|
INSERT INTO ai_usage (
|
||||||
|
account_id, operation, requested_model, actual_model, outcome,
|
||||||
|
prompt_tokens, completion_tokens, total_tokens, cost_usd,
|
||||||
|
latency_ms, error, created_at
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
`).run(
|
||||||
|
input.accountId ?? null,
|
||||||
|
input.operation,
|
||||||
|
input.requestedModel,
|
||||||
|
input.actualModel ?? null,
|
||||||
|
input.outcome,
|
||||||
|
input.promptTokens ?? 0,
|
||||||
|
input.completionTokens ?? 0,
|
||||||
|
input.totalTokens ?? 0,
|
||||||
|
input.costUsd ?? null,
|
||||||
|
Math.max(0, Math.round(input.latencyMs)),
|
||||||
|
input.error?.slice(0, 500) ?? null,
|
||||||
|
input.createdAt ?? Date.now(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
getSummary(days = 30, now = Date.now()): AiUsageSummary {
|
||||||
|
const safeDays = Math.max(1, Math.min(365, Math.trunc(days)));
|
||||||
|
const since = now - safeDays * 86_400_000;
|
||||||
|
const fields = `
|
||||||
|
COUNT(*) AS requests,
|
||||||
|
SUM(CASE WHEN outcome = 'success' THEN 1 ELSE 0 END) AS successes,
|
||||||
|
SUM(CASE WHEN outcome = 'refusal' THEN 1 ELSE 0 END) AS refusals,
|
||||||
|
SUM(CASE WHEN outcome IN ('empty', 'error') THEN 1 ELSE 0 END) AS errors,
|
||||||
|
COALESCE(SUM(prompt_tokens), 0) AS prompt_tokens,
|
||||||
|
COALESCE(SUM(completion_tokens), 0) AS completion_tokens,
|
||||||
|
COALESCE(SUM(total_tokens), 0) AS total_tokens,
|
||||||
|
COALESCE(SUM(cost_usd), 0) AS cost_usd,
|
||||||
|
COALESCE(ROUND(AVG(latency_ms)), 0) AS average_latency_ms
|
||||||
|
`;
|
||||||
|
const total = this.db.prepare(`SELECT ${fields} FROM ai_usage WHERE created_at >= ?`)
|
||||||
|
.get(since) as SummaryRow;
|
||||||
|
const models = this.db.prepare(`
|
||||||
|
SELECT COALESCE(actual_model, requested_model) AS model, ${fields}
|
||||||
|
FROM ai_usage
|
||||||
|
WHERE created_at >= ?
|
||||||
|
GROUP BY COALESCE(actual_model, requested_model)
|
||||||
|
ORDER BY cost_usd DESC, requests DESC, model ASC
|
||||||
|
`).all(since) as SummaryRow[];
|
||||||
|
return {
|
||||||
|
days: safeDays,
|
||||||
|
...mapSummary(total),
|
||||||
|
models: models.map((row) => ({ model: row.model ?? 'unknown', ...mapSummary(row) })),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function mapSummary(row: SummaryRow): Omit<AiUsageModelSummary, 'model'> {
|
||||||
|
return {
|
||||||
|
requests: Number(row.requests ?? 0),
|
||||||
|
successes: Number(row.successes ?? 0),
|
||||||
|
refusals: Number(row.refusals ?? 0),
|
||||||
|
errors: Number(row.errors ?? 0),
|
||||||
|
promptTokens: Number(row.prompt_tokens ?? 0),
|
||||||
|
completionTokens: Number(row.completion_tokens ?? 0),
|
||||||
|
totalTokens: Number(row.total_tokens ?? 0),
|
||||||
|
costUsd: Math.round(Number(row.cost_usd ?? 0) * 1_000_000_000_000) / 1_000_000_000_000,
|
||||||
|
averageLatencyMs: Number(row.average_latency_ms ?? 0),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -251,6 +251,27 @@ export function runMigrations(db: Database.Database): void {
|
|||||||
CREATE INDEX idx_conversation_automation_claims_status
|
CREATE INDEX idx_conversation_automation_claims_status
|
||||||
ON conversation_automation_claims(account_id, status, claimed_at);
|
ON conversation_automation_claims(account_id, status, claimed_at);
|
||||||
`),
|
`),
|
||||||
|
() => db.exec(`
|
||||||
|
CREATE TABLE IF NOT EXISTS ai_usage (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
account_id TEXT,
|
||||||
|
operation TEXT NOT NULL,
|
||||||
|
requested_model TEXT NOT NULL,
|
||||||
|
actual_model TEXT,
|
||||||
|
outcome TEXT NOT NULL CHECK(outcome IN ('success', 'refusal', 'empty', 'error')),
|
||||||
|
prompt_tokens INTEGER NOT NULL DEFAULT 0,
|
||||||
|
completion_tokens INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_tokens INTEGER NOT NULL DEFAULT 0,
|
||||||
|
cost_usd REAL,
|
||||||
|
latency_ms INTEGER NOT NULL,
|
||||||
|
error TEXT,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
FOREIGN KEY (account_id) REFERENCES reddit_accounts(account_id) ON DELETE SET NULL
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ai_usage_created_at ON ai_usage(created_at DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ai_usage_model ON ai_usage(actual_model, requested_model, created_at DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ai_usage_account ON ai_usage(account_id, created_at DESC);
|
||||||
|
`),
|
||||||
];
|
];
|
||||||
|
|
||||||
let currentVersion = db.pragma('user_version', { simple: true }) as number;
|
let currentVersion = db.pragma('user_version', { simple: true }) as number;
|
||||||
|
|||||||
@@ -227,6 +227,16 @@ export class DeliveryOutboxRepository {
|
|||||||
.get(this.accountId) as { count: number }).count;
|
.get(this.accountId) as { count: number }).count;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getFailed(limit = 50): DeliveryOutboxRecord[] {
|
||||||
|
const rows = this.db.prepare(`
|
||||||
|
SELECT * FROM delivery_outbox
|
||||||
|
WHERE account_id = ? AND status = 'failed'
|
||||||
|
ORDER BY updated_at DESC
|
||||||
|
LIMIT ?
|
||||||
|
`).all(this.accountId, limit) as Record<string, unknown>[];
|
||||||
|
return rows.map((row) => this.mapRow(row));
|
||||||
|
}
|
||||||
|
|
||||||
markPrepared(id: number, now: number = Date.now()): boolean {
|
markPrepared(id: number, now: number = Date.now()): boolean {
|
||||||
return this.db.prepare(`
|
return this.db.prepare(`
|
||||||
UPDATE delivery_outbox SET status = 'prepared', updated_at = ?, last_error = NULL
|
UPDATE delivery_outbox SET status = 'prepared', updated_at = ?, last_error = NULL
|
||||||
@@ -1081,6 +1091,32 @@ export class ConversationStateRepository {
|
|||||||
const row = this.getStmt.get(this.accountId, conversationId) as { state_json: string } | undefined;
|
const row = this.getStmt.get(this.accountId, conversationId) as { state_json: string } | undefined;
|
||||||
return row ? parseConversationState(row.state_json) ?? undefined : undefined;
|
return row ? parseConversationState(row.state_json) ?? undefined : undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
listAttention(): Array<{
|
||||||
|
conversationId: string;
|
||||||
|
status: 'human-required' | 'boundary';
|
||||||
|
reason: string | null;
|
||||||
|
updatedAt: number;
|
||||||
|
}> {
|
||||||
|
const rows = this.db.prepare(`
|
||||||
|
SELECT conversation_id, state_json, updated_at
|
||||||
|
FROM conversation_states
|
||||||
|
WHERE account_id = ?
|
||||||
|
ORDER BY updated_at DESC
|
||||||
|
`).all(this.accountId) as Array<{ conversation_id: string; state_json: string; updated_at: number }>;
|
||||||
|
return rows.flatMap((row) => {
|
||||||
|
const state = parseConversationState(row.state_json);
|
||||||
|
if (!state) return [];
|
||||||
|
const status = state.automation.status;
|
||||||
|
if (status !== 'human-required' && status !== 'boundary') return [];
|
||||||
|
return [{
|
||||||
|
conversationId: row.conversation_id,
|
||||||
|
status,
|
||||||
|
reason: state.automation.handoffReason,
|
||||||
|
updatedAt: row.updated_at,
|
||||||
|
}];
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Durable idempotency claims for AI replies to individual inbound events. */
|
/** Durable idempotency claims for AI replies to individual inbound events. */
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ import { RecipientDeduplicator } from './recipient-deduplicator.js';
|
|||||||
import { ConfigStore } from '../config/persistence.js';
|
import { ConfigStore } from '../config/persistence.js';
|
||||||
import { createConfigBackedSubredditPollingControl } from './subreddit-polling-control.js';
|
import { createConfigBackedSubredditPollingControl } from './subreddit-polling-control.js';
|
||||||
import { ConversationAutomationService } from './conversation-automation.js';
|
import { ConversationAutomationService } from './conversation-automation.js';
|
||||||
|
import { AiUsageRepository } from '../database/ai-usage-repository.js';
|
||||||
|
|
||||||
export interface ConcreteRuntimeFactoryOptions {
|
export interface ConcreteRuntimeFactoryOptions {
|
||||||
config: Config;
|
config: Config;
|
||||||
@@ -118,7 +119,10 @@ async function buildRuntime(
|
|||||||
const redditApi = new RedditApiClient(accountConfig, logger, redditRequestCoordinator);
|
const redditApi = new RedditApiClient(accountConfig, logger, redditRequestCoordinator);
|
||||||
redditApi.setBrowserContext(browserClient.getContext()!);
|
redditApi.setBrowserContext(browserClient.getContext()!);
|
||||||
const chatPage = await browserClient.getContext()!.newPage();
|
const chatPage = await browserClient.getContext()!.newPage();
|
||||||
const aiClient = new OpenRouterClient(accountConfig, logger);
|
const aiUsageRepo = new AiUsageRepository(db);
|
||||||
|
const aiClient = new OpenRouterClient(accountConfig, logger, undefined, (usage) => {
|
||||||
|
aiUsageRepo.record({ ...usage, accountId: session.accountId });
|
||||||
|
});
|
||||||
const monitorService = new MonitorService(
|
const monitorService = new MonitorService(
|
||||||
redditApi, postsRepo, recipientDeduplicator, accountConfig, logger,
|
redditApi, postsRepo, recipientDeduplicator, accountConfig, logger,
|
||||||
1000, createConfigBackedSubredditPollingControl(config, new ConfigStore(db), logger),
|
1000, createConfigBackedSubredditPollingControl(config, new ConfigStore(db), logger),
|
||||||
|
|||||||
@@ -94,6 +94,8 @@ export const API_OPERATIONS: readonly ApiOperation[] = [
|
|||||||
operation('getStrategyAnalytics', 'GET', '/api/analytics/by-strategy', 'Get performance by strategy', ['Analytics']),
|
operation('getStrategyAnalytics', 'GET', '/api/analytics/by-strategy', 'Get performance by strategy', ['Analytics']),
|
||||||
operation('getFunnelAnalytics', 'GET', '/api/analytics/funnel', 'Get mature message outcome funnel', ['Analytics']),
|
operation('getFunnelAnalytics', 'GET', '/api/analytics/funnel', 'Get mature message outcome funnel', ['Analytics']),
|
||||||
operation('getAnalyticsTrend', 'GET', '/api/analytics/trend', 'Get daily performance trend', ['Analytics']),
|
operation('getAnalyticsTrend', 'GET', '/api/analytics/trend', 'Get daily performance trend', ['Analytics']),
|
||||||
|
operation('getAiUsageAnalytics', 'GET', '/api/analytics/ai-usage', 'Get OpenRouter cost and model reliability', ['Analytics']),
|
||||||
|
operation('getAttention', 'GET', '/api/attention', 'Get operator attention items', ['Operations']),
|
||||||
operation('getConfig', 'GET', '/api/config', 'Get non-secret application configuration', ['Configuration']),
|
operation('getConfig', 'GET', '/api/config', 'Get non-secret application configuration', ['Configuration']),
|
||||||
operation('updateConfig', 'PUT', '/api/config', 'Update application configuration', ['Configuration']),
|
operation('updateConfig', 'PUT', '/api/config', 'Update application configuration', ['Configuration']),
|
||||||
operation('getLoginStatus', 'GET', '/api/login/status', 'Get Reddit session state', ['Accounts']),
|
operation('getLoginStatus', 'GET', '/api/login/status', 'Get Reddit session state', ['Accounts']),
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import type { AccountAuthContext } from '../services/account-auth-context.js';
|
|||||||
import type { ConfigStore } from '../config/persistence.js';
|
import type { ConfigStore } from '../config/persistence.js';
|
||||||
import type { RedditAccountsService } from '../services/reddit-accounts.js';
|
import type { RedditAccountsService } from '../services/reddit-accounts.js';
|
||||||
import type { RedditAccountRuntimeManager } from '../services/reddit-account-runtime.js';
|
import type { RedditAccountRuntimeManager } from '../services/reddit-account-runtime.js';
|
||||||
|
import type { AiUsageRepository } from '../database/ai-usage-repository.js';
|
||||||
|
|
||||||
/** Bag of everything the route modules might need. */
|
/** Bag of everything the route modules might need. */
|
||||||
export interface WebContext {
|
export interface WebContext {
|
||||||
@@ -35,6 +36,7 @@ export interface WebContext {
|
|||||||
posts: PostsRepository;
|
posts: PostsRepository;
|
||||||
messages: MessagesRepository;
|
messages: MessagesRepository;
|
||||||
successful: SuccessfulMessagesRepository;
|
successful: SuccessfulMessagesRepository;
|
||||||
|
aiUsage?: AiUsageRepository;
|
||||||
};
|
};
|
||||||
|
|
||||||
services: {
|
services: {
|
||||||
|
|||||||
@@ -37,5 +37,14 @@ export function createAnalyticsRouter(ctx: WebContext): Router {
|
|||||||
res.json(ctx.repos.messages.getResponseRateTrend(days));
|
res.json(ctx.repos.messages.getResponseRateTrend(days));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
router.get('/analytics/ai-usage', (req, res) => {
|
||||||
|
if (!ctx.repos.aiUsage) {
|
||||||
|
res.status(503).json({ error: 'AI usage accounting is not initialized' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const days = parseInt(req.query.days as string) || 30;
|
||||||
|
res.json(ctx.repos.aiUsage.getSummary(days));
|
||||||
|
});
|
||||||
|
|
||||||
return router;
|
return router;
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
import { Router } from 'express';
|
||||||
|
import type { WebContext } from '../context.js';
|
||||||
|
|
||||||
|
interface AttentionItem {
|
||||||
|
id: string;
|
||||||
|
kind: 'human-required' | 'boundary' | 'failed-delivery' | 'account-login-required' | 'runtime-failed' | 'invitation-cooldown';
|
||||||
|
severity: 'warning' | 'critical';
|
||||||
|
title: string;
|
||||||
|
detail: string;
|
||||||
|
occurredAt: number;
|
||||||
|
owner?: { accountId: string; redditUsername: string };
|
||||||
|
conversationId?: string;
|
||||||
|
username?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createAttentionRouter(ctx: WebContext): Router {
|
||||||
|
const router = Router();
|
||||||
|
|
||||||
|
router.get('/attention', async (_req, res) => {
|
||||||
|
const items: AttentionItem[] = [];
|
||||||
|
const accounts = ctx.services.redditAccounts?.list() ?? [];
|
||||||
|
for (const account of accounts) {
|
||||||
|
if (account.status !== 'login-required') continue;
|
||||||
|
items.push({
|
||||||
|
id: `account:${account.accountId}`,
|
||||||
|
kind: 'account-login-required',
|
||||||
|
severity: 'critical',
|
||||||
|
title: `u/${account.redditUsername} needs a new session`,
|
||||||
|
detail: 'Import a fresh extension session before this account can resume.',
|
||||||
|
occurredAt: account.updatedAt,
|
||||||
|
owner: { accountId: account.accountId, redditUsername: account.redditUsername },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const snapshots = ctx.services.accountRuntimes?.list() ?? [];
|
||||||
|
for (const snapshot of snapshots) {
|
||||||
|
const owner = { accountId: snapshot.accountId, redditUsername: snapshot.redditUsername };
|
||||||
|
if (snapshot.state === 'failed') {
|
||||||
|
items.push({
|
||||||
|
id: `runtime:${snapshot.accountId}`,
|
||||||
|
kind: 'runtime-failed',
|
||||||
|
severity: 'critical',
|
||||||
|
title: `u/${snapshot.redditUsername} runtime failed`,
|
||||||
|
detail: snapshot.failure ?? 'The account runtime stopped unexpectedly.',
|
||||||
|
occurredAt: snapshot.updatedAt,
|
||||||
|
owner,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const runtime = ctx.services.accountRuntimes?.get(snapshot.accountId);
|
||||||
|
if (!runtime) continue;
|
||||||
|
const conversations = await runtime.resources.chatService.getCachedConversations();
|
||||||
|
const usernames = new Map(conversations.flatMap((conversation) => conversation.roomId
|
||||||
|
? [[conversation.roomId, conversation.username] as const]
|
||||||
|
: []));
|
||||||
|
for (const state of runtime.resources.conversationStateRepo.listAttention()) {
|
||||||
|
const username = usernames.get(state.conversationId);
|
||||||
|
items.push({
|
||||||
|
id: `conversation:${snapshot.accountId}:${state.conversationId}:${state.status}`,
|
||||||
|
kind: state.status,
|
||||||
|
severity: state.status === 'boundary' ? 'critical' : 'warning',
|
||||||
|
title: state.status === 'boundary'
|
||||||
|
? `Boundary detected${username ? ` from u/${username}` : ''}`
|
||||||
|
: `Human follow-up required${username ? ` for u/${username}` : ''}`,
|
||||||
|
detail: state.reason ?? (state.status === 'boundary'
|
||||||
|
? 'Automation stopped after the recipient set a boundary.'
|
||||||
|
: 'Automated conversation turns are exhausted.'),
|
||||||
|
occurredAt: state.updatedAt,
|
||||||
|
owner,
|
||||||
|
conversationId: `${snapshot.accountId}~${state.conversationId}`,
|
||||||
|
...(username ? { username } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
for (const delivery of runtime.resources.deliveryOutboxRepo.getFailed()) {
|
||||||
|
items.push({
|
||||||
|
id: `delivery:${snapshot.accountId}:${delivery.id}`,
|
||||||
|
kind: 'failed-delivery',
|
||||||
|
severity: 'warning',
|
||||||
|
title: `Message to u/${delivery.recipient} failed`,
|
||||||
|
detail: delivery.lastError ?? `Delivery failed after ${delivery.attemptCount} attempt(s).`,
|
||||||
|
occurredAt: delivery.updatedAt,
|
||||||
|
owner,
|
||||||
|
username: delivery.recipient,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const cooldown = runtime.resources.chatService.getInvitationCooldown();
|
||||||
|
if (cooldown) {
|
||||||
|
items.push({
|
||||||
|
id: `cooldown:${snapshot.accountId}:${cooldown.until}`,
|
||||||
|
kind: 'invitation-cooldown',
|
||||||
|
severity: 'warning',
|
||||||
|
title: `u/${snapshot.redditUsername} invitation cooldown`,
|
||||||
|
detail: cooldown.reason,
|
||||||
|
occurredAt: Date.now(),
|
||||||
|
owner,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
res.json({ generatedAt: Date.now(), items: items.sort((left, right) => right.occurredAt - left.occurredAt) });
|
||||||
|
});
|
||||||
|
|
||||||
|
return router;
|
||||||
|
}
|
||||||
@@ -41,6 +41,8 @@ import { createLoginRouter } from './routes/login.js';
|
|||||||
import { createDashboardAuth, createDashboardSessionRouter } from './auth.js';
|
import { createDashboardAuth, createDashboardSessionRouter } from './auth.js';
|
||||||
import { resolveDashboardAssets } from './dashboard-assets.js';
|
import { resolveDashboardAssets } from './dashboard-assets.js';
|
||||||
import { createAgentRouter, createHealthRouter } from './routes/agent.js';
|
import { createAgentRouter, createHealthRouter } from './routes/agent.js';
|
||||||
|
import { createAttentionRouter } from './routes/attention.js';
|
||||||
|
import type { AiUsageRepository } from '../database/ai-usage-repository.js';
|
||||||
|
|
||||||
export type { BotStats, ActivityItem, LogEntry, RuntimeConfig } from './types.js';
|
export type { BotStats, ActivityItem, LogEntry, RuntimeConfig } from './types.js';
|
||||||
|
|
||||||
@@ -74,6 +76,7 @@ export class WebServer {
|
|||||||
private logger: Logger,
|
private logger: Logger,
|
||||||
broadcaster?: LogBroadcaster,
|
broadcaster?: LogBroadcaster,
|
||||||
private workspaceDirectory = process.cwd(),
|
private workspaceDirectory = process.cwd(),
|
||||||
|
private aiUsageRepo?: AiUsageRepository,
|
||||||
) {
|
) {
|
||||||
this.broadcaster = broadcaster ?? new LogBroadcaster();
|
this.broadcaster = broadcaster ?? new LogBroadcaster();
|
||||||
this.activity = new ActivityLog(activityLogRepo);
|
this.activity = new ActivityLog(activityLogRepo);
|
||||||
@@ -104,6 +107,7 @@ export class WebServer {
|
|||||||
posts: this.postsRepo,
|
posts: this.postsRepo,
|
||||||
messages: this.messagesRepo,
|
messages: this.messagesRepo,
|
||||||
successful: this.successfulMessagesRepo,
|
successful: this.successfulMessagesRepo,
|
||||||
|
aiUsage: this.aiUsageRepo,
|
||||||
},
|
},
|
||||||
services: this.services,
|
services: this.services,
|
||||||
getBotRunning: () => this.botRunning,
|
getBotRunning: () => this.botRunning,
|
||||||
@@ -147,6 +151,7 @@ export class WebServer {
|
|||||||
const ctx = this.context();
|
const ctx = this.context();
|
||||||
this.app.use('/api', createAgentRouter(ctx));
|
this.app.use('/api', createAgentRouter(ctx));
|
||||||
this.app.use('/api', createStatsRouter(ctx));
|
this.app.use('/api', createStatsRouter(ctx));
|
||||||
|
this.app.use('/api', createAttentionRouter(ctx));
|
||||||
this.app.use('/api', createChatsRouter(ctx));
|
this.app.use('/api', createChatsRouter(ctx));
|
||||||
this.app.use('/api', createConversationsRouter(ctx));
|
this.app.use('/api', createConversationsRouter(ctx));
|
||||||
this.app.use('/api', createMediaRouter(ctx));
|
this.app.use('/api', createMediaRouter(ctx));
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { AiUsageRepository } from '../src/database/ai-usage-repository.js';
|
||||||
|
import { createTestDb } from './testUtils.js';
|
||||||
|
|
||||||
|
describe('AiUsageRepository', () => {
|
||||||
|
it('aggregates cost, tokens, outcomes, and latency by actual model', () => {
|
||||||
|
const db = createTestDb();
|
||||||
|
const repo = new AiUsageRepository(db);
|
||||||
|
const now = Date.parse('2026-08-09T10:00:00Z');
|
||||||
|
repo.record({ operation: 'generateReply', requestedModel: 'router', actualModel: 'model-a', outcome: 'success', promptTokens: 100, completionTokens: 20, totalTokens: 120, costUsd: 0.0012, latencyMs: 400, createdAt: now });
|
||||||
|
repo.record({ operation: 'generateReply', requestedModel: 'model-a', actualModel: 'model-a', outcome: 'refusal', promptTokens: 40, completionTokens: 5, totalTokens: 45, costUsd: 0.0003, latencyMs: 200, createdAt: now });
|
||||||
|
repo.record({ operation: 'analyzeImage', requestedModel: 'model-b', outcome: 'error', latencyMs: 900, error: 'provider unavailable', createdAt: now });
|
||||||
|
|
||||||
|
expect(repo.getSummary(30, now)).toEqual({
|
||||||
|
days: 30,
|
||||||
|
requests: 3,
|
||||||
|
successes: 1,
|
||||||
|
refusals: 1,
|
||||||
|
errors: 1,
|
||||||
|
promptTokens: 140,
|
||||||
|
completionTokens: 25,
|
||||||
|
totalTokens: 165,
|
||||||
|
costUsd: 0.0015,
|
||||||
|
averageLatencyMs: 500,
|
||||||
|
models: [
|
||||||
|
{ model: 'model-a', requests: 2, successes: 1, refusals: 1, errors: 0, promptTokens: 140, completionTokens: 25, totalTokens: 165, costUsd: 0.0015, averageLatencyMs: 300 },
|
||||||
|
{ model: 'model-b', requests: 1, successes: 0, refusals: 0, errors: 1, promptTokens: 0, completionTokens: 0, totalTokens: 0, costUsd: 0, averageLatencyMs: 900 },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
db.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -79,7 +79,7 @@ describe('dashboard foundation', () => {
|
|||||||
const { router } = renderDashboard('/');
|
const { router } = renderDashboard('/');
|
||||||
|
|
||||||
const navigation = screen.getByRole('navigation', { name: 'Primary navigation' });
|
const navigation = screen.getByRole('navigation', { name: 'Primary navigation' });
|
||||||
for (const label of ['Overview', 'Conversations', 'Media', 'Performance', 'Operations', 'Settings']) {
|
for (const label of ['Overview', 'Needs attention', 'Conversations', 'Media', 'Performance', 'Operations', 'Settings']) {
|
||||||
expect(within(navigation).getByRole('link', { name: label })).toBeInTheDocument();
|
expect(within(navigation).getByRole('link', { name: label })).toBeInTheDocument();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -91,6 +91,24 @@ describe('dashboard foundation', () => {
|
|||||||
expect(screen.getByRole('heading', { name: 'Operations' })).toBeInTheDocument();
|
expect(screen.getByRole('heading', { name: 'Operations' })).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('shows actionable attention items and links handoffs to their conversation', async () => {
|
||||||
|
installSuccessfulApi();
|
||||||
|
server.use(http.get('/api/attention', () => HttpResponse.json({
|
||||||
|
generatedAt: Date.parse('2026-08-09T08:00:00Z'),
|
||||||
|
items: [{
|
||||||
|
id: 'conversation:one', kind: 'human-required', severity: 'warning',
|
||||||
|
title: 'Human follow-up required for u/alice', detail: 'Automated turns are exhausted.',
|
||||||
|
occurredAt: Date.parse('2026-08-09T07:00:00Z'),
|
||||||
|
owner: { accountId: '@one:reddit.com', redditUsername: 'operator' },
|
||||||
|
conversationId: '@one:reddit.com~!room:reddit.com', username: 'alice',
|
||||||
|
}],
|
||||||
|
})));
|
||||||
|
renderDashboard('/attention');
|
||||||
|
expect(await screen.findByRole('heading', { name: 'Needs attention' })).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('Human follow-up required for u/alice')).toBeInTheDocument();
|
||||||
|
expect(screen.getByRole('link', { name: 'Open conversation' })).toHaveAttribute('href', '/conversations/%40one%3Areddit.com~!room%3Areddit.com');
|
||||||
|
});
|
||||||
|
|
||||||
it('hydrates media filters from the routed URL', async () => {
|
it('hydrates media filters from the routed URL', async () => {
|
||||||
installSuccessfulApi();
|
installSuccessfulApi();
|
||||||
server.use(
|
server.use(
|
||||||
|
|||||||
@@ -83,12 +83,24 @@ describe('PerformancePage', () => {
|
|||||||
upsell: 3,
|
upsell: 3,
|
||||||
images: 2,
|
images: 2,
|
||||||
})),
|
})),
|
||||||
|
http.get('/api/analytics/ai-usage', ({ request }) => {
|
||||||
|
expect(new URL(request.url).searchParams.get('days')).toBe('30');
|
||||||
|
return HttpResponse.json({
|
||||||
|
days: 30, requests: 12, successes: 9, refusals: 2, errors: 1,
|
||||||
|
promptTokens: 1200, completionTokens: 300, totalTokens: 1500,
|
||||||
|
costUsd: 0.0123, averageLatencyMs: 420,
|
||||||
|
models: [{ model: 'model-a', requests: 12, successes: 9, refusals: 2, errors: 1, promptTokens: 1200, completionTokens: 300, totalTokens: 1500, costUsd: 0.0123, averageLatencyMs: 420 }],
|
||||||
|
});
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
renderPage(<PerformancePage />);
|
renderPage(<PerformancePage />);
|
||||||
|
|
||||||
expect(await screen.findByRole('heading', { name: 'Performance' })).toBeInTheDocument();
|
expect(await screen.findByRole('heading', { name: 'Performance' })).toBeInTheDocument();
|
||||||
expect(screen.getByRole('heading', { name: 'Outcome funnel' })).toBeInTheDocument();
|
expect(screen.getByRole('heading', { name: 'Outcome funnel' })).toBeInTheDocument();
|
||||||
|
expect(screen.getByRole('heading', { name: 'AI cost and reliability' })).toBeInTheDocument();
|
||||||
|
expect(screen.getByRole('table', { name: 'AI usage by model' })).toHaveTextContent('model-a');
|
||||||
|
expect(screen.getByLabelText('AI usage totals')).toHaveTextContent('$0.0123');
|
||||||
expect(screen.getByText('5 awaiting maturity')).toBeInTheDocument();
|
expect(screen.getByText('5 awaiting maturity')).toBeInTheDocument();
|
||||||
const subredditTable = screen.getByRole('table', { name: 'Subreddit performance' });
|
const subredditTable = screen.getByRole('table', { name: 'Subreddit performance' });
|
||||||
expect(within(subredditTable).getByText('n = 40')).toBeInTheDocument();
|
expect(within(subredditTable).getByText('n = 40')).toBeInTheDocument();
|
||||||
|
|||||||
@@ -107,7 +107,7 @@ describe('Database Client', () => {
|
|||||||
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
|
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
|
||||||
.all() as { name: string }[];
|
.all() as { name: string }[];
|
||||||
// Existing domain/config tables plus account, conversation, sync and media ownership stores.
|
// Existing domain/config tables plus account, conversation, sync and media ownership stores.
|
||||||
expect(tables.length).toBe(15);
|
expect(tables.length).toBe(16);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('tracks schema version and migrates a legacy sent_messages table transactionally', () => {
|
it('tracks schema version and migrates a legacy sent_messages table transactionally', () => {
|
||||||
@@ -154,8 +154,9 @@ describe('Database Client', () => {
|
|||||||
db.exec(`
|
db.exec(`
|
||||||
DELETE FROM recipient_suppressions;
|
DELETE FROM recipient_suppressions;
|
||||||
DROP TABLE conversation_automation_claims;
|
DROP TABLE conversation_automation_claims;
|
||||||
|
DROP TABLE ai_usage;
|
||||||
`);
|
`);
|
||||||
db.pragma(`user_version = ${latestVersion - 1}`);
|
db.pragma(`user_version = ${latestVersion - 2}`);
|
||||||
runMigrations(db);
|
runMigrations(db);
|
||||||
|
|
||||||
expect(db.prepare('SELECT recipient, reason FROM recipient_suppressions ORDER BY recipient COLLATE NOCASE').all())
|
expect(db.prepare('SELECT recipient, reason FROM recipient_suppressions ORDER BY recipient COLLATE NOCASE').all())
|
||||||
|
|||||||
@@ -703,6 +703,36 @@ describe('OpenRouterClient', () => {
|
|||||||
).rejects.toThrow('network');
|
).rejects.toThrow('network');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('records model, token, cost, latency, and outcome for each request attempt', async () => {
|
||||||
|
const usageRecorder = vi.fn();
|
||||||
|
const config = getTestConfig();
|
||||||
|
const localClient = new OpenRouterClient(
|
||||||
|
config,
|
||||||
|
logger,
|
||||||
|
{ chat: { completions: { create: mockCreate } } } as unknown as OpenAI,
|
||||||
|
usageRecorder,
|
||||||
|
);
|
||||||
|
mockCreate.mockResolvedValueOnce({
|
||||||
|
model: 'provider/model-a',
|
||||||
|
choices: [{ message: { content: 'A useful reply' } }],
|
||||||
|
usage: { prompt_tokens: 100, completion_tokens: 12, total_tokens: 112, cost: 0.0017 },
|
||||||
|
});
|
||||||
|
|
||||||
|
await localClient.generateReply([{ sender: 'them', text: 'hello?', isSelf: false }], 'them');
|
||||||
|
|
||||||
|
expect(usageRecorder).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
operation: 'generateReply',
|
||||||
|
requestedModel: config.openrouter.chatModel,
|
||||||
|
actualModel: 'provider/model-a',
|
||||||
|
outcome: 'success',
|
||||||
|
promptTokens: 100,
|
||||||
|
completionTokens: 12,
|
||||||
|
totalTokens: 112,
|
||||||
|
costUsd: 0.0017,
|
||||||
|
latencyMs: expect.any(Number),
|
||||||
|
}));
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('stripDashes', () => {
|
describe('stripDashes', () => {
|
||||||
|
|||||||
@@ -70,4 +70,17 @@ describe('DeliveryOutboxRepository', () => {
|
|||||||
expect(outbox.markSent(delivery.id, 'worker-a', sentMessageId, 130)).toBe(true);
|
expect(outbox.markSent(delivery.id, 'worker-a', sentMessageId, 130)).toBe(true);
|
||||||
expect(outbox.get(delivery.id)).toMatchObject({ status: 'sent', sentMessageId, leaseOwner: null });
|
expect(outbox.get(delivery.id)).toMatchObject({ status: 'sent', sentMessageId, leaseOwner: null });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('surfaces failed deliveries for the attention inbox', () => {
|
||||||
|
const delivery = outbox.discover({
|
||||||
|
postId: 'post-1', recipient: 'recipient', subject: 'subject', message: 'message', idempotencyKey: 'post-1:v1', now: 100,
|
||||||
|
});
|
||||||
|
outbox.markPrepared(delivery.id, 110);
|
||||||
|
outbox.claim(delivery.id, 'worker-a', 50, 120);
|
||||||
|
outbox.markFailed(delivery.id, 'worker-a', 'Reddit rejected the invitation', 130);
|
||||||
|
|
||||||
|
expect(outbox.getFailed()).toEqual([
|
||||||
|
expect.objectContaining({ id: delivery.id, status: 'failed', lastError: 'Reddit rejected the invitation' }),
|
||||||
|
]);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
@@ -607,6 +607,21 @@ describe('ConversationStateRepository', () => {
|
|||||||
expect(repo.get('!room:reddit.com')?.persona.age).toBe(34);
|
expect(repo.get('!room:reddit.com')?.persona.age).toBe(34);
|
||||||
expect(repo.get('missing')).toBeUndefined();
|
expect(repo.get('missing')).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('lists only durable boundary and human-handoff states for operator attention', () => {
|
||||||
|
const db = createTestDb();
|
||||||
|
const repo = new ConversationStateRepository(db);
|
||||||
|
const active = buildConversationState([]);
|
||||||
|
const boundary = buildConversationState([{ sender: 'them', text: 'Stop messaging me.', isSelf: false }]);
|
||||||
|
boundary.automation.status = 'boundary';
|
||||||
|
boundary.automation.handoffReason = 'Recipient asked us to stop.';
|
||||||
|
repo.save('active-room', active);
|
||||||
|
repo.save('boundary-room', boundary);
|
||||||
|
|
||||||
|
expect(repo.listAttention()).toEqual([
|
||||||
|
expect.objectContaining({ conversationId: 'boundary-room', status: 'boundary', reason: 'Recipient asked us to stop.' }),
|
||||||
|
]);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('Mature performance and guardrails', () => {
|
describe('Mature performance and guardrails', () => {
|
||||||
|
|||||||
Reference in new issue
Block a user