Add attention inbox and AI usage observability

This commit is contained in:
jjbrunton committed 2026-08-09 08:52:32 +01:00
1 parent ca59ba0f1f
commit 83b5f1f5b6
30 files changed
+702 -28

No files matched your search

+1
View File
@@ -6,6 +6,7 @@ export interface NavigationItem {
export const navigationItems: readonly NavigationItem[] = [ export const navigationItems: readonly NavigationItem[] = [
{ label: 'Overview', path: '/', description: 'System health and current activity' }, { label: 'Overview', path: '/', description: 'System health and current activity' },
{ label: 'Needs attention', path: '/attention', description: 'Handoffs, failures, and account risks' },
{ label: 'Conversations', path: '/conversations', description: 'Inbox and message history' }, { label: 'Conversations', path: '/conversations', description: 'Inbox and message history' },
{ label: 'Media', path: '/media', description: 'Private media and sync controls' }, { label: 'Media', path: '/media', description: 'Private media and sync controls' },
{ label: 'Performance', path: '/performance', description: 'Response and strategy trends' }, { label: 'Performance', path: '/performance', description: 'Response and strategy trends' },
+2
View File
@@ -7,6 +7,7 @@ import { OperationsPage } from '../features/operations/operations-page.js';
import { PerformancePage } from '../features/performance/performance-page.js'; import { PerformancePage } from '../features/performance/performance-page.js';
import { SettingsRoute } from '../features/settings/settings-route.js'; import { SettingsRoute } from '../features/settings/settings-route.js';
import { RouteErrorPage } from './route-error-page.js'; import { RouteErrorPage } from './route-error-page.js';
import { AttentionPage } from '../features/attention/attention-page.js';
export function createDashboardRoutes(): RouteObject[] { export function createDashboardRoutes(): RouteObject[] {
return [ return [
@@ -16,6 +17,7 @@ export function createDashboardRoutes(): RouteObject[] {
errorElement: <RouteErrorPage />, errorElement: <RouteErrorPage />,
children: [ children: [
{ index: true, element: <OverviewPage /> }, { index: true, element: <OverviewPage /> },
{ path: 'attention', element: <AttentionPage /> },
{ path: 'conversations', element: <ConversationRoute /> }, { path: 'conversations', element: <ConversationRoute /> },
{ path: 'conversations/:conversationId', element: <ConversationRoute /> }, { path: 'conversations/:conversationId', element: <ConversationRoute /> },
{ path: 'media', element: <MediaRoute /> }, { path: 'media', element: <MediaRoute /> },
@@ -0,0 +1,16 @@
import { AttentionResponseSchema } from '../../../../../packages/api-contracts/src/index.js';
import { createDashboardRequest } from '../../api/request.js';
export function createAttentionApi(fetchImplementation?: typeof fetch) {
const request = createDashboardRequest(fetchImplementation);
return {
async getAttention() {
const response = await request('/api/attention');
const payload: unknown = await response.json();
if (!response.ok) throw new Error(`Attention inbox unavailable (${response.status})`);
return AttentionResponseSchema.parse(payload);
},
};
}
export const attentionApi = createAttentionApi();
@@ -0,0 +1,33 @@
import { useQuery } from '@tanstack/react-query';
import { Link } from 'react-router-dom';
import type { AttentionItem } from '../../../../../packages/api-contracts/src/index.js';
import { AccountOwnerBadge } from '../../components/account-owner-badge.js';
import { attentionApi } from './attention-api.js';
import './attention.css';
function formatTime(timestamp: number): string {
return new Intl.DateTimeFormat(undefined, { dateStyle: 'medium', timeStyle: 'short' }).format(timestamp);
}
function itemAction(item: AttentionItem) {
if (!item.conversationId) return null;
return <Link className="button secondary" to={`/conversations/${encodeURIComponent(item.conversationId)}`}>Open conversation</Link>;
}
export function AttentionPage() {
const query = useQuery({ queryKey: ['attention'], queryFn: attentionApi.getAttention, refetchInterval: 30_000 });
if (query.isPending) return <div className="page-state" role="status">Loading attention inbox…</div>;
if (query.isError) return <div className="page-state error-state" role="alert"><strong>Attention inbox unavailable</strong><span>{query.error.message}</span><button className="button secondary" onClick={() => void query.refetch()} type="button">Retry</button></div>;
const critical = query.data.items.filter((item) => item.severity === 'critical').length;
return (
<div className="page attention-page">
<header className="page-header"><div><span className="eyebrow">Operator queue</span><h1>Needs attention</h1><p>Boundaries, human handoffs, delivery failures, account problems, and Reddit cooldowns in one actionable inbox.</p></div><button className="button secondary" onClick={() => void query.refetch()} type="button">Refresh</button></header>
<section aria-label="Attention totals" className="status-strip"><div><span className={`status-dot ${query.data.items.length === 0 ? 'running' : 'stopped'}`} aria-hidden="true" /><div><small>Open items</small><strong>{query.data.items.length}</strong></div></div><span className={critical > 0 ? 'pill warning' : 'pill success'}>{critical} critical</span><span className="status-note">Updated {formatTime(query.data.generatedAt)}</span></section>
{query.data.items.length === 0 ? <div className="empty-state panel"><strong>Nothing needs attention.</strong><span>Automation and account health have no outstanding operator items.</span></div> : (
<ol aria-label="Attention items" className="attention-list">{query.data.items.map((item) => (
<li className={`attention-item is-${item.severity}`} key={item.id}><div className="attention-item-heading"><div><span className={`pill ${item.severity === 'critical' ? 'warning' : 'neutral'}`}>{item.kind.replaceAll('-', ' ')}</span><h2>{item.title}</h2></div><time dateTime={new Date(item.occurredAt).toISOString()}>{formatTime(item.occurredAt)}</time></div><p>{item.detail}</p>{item.owner ? <AccountOwnerBadge owner={item.owner} /> : null}{itemAction(item)}</li>
))}</ol>
)}
</div>
);
}
@@ -0,0 +1,8 @@
.attention-list { display: grid; gap: .8rem; padding: 0; margin: 0; list-style: none; }
.attention-item { display: grid; gap: .8rem; padding: 1rem; background: var(--surface); border: 1px solid var(--border); border-left: .3rem solid var(--warning); border-radius: .9rem; }
.attention-item.is-critical { border-left-color: var(--danger); }
.attention-item-heading { display: flex; align-items: flex-start; justify-content: space-between; gap: 1rem; }
.attention-item-heading > div { display: grid; gap: .55rem; }
.attention-item-heading time { color: var(--muted); font-size: .8rem; text-align: right; }
.attention-item p { margin: 0; color: var(--muted); }
.attention-item .button { justify-self: start; text-decoration: none; }
@@ -4,6 +4,7 @@ import {
SubredditPerformanceResponseSchema, SubredditPerformanceResponseSchema,
SuccessfulExamplesResponseSchema, SuccessfulExamplesResponseSchema,
FunnelResponseSchema, FunnelResponseSchema,
AiUsageSummaryResponseSchema,
} from '../../../../../packages/api-contracts/src/index.js'; } from '../../../../../packages/api-contracts/src/index.js';
import type { z } from 'zod'; import type { z } from 'zod';
import { createDashboardRequest } from '../../api/request.js'; import { createDashboardRequest } from '../../api/request.js';
@@ -13,6 +14,7 @@ type StrategyPerformanceResponse = z.infer<typeof StrategyPerformanceResponseSch
type DailyTrendResponse = z.infer<typeof DailyTrendResponseSchema>; type DailyTrendResponse = z.infer<typeof DailyTrendResponseSchema>;
type SuccessfulExamplesResponse = z.infer<typeof SuccessfulExamplesResponseSchema>; type SuccessfulExamplesResponse = z.infer<typeof SuccessfulExamplesResponseSchema>;
type FunnelResponse = z.infer<typeof FunnelResponseSchema>; type FunnelResponse = z.infer<typeof FunnelResponseSchema>;
type AiUsageSummaryResponse = z.infer<typeof AiUsageSummaryResponseSchema>;
async function parseResponse<T>( async function parseResponse<T>(
response: Response, response: Response,
@@ -38,6 +40,7 @@ export interface PerformanceApi {
getDailyTrend(days: number): Promise<DailyTrendResponse>; getDailyTrend(days: number): Promise<DailyTrendResponse>;
getSuccessfulExamples(): Promise<SuccessfulExamplesResponse>; getSuccessfulExamples(): Promise<SuccessfulExamplesResponse>;
getFunnel(): Promise<FunnelResponse>; getFunnel(): Promise<FunnelResponse>;
getAiUsage(days: number): Promise<AiUsageSummaryResponse>;
} }
export function createPerformanceApi( export function createPerformanceApi(
@@ -75,6 +78,12 @@ export function createPerformanceApi(
FunnelResponseSchema, FunnelResponseSchema,
); );
}, },
async getAiUsage(days: number) {
return parseResponse(
await request(`/api/analytics/ai-usage?days=${days}`),
AiUsageSummaryResponseSchema,
);
},
} satisfies PerformanceApi; } satisfies PerformanceApi;
} }
@@ -5,6 +5,7 @@ import type {
SubredditPerformanceResponseSchema, SubredditPerformanceResponseSchema,
SuccessfulExample, SuccessfulExample,
FunnelResponseSchema, FunnelResponseSchema,
AiUsageSummaryResponseSchema,
} from '../../../../../packages/api-contracts/src/index.js'; } from '../../../../../packages/api-contracts/src/index.js';
import { import {
useDailyTrendQuery, useDailyTrendQuery,
@@ -12,6 +13,7 @@ import {
useSubredditPerformanceQuery, useSubredditPerformanceQuery,
useSuccessfulExamplesQuery, useSuccessfulExamplesQuery,
useFunnelQuery, useFunnelQuery,
useAiUsageQuery,
} from './performance-queries.js'; } from './performance-queries.js';
import './performance.css'; import './performance.css';
@@ -20,6 +22,33 @@ type SubredditPerformance = z.infer<typeof SubredditPerformanceResponseSchema>[n
type StrategyPerformance = z.infer<typeof StrategyPerformanceResponseSchema>[number]; type StrategyPerformance = z.infer<typeof StrategyPerformanceResponseSchema>[number];
type DailyTrend = z.infer<typeof DailyTrendResponseSchema>[number]; type DailyTrend = z.infer<typeof DailyTrendResponseSchema>[number];
type Funnel = z.infer<typeof FunnelResponseSchema>; type Funnel = z.infer<typeof FunnelResponseSchema>;
type AiUsageSummary = z.infer<typeof AiUsageSummaryResponseSchema>;
function formatUsd(value: number): string {
return new Intl.NumberFormat(undefined, {
style: 'currency', currency: 'USD', minimumFractionDigits: 2, maximumFractionDigits: 4,
}).format(value);
}
function AiUsagePanel({ usage }: { usage: AiUsageSummary }) {
const successRate = usage.requests > 0 ? Math.round((usage.successes / usage.requests) * 100) : 0;
return (
<section aria-labelledby="ai-usage-heading" className="panel feature-panel">
<div className="panel-heading"><div><span className="eyebrow">OpenRouter · last {usage.days} days</span><h2 id="ai-usage-heading">AI cost and reliability</h2></div><span className="sample-size">{usage.requests} requests</span></div>
<div aria-label="AI usage totals" className="metrics-grid performance-metrics">
<article className="metric-card"><span>Reported cost</span><strong>{formatUsd(usage.costUsd)}</strong><small>OpenRouter usage accounting</small></article>
<article className="metric-card"><span>Tokens</span><strong>{usage.totalTokens.toLocaleString()}</strong><small>{usage.promptTokens.toLocaleString()} prompt · {usage.completionTokens.toLocaleString()} completion</small></article>
<article className="metric-card"><span>Success rate</span><strong>{successRate}%</strong><small>{usage.refusals} refusals · {usage.errors} errors/empty</small></article>
<article className="metric-card"><span>Average latency</span><strong>{usage.averageLatencyMs.toLocaleString()} ms</strong><small>All recorded attempts</small></article>
</div>
{usage.models.length === 0 ? <p className="feature-empty">No AI usage has been recorded yet. New requests will appear here.</p> : (
<div className="table-scroll"><table aria-label="AI usage by model" className="performance-table"><thead><tr><th scope="col">Model</th><th scope="col">Requests</th><th scope="col">Success</th><th scope="col">Refusals</th><th scope="col">Errors</th><th scope="col">Tokens</th><th scope="col">Cost</th><th scope="col">Latency</th></tr></thead><tbody>{usage.models.map((model) => (
<tr key={model.model}><th scope="row">{model.model}</th><td>{model.requests}</td><td>{model.successes}</td><td>{model.refusals}</td><td>{model.errors}</td><td>{model.totalTokens.toLocaleString()}</td><td>{formatUsd(model.costUsd)}</td><td>{model.averageLatencyMs.toLocaleString()} ms</td></tr>
))}</tbody></table></div>
)}
</section>
);
}
function SampleSize({ totalSent }: { totalSent: number }) { function SampleSize({ totalSent }: { totalSent: number }) {
const isLowConfidence = totalSent < LOW_CONFIDENCE_SAMPLE_SIZE; const isLowConfidence = totalSent < LOW_CONFIDENCE_SAMPLE_SIZE;
@@ -129,7 +158,8 @@ export function PerformancePage() {
const trend = useDailyTrendQuery(30); const trend = useDailyTrendQuery(30);
const examples = useSuccessfulExamplesQuery(); const examples = useSuccessfulExamplesQuery();
const funnel = useFunnelQuery(); const funnel = useFunnelQuery();
const queries = [subreddits, strategies, trend, examples, funnel]; const aiUsage = useAiUsageQuery(30);
const queries = [subreddits, strategies, trend, examples, funnel, aiUsage];
if (queries.some((query) => query.isPending)) { if (queries.some((query) => query.isPending)) {
return <div className="page-state" role="status">Loading performance…</div>; return <div className="page-state" role="status">Loading performance…</div>;
} }
@@ -142,6 +172,7 @@ export function PerformancePage() {
<div className="page performance-page"> <div className="page performance-page">
<header className="page-header"><div><span className="eyebrow">Decision support</span><h1>Performance</h1><p>Compare response and received-image outcomes without hiding the sample sizes behind the percentages.</p></div></header> <header className="page-header"><div><span className="eyebrow">Decision support</span><h1>Performance</h1><p>Compare response and received-image outcomes without hiding the sample sizes behind the percentages.</p></div></header>
{funnel.data ? <FunnelPanel funnel={funnel.data} /> : null} {funnel.data ? <FunnelPanel funnel={funnel.data} /> : null}
{aiUsage.data ? <AiUsagePanel usage={aiUsage.data} /> : null}
<section aria-labelledby="trend-heading" className="panel feature-panel"><div className="panel-heading"><div><span className="eyebrow">Last 30 days</span><h2 id="trend-heading">Response trend</h2></div></div><Trend rows={trend.data ?? []} /></section> <section aria-labelledby="trend-heading" className="panel feature-panel"><div className="panel-heading"><div><span className="eyebrow">Last 30 days</span><h2 id="trend-heading">Response trend</h2></div></div><Trend rows={trend.data ?? []} /></section>
<div className="performance-grid"> <div className="performance-grid">
<section aria-labelledby="subreddit-heading" className="panel feature-panel"><div className="panel-heading"><h2 id="subreddit-heading">By subreddit</h2></div><PerformanceTable label="Subreddit performance" rows={subreddits.data ?? []} rowLabel={(row) => 'subreddit' in row ? row.subreddit : row.strategyId} /></section> <section aria-labelledby="subreddit-heading" className="panel feature-panel"><div className="panel-heading"><h2 id="subreddit-heading">By subreddit</h2></div><PerformanceTable label="Subreddit performance" rows={subreddits.data ?? []} rowLabel={(row) => 'subreddit' in row ? row.subreddit : row.strategyId} /></section>
@@ -7,6 +7,7 @@ export const performanceQueryKeys = {
trend: (days: number) => ['performance', 'trend', days] as const, trend: (days: number) => ['performance', 'trend', days] as const,
successfulExamples: ['performance', 'successful-examples'] as const, successfulExamples: ['performance', 'successful-examples'] as const,
funnel: ['performance', 'funnel'] as const, funnel: ['performance', 'funnel'] as const,
aiUsage: (days: number) => ['performance', 'ai-usage', days] as const,
}; };
export function useSubredditPerformanceQuery() { export function useSubredditPerformanceQuery() {
@@ -43,3 +44,10 @@ export function useFunnelQuery() {
queryFn: performanceApi.getFunnel, queryFn: performanceApi.getFunnel,
}); });
} }
export function useAiUsageQuery(days: number) {
return useQuery({
queryKey: performanceQueryKeys.aiUsage(days),
queryFn: () => performanceApi.getAiUsage(days),
});
}
+2
View File
@@ -86,6 +86,8 @@ an administrator credential.
| `GET` | `/api/analytics/by-strategy` | Get performance by strategy | read | no | | `GET` | `/api/analytics/by-strategy` | Get performance by strategy | read | no |
| `GET` | `/api/analytics/funnel` | Get mature message outcome funnel | read | no | | `GET` | `/api/analytics/funnel` | Get mature message outcome funnel | read | no |
| `GET` | `/api/analytics/trend` | Get daily performance trend | read | no | | `GET` | `/api/analytics/trend` | Get daily performance trend | read | no |
| `GET` | `/api/analytics/ai-usage` | Get OpenRouter cost and model reliability | read | no |
| `GET` | `/api/attention` | Get operator attention items | read | no |
| `GET` | `/api/config` | Get non-secret application configuration | read | no | | `GET` | `/api/config` | Get non-secret application configuration | read | no |
| `PUT` | `/api/config` | Update application configuration | admin | yes | | `PUT` | `/api/config` | Update application configuration | admin | yes |
| `GET` | `/api/login/status` | Get Reddit session state | read | no | | `GET` | `/api/login/status` | Get Reddit session state | read | no |
+31
View File
@@ -0,0 +1,31 @@
import { z } from 'zod';
import { RedditAccountOwnerSchema } from './session.js';
export const AttentionKindSchema = z.enum([
'human-required',
'boundary',
'failed-delivery',
'account-login-required',
'runtime-failed',
'invitation-cooldown',
]);
export const AttentionItemSchema = z.strictObject({
id: z.string(),
kind: AttentionKindSchema,
severity: z.enum(['warning', 'critical']),
title: z.string(),
detail: z.string(),
occurredAt: z.number(),
owner: RedditAccountOwnerSchema.optional(),
conversationId: z.string().optional(),
username: z.string().optional(),
});
export const AttentionResponseSchema = z.strictObject({
generatedAt: z.number(),
items: z.array(AttentionItemSchema),
});
export type AttentionItem = z.infer<typeof AttentionItemSchema>;
export type AttentionResponse = z.infer<typeof AttentionResponseSchema>;
+1
View File
@@ -1,4 +1,5 @@
export * from './activity.js'; export * from './activity.js';
export * from './attention.js';
export * from './chats.js'; export * from './chats.js';
export * from './common.js'; export * from './common.js';
export * from './config.js'; export * from './config.js';
+19
View File
@@ -74,5 +74,24 @@ export const FunnelResponseSchema = z.strictObject({
images: z.number(), images: z.number(),
}); });
const AiUsageFields = {
requests: z.number().int().nonnegative(),
successes: z.number().int().nonnegative(),
refusals: z.number().int().nonnegative(),
errors: z.number().int().nonnegative(),
promptTokens: z.number().int().nonnegative(),
completionTokens: z.number().int().nonnegative(),
totalTokens: z.number().int().nonnegative(),
costUsd: z.number().nonnegative(),
averageLatencyMs: z.number().nonnegative(),
};
export const AiUsageSummaryResponseSchema = z.strictObject({
days: z.number().int().positive(),
...AiUsageFields,
models: z.array(z.strictObject({ model: z.string(), ...AiUsageFields })),
});
export type StatsResponse = z.infer<typeof StatsResponseSchema>; export type StatsResponse = z.infer<typeof StatsResponseSchema>;
export type TimersResponse = z.infer<typeof TimersResponseSchema>; export type TimersResponse = z.infer<typeof TimersResponseSchema>;
export type AiUsageSummaryResponse = z.infer<typeof AiUsageSummaryResponseSchema>;
+96 -22
View File
@@ -1,6 +1,7 @@
import OpenAI from 'openai'; import OpenAI from 'openai';
import type { Config } from '../config/index.js'; import type { Config } from '../config/index.js';
import type { Logger } from '../utils/logger.js'; import type { Logger } from '../utils/logger.js';
import type { AiUsageRecordInput, AiUsageOutcome } from '../database/ai-usage-repository.js';
import { import {
formatConversationAsUntrustedData, formatConversationAsUntrustedData,
isSafeConversationReply, isSafeConversationReply,
@@ -29,6 +30,8 @@ interface ProtectedTextPrompt {
user: string; user: string;
} }
export type AiUsageRecorder = (usage: AiUsageRecordInput) => void;
export class OpenRouterClient { export class OpenRouterClient {
private client: OpenAI; private client: OpenAI;
@@ -36,6 +39,7 @@ export class OpenRouterClient {
private config: Config, private config: Config,
private logger: Logger, private logger: Logger,
client?: OpenAI, client?: OpenAI,
private readonly usageRecorder?: AiUsageRecorder,
) { ) {
this.client = client ?? new OpenAI({ this.client = client ?? new OpenAI({
baseURL: 'https://openrouter.ai/api/v1', baseURL: 'https://openrouter.ai/api/v1',
@@ -48,6 +52,7 @@ export class OpenRouterClient {
} }
async detectGenderOnly(imageBase64: string, mimeType: string): Promise<DetectedGender> { async detectGenderOnly(imageBase64: string, mimeType: string): Promise<DetectedGender> {
const startedAt = Date.now();
try { try {
this.logger.debug('Quick gender detection...'); this.logger.debug('Quick gender detection...');
@@ -74,12 +79,14 @@ export class OpenRouterClient {
}); });
const result = response.choices[0]?.message?.content?.toLowerCase().trim() || 'unknown'; const result = response.choices[0]?.message?.content?.toLowerCase().trim() || 'unknown';
this.recordUsage('detectGender', this.config.openrouter.imageAnalysisModel, startedAt, response, result ? 'success' : 'empty');
if (result.includes('female')) return 'female'; if (result.includes('female')) return 'female';
if (result.includes('male') && !result.includes('female')) return 'male'; if (result.includes('male') && !result.includes('female')) return 'male';
if (result.includes('other')) return 'other'; if (result.includes('other')) return 'other';
return 'unknown'; return 'unknown';
} catch (error) { } catch (error) {
this.recordError('detectGender', this.config.openrouter.imageAnalysisModel, startedAt, error);
this.logger.error('Gender detection failed:', error); this.logger.error('Gender detection failed:', error);
return 'unknown'; return 'unknown';
} }
@@ -134,33 +141,40 @@ export class OpenRouterClient {
} }
private async tryAnalyzeWithModel(model: string, imageBase64: string, mimeType: string): Promise<ImageAnalysisResult | null> { private async tryAnalyzeWithModel(model: string, imageBase64: string, mimeType: string): Promise<ImageAnalysisResult | null> {
const startedAt = Date.now();
const genderPrompt = `First, identify the gender of the person in this image. Respond with EXACTLY one of these on the first line: GENDER:male, GENDER:female, GENDER:other, or GENDER:unknown const genderPrompt = `First, identify the gender of the person in this image. Respond with EXACTLY one of these on the first line: GENDER:male, GENDER:female, GENDER:other, or GENDER:unknown
Then provide your analysis: Then provide your analysis:
${this.config.openrouter.imageAnalysisPrompt}`; ${this.config.openrouter.imageAnalysisPrompt}`;
const response = await this.client.chat.completions.create({ let response;
model, try {
messages: [ response = await this.client.chat.completions.create({
{ model,
role: 'user', messages: [
content: [ {
{ role: 'user',
type: 'text', content: [
text: genderPrompt, {
}, type: 'text',
{ text: genderPrompt,
type: 'image_url',
image_url: {
url: `data:${mimeType};base64,${imageBase64}`,
}, },
}, {
], type: 'image_url',
}, image_url: {
], url: `data:${mimeType};base64,${imageBase64}`,
max_tokens: 600, },
}); },
],
},
],
max_tokens: 600,
});
} catch (error) {
this.recordError('analyzeImage', model, startedAt, error);
throw error;
}
const fullResponse = response.choices[0]?.message?.content || ''; const fullResponse = response.choices[0]?.message?.content || '';
@@ -190,9 +204,12 @@ ${this.config.openrouter.imageAnalysisPrompt}`;
// Detect AI refusal - return null to signal fallback should be tried // Detect AI refusal - return null to signal fallback should be tried
if (this.isRefusal(description)) { if (this.isRefusal(description)) {
this.recordUsage('analyzeImage', model, startedAt, response, 'refusal');
return null; return null;
} }
this.recordUsage('analyzeImage', model, startedAt, response, description ? 'success' : 'empty');
this.logger.debug(`Image analysis complete (${model}) - Gender: ${detectedGender}, Rating: ${attractivenessScore ?? 'n/a'}`); this.logger.debug(`Image analysis complete (${model}) - Gender: ${detectedGender}, Rating: ${attractivenessScore ?? 'n/a'}`);
this.logger.debug(`Description: ${description}`); this.logger.debug(`Description: ${description}`);
@@ -480,7 +497,10 @@ Just write the message, nothing else.`;
if (isFallback) { if (isFallback) {
this.logger.info(`[AI] ${label} trying fallback model: ${model}`); this.logger.info(`[AI] ${label} trying fallback model: ${model}`);
} }
const response = await this.client.chat.completions.create({ const startedAt = Date.now();
let response;
try {
response = await this.client.chat.completions.create({
model, model,
messages: typeof prompt === 'string' messages: typeof prompt === 'string'
? [{ role: 'user', content: prompt }] ? [{ role: 'user', content: prompt }]
@@ -490,7 +510,11 @@ Just write the message, nothing else.`;
], ],
max_tokens: opts.max_tokens, max_tokens: opts.max_tokens,
temperature: opts.temperature, temperature: opts.temperature,
}); });
} catch (error) {
this.recordError(label, model, startedAt, error);
throw error;
}
const raw = response.choices[0]?.message?.content || ''; const raw = response.choices[0]?.message?.content || '';
const choice = response.choices[0] as unknown as Record<string, unknown> | undefined; const choice = response.choices[0] as unknown as Record<string, unknown> | undefined;
@@ -508,14 +532,18 @@ Just write the message, nothing else.`;
const trimmed = raw.trim(); const trimmed = raw.trim();
if (!trimmed) { if (!trimmed) {
this.recordUsage(label, model, startedAt, response, 'empty');
this.logger.warn(`[AI] ${label}: ${model} returned empty content${i < models.length - 1 ? ', trying fallback...' : ''}`); this.logger.warn(`[AI] ${label}: ${model} returned empty content${i < models.length - 1 ? ', trying fallback...' : ''}`);
continue; continue;
} }
if (this.isRefusal(trimmed)) { if (this.isRefusal(trimmed)) {
this.recordUsage(label, model, startedAt, response, 'refusal');
this.logger.warn(`[AI] ${label}: ${model} returned a refusal${i < models.length - 1 ? ', trying fallback...' : ''}`); this.logger.warn(`[AI] ${label}: ${model} returned a refusal${i < models.length - 1 ? ', trying fallback...' : ''}`);
continue; continue;
} }
this.recordUsage(label, model, startedAt, response, 'success');
if (isFallback) { if (isFallback) {
this.logger.info(`[AI] ${label}: fallback model ${model} succeeded`); this.logger.info(`[AI] ${label}: fallback model ${model} succeeded`);
} }
@@ -532,6 +560,48 @@ Just write the message, nothing else.`;
return ''; return '';
} }
private recordUsage(
operation: string,
requestedModel: string,
startedAt: number,
response: unknown,
outcome: AiUsageOutcome,
): void {
if (!this.usageRecorder) return;
const record = response as Record<string, unknown>;
const usage = (record.usage ?? {}) as Record<string, unknown>;
try {
this.usageRecorder({
operation,
requestedModel,
actualModel: typeof record.model === 'string' ? record.model : requestedModel,
outcome,
promptTokens: numericUsage(usage.prompt_tokens),
completionTokens: numericUsage(usage.completion_tokens),
totalTokens: numericUsage(usage.total_tokens),
costUsd: typeof usage.cost === 'number' && Number.isFinite(usage.cost) ? usage.cost : null,
latencyMs: Date.now() - startedAt,
});
} catch (error) {
this.logger.warn('Failed to persist AI usage accounting:', error);
}
}
private recordError(operation: string, requestedModel: string, startedAt: number, error: unknown): void {
if (!this.usageRecorder) return;
try {
this.usageRecorder({
operation,
requestedModel,
outcome: 'error',
latencyMs: Date.now() - startedAt,
error: error instanceof Error ? error.message : String(error),
});
} catch (recordingError) {
this.logger.warn('Failed to persist AI usage error:', recordingError);
}
}
async analyzeAndGenerateMessage( async analyzeAndGenerateMessage(
imageBase64: string, imageBase64: string,
mimeType: string, mimeType: string,
@@ -547,3 +617,7 @@ Just write the message, nothing else.`;
return { analysis, message }; return { analysis, message };
} }
} }
function numericUsage(value: unknown): number {
return typeof value === 'number' && Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0;
}
+7 -1
View File
@@ -47,6 +47,7 @@ import type { Cookie } from 'playwright';
import { ChatCache } from './services/chat/cache.js'; import { ChatCache } from './services/chat/cache.js';
import { RedditAccountRuntimeManager } from './services/reddit-account-runtime.js'; import { RedditAccountRuntimeManager } from './services/reddit-account-runtime.js';
import { createConcreteRedditAccountRuntimeFactory } from './services/reddit-account-runtime-builder.js'; import { createConcreteRedditAccountRuntimeFactory } from './services/reddit-account-runtime-builder.js';
import { AiUsageRepository } from './database/ai-usage-repository.js';
import { importLegacyAccountContent } from './database/legacy-content-import.js'; import { importLegacyAccountContent } from './database/legacy-content-import.js';
import { RecipientDeduplicator } from './services/recipient-deduplicator.js'; import { RecipientDeduplicator } from './services/recipient-deduplicator.js';
import { createConfigBackedSubredditPollingControl } from './services/subreddit-polling-control.js'; import { createConfigBackedSubredditPollingControl } from './services/subreddit-polling-control.js';
@@ -127,6 +128,7 @@ export async function bootstrap(
const recipientDeduplicator = new RecipientDeduplicator(new RecipientHistoryRepository(db)); const recipientDeduplicator = new RecipientDeduplicator(new RecipientHistoryRepository(db));
const redditAccountsRepo = bootstrapAccountsRepo; const redditAccountsRepo = bootstrapAccountsRepo;
const accountContentRepo = new AccountContentRepository(db); const accountContentRepo = new AccountContentRepository(db);
const aiUsageRepo = new AiUsageRepository(db);
const legacyImport = importLegacyAccountContent(db); const legacyImport = importLegacyAccountContent(db);
if (legacyImport.conversations > 0 || legacyImport.media > 0) { if (legacyImport.conversations > 0 || legacyImport.media > 0) {
logger.info(`Imported legacy unassigned content into SQLite (${legacyImport.conversations} conversations, ${legacyImport.media} media items)`); logger.info(`Imported legacy unassigned content into SQLite (${legacyImport.conversations} conversations, ${legacyImport.media} media items)`);
@@ -168,7 +170,9 @@ export async function bootstrap(
const chatPage = await browserClient.getContext()!.newPage(); const chatPage = await browserClient.getContext()!.newPage();
// AI + services // AI + services
const aiClient = new OpenRouterClient(config, logger); const aiClient = new OpenRouterClient(config, logger, undefined, (usage) => {
aiUsageRepo.record({ ...usage, accountId: accountAuth.getSnapshot().accountId });
});
const monitorService = new MonitorService( const monitorService = new MonitorService(
redditApi, postsRepo, recipientDeduplicator, config, logger, 1000, redditApi, postsRepo, recipientDeduplicator, config, logger, 1000,
createConfigBackedSubredditPollingControl(config, new ConfigStore(db), logger), createConfigBackedSubredditPollingControl(config, new ConfigStore(db), logger),
@@ -344,6 +348,8 @@ export async function bootstrap(
activityLogRepo, activityLogRepo,
logger, logger,
broadcaster, broadcaster,
process.cwd(),
aiUsageRepo,
); );
webServer.setChatService(chatService); webServer.setChatService(chatService);
+130
View File
@@ -0,0 +1,130 @@
import type Database from 'better-sqlite3';
export type AiUsageOutcome = 'success' | 'refusal' | 'empty' | 'error';
export interface AiUsageRecordInput {
accountId?: string | null;
operation: string;
requestedModel: string;
actualModel?: string | null;
outcome: AiUsageOutcome;
promptTokens?: number;
completionTokens?: number;
totalTokens?: number;
costUsd?: number | null;
latencyMs: number;
error?: string | null;
createdAt?: number;
}
export interface AiUsageModelSummary {
model: string;
requests: number;
successes: number;
refusals: number;
errors: number;
promptTokens: number;
completionTokens: number;
totalTokens: number;
costUsd: number;
averageLatencyMs: number;
}
export interface AiUsageSummary {
days: number;
requests: number;
successes: number;
refusals: number;
errors: number;
promptTokens: number;
completionTokens: number;
totalTokens: number;
costUsd: number;
averageLatencyMs: number;
models: AiUsageModelSummary[];
}
type SummaryRow = {
model?: string;
requests: number;
successes: number;
refusals: number;
errors: number;
prompt_tokens: number;
completion_tokens: number;
total_tokens: number;
cost_usd: number;
average_latency_ms: number;
};
/** Durable, prompt-free accounting for OpenRouter requests. */
export class AiUsageRepository {
constructor(private readonly db: Database.Database) {}
record(input: AiUsageRecordInput): void {
this.db.prepare(`
INSERT INTO ai_usage (
account_id, operation, requested_model, actual_model, outcome,
prompt_tokens, completion_tokens, total_tokens, cost_usd,
latency_ms, error, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
input.accountId ?? null,
input.operation,
input.requestedModel,
input.actualModel ?? null,
input.outcome,
input.promptTokens ?? 0,
input.completionTokens ?? 0,
input.totalTokens ?? 0,
input.costUsd ?? null,
Math.max(0, Math.round(input.latencyMs)),
input.error?.slice(0, 500) ?? null,
input.createdAt ?? Date.now(),
);
}
getSummary(days = 30, now = Date.now()): AiUsageSummary {
const safeDays = Math.max(1, Math.min(365, Math.trunc(days)));
const since = now - safeDays * 86_400_000;
const fields = `
COUNT(*) AS requests,
SUM(CASE WHEN outcome = 'success' THEN 1 ELSE 0 END) AS successes,
SUM(CASE WHEN outcome = 'refusal' THEN 1 ELSE 0 END) AS refusals,
SUM(CASE WHEN outcome IN ('empty', 'error') THEN 1 ELSE 0 END) AS errors,
COALESCE(SUM(prompt_tokens), 0) AS prompt_tokens,
COALESCE(SUM(completion_tokens), 0) AS completion_tokens,
COALESCE(SUM(total_tokens), 0) AS total_tokens,
COALESCE(SUM(cost_usd), 0) AS cost_usd,
COALESCE(ROUND(AVG(latency_ms)), 0) AS average_latency_ms
`;
const total = this.db.prepare(`SELECT ${fields} FROM ai_usage WHERE created_at >= ?`)
.get(since) as SummaryRow;
const models = this.db.prepare(`
SELECT COALESCE(actual_model, requested_model) AS model, ${fields}
FROM ai_usage
WHERE created_at >= ?
GROUP BY COALESCE(actual_model, requested_model)
ORDER BY cost_usd DESC, requests DESC, model ASC
`).all(since) as SummaryRow[];
return {
days: safeDays,
...mapSummary(total),
models: models.map((row) => ({ model: row.model ?? 'unknown', ...mapSummary(row) })),
};
}
}
function mapSummary(row: SummaryRow): Omit<AiUsageModelSummary, 'model'> {
return {
requests: Number(row.requests ?? 0),
successes: Number(row.successes ?? 0),
refusals: Number(row.refusals ?? 0),
errors: Number(row.errors ?? 0),
promptTokens: Number(row.prompt_tokens ?? 0),
completionTokens: Number(row.completion_tokens ?? 0),
totalTokens: Number(row.total_tokens ?? 0),
costUsd: Math.round(Number(row.cost_usd ?? 0) * 1_000_000_000_000) / 1_000_000_000_000,
averageLatencyMs: Number(row.average_latency_ms ?? 0),
};
}
+21
View File
@@ -251,6 +251,27 @@ export function runMigrations(db: Database.Database): void {
CREATE INDEX idx_conversation_automation_claims_status CREATE INDEX idx_conversation_automation_claims_status
ON conversation_automation_claims(account_id, status, claimed_at); ON conversation_automation_claims(account_id, status, claimed_at);
`), `),
() => db.exec(`
CREATE TABLE IF NOT EXISTS ai_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT,
account_id TEXT,
operation TEXT NOT NULL,
requested_model TEXT NOT NULL,
actual_model TEXT,
outcome TEXT NOT NULL CHECK(outcome IN ('success', 'refusal', 'empty', 'error')),
prompt_tokens INTEGER NOT NULL DEFAULT 0,
completion_tokens INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
cost_usd REAL,
latency_ms INTEGER NOT NULL,
error TEXT,
created_at INTEGER NOT NULL,
FOREIGN KEY (account_id) REFERENCES reddit_accounts(account_id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS idx_ai_usage_created_at ON ai_usage(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_ai_usage_model ON ai_usage(actual_model, requested_model, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_ai_usage_account ON ai_usage(account_id, created_at DESC);
`),
]; ];
let currentVersion = db.pragma('user_version', { simple: true }) as number; let currentVersion = db.pragma('user_version', { simple: true }) as number;
+36
View File
@@ -227,6 +227,16 @@ export class DeliveryOutboxRepository {
.get(this.accountId) as { count: number }).count; .get(this.accountId) as { count: number }).count;
} }
getFailed(limit = 50): DeliveryOutboxRecord[] {
const rows = this.db.prepare(`
SELECT * FROM delivery_outbox
WHERE account_id = ? AND status = 'failed'
ORDER BY updated_at DESC
LIMIT ?
`).all(this.accountId, limit) as Record<string, unknown>[];
return rows.map((row) => this.mapRow(row));
}
markPrepared(id: number, now: number = Date.now()): boolean { markPrepared(id: number, now: number = Date.now()): boolean {
return this.db.prepare(` return this.db.prepare(`
UPDATE delivery_outbox SET status = 'prepared', updated_at = ?, last_error = NULL UPDATE delivery_outbox SET status = 'prepared', updated_at = ?, last_error = NULL
@@ -1081,6 +1091,32 @@ export class ConversationStateRepository {
const row = this.getStmt.get(this.accountId, conversationId) as { state_json: string } | undefined; const row = this.getStmt.get(this.accountId, conversationId) as { state_json: string } | undefined;
return row ? parseConversationState(row.state_json) ?? undefined : undefined; return row ? parseConversationState(row.state_json) ?? undefined : undefined;
} }
listAttention(): Array<{
conversationId: string;
status: 'human-required' | 'boundary';
reason: string | null;
updatedAt: number;
}> {
const rows = this.db.prepare(`
SELECT conversation_id, state_json, updated_at
FROM conversation_states
WHERE account_id = ?
ORDER BY updated_at DESC
`).all(this.accountId) as Array<{ conversation_id: string; state_json: string; updated_at: number }>;
return rows.flatMap((row) => {
const state = parseConversationState(row.state_json);
if (!state) return [];
const status = state.automation.status;
if (status !== 'human-required' && status !== 'boundary') return [];
return [{
conversationId: row.conversation_id,
status,
reason: state.automation.handoffReason,
updatedAt: row.updated_at,
}];
});
}
} }
/** Durable idempotency claims for AI replies to individual inbound events. */ /** Durable idempotency claims for AI replies to individual inbound events. */
@@ -43,6 +43,7 @@ import { RecipientDeduplicator } from './recipient-deduplicator.js';
import { ConfigStore } from '../config/persistence.js'; import { ConfigStore } from '../config/persistence.js';
import { createConfigBackedSubredditPollingControl } from './subreddit-polling-control.js'; import { createConfigBackedSubredditPollingControl } from './subreddit-polling-control.js';
import { ConversationAutomationService } from './conversation-automation.js'; import { ConversationAutomationService } from './conversation-automation.js';
import { AiUsageRepository } from '../database/ai-usage-repository.js';
export interface ConcreteRuntimeFactoryOptions { export interface ConcreteRuntimeFactoryOptions {
config: Config; config: Config;
@@ -118,7 +119,10 @@ async function buildRuntime(
const redditApi = new RedditApiClient(accountConfig, logger, redditRequestCoordinator); const redditApi = new RedditApiClient(accountConfig, logger, redditRequestCoordinator);
redditApi.setBrowserContext(browserClient.getContext()!); redditApi.setBrowserContext(browserClient.getContext()!);
const chatPage = await browserClient.getContext()!.newPage(); const chatPage = await browserClient.getContext()!.newPage();
const aiClient = new OpenRouterClient(accountConfig, logger); const aiUsageRepo = new AiUsageRepository(db);
const aiClient = new OpenRouterClient(accountConfig, logger, undefined, (usage) => {
aiUsageRepo.record({ ...usage, accountId: session.accountId });
});
const monitorService = new MonitorService( const monitorService = new MonitorService(
redditApi, postsRepo, recipientDeduplicator, accountConfig, logger, redditApi, postsRepo, recipientDeduplicator, accountConfig, logger,
1000, createConfigBackedSubredditPollingControl(config, new ConfigStore(db), logger), 1000, createConfigBackedSubredditPollingControl(config, new ConfigStore(db), logger),
+2
View File
@@ -94,6 +94,8 @@ export const API_OPERATIONS: readonly ApiOperation[] = [
operation('getStrategyAnalytics', 'GET', '/api/analytics/by-strategy', 'Get performance by strategy', ['Analytics']), operation('getStrategyAnalytics', 'GET', '/api/analytics/by-strategy', 'Get performance by strategy', ['Analytics']),
operation('getFunnelAnalytics', 'GET', '/api/analytics/funnel', 'Get mature message outcome funnel', ['Analytics']), operation('getFunnelAnalytics', 'GET', '/api/analytics/funnel', 'Get mature message outcome funnel', ['Analytics']),
operation('getAnalyticsTrend', 'GET', '/api/analytics/trend', 'Get daily performance trend', ['Analytics']), operation('getAnalyticsTrend', 'GET', '/api/analytics/trend', 'Get daily performance trend', ['Analytics']),
operation('getAiUsageAnalytics', 'GET', '/api/analytics/ai-usage', 'Get OpenRouter cost and model reliability', ['Analytics']),
operation('getAttention', 'GET', '/api/attention', 'Get operator attention items', ['Operations']),
operation('getConfig', 'GET', '/api/config', 'Get non-secret application configuration', ['Configuration']), operation('getConfig', 'GET', '/api/config', 'Get non-secret application configuration', ['Configuration']),
operation('updateConfig', 'PUT', '/api/config', 'Update application configuration', ['Configuration']), operation('updateConfig', 'PUT', '/api/config', 'Update application configuration', ['Configuration']),
operation('getLoginStatus', 'GET', '/api/login/status', 'Get Reddit session state', ['Accounts']), operation('getLoginStatus', 'GET', '/api/login/status', 'Get Reddit session state', ['Accounts']),
+2
View File
@@ -22,6 +22,7 @@ import type { AccountAuthContext } from '../services/account-auth-context.js';
import type { ConfigStore } from '../config/persistence.js'; import type { ConfigStore } from '../config/persistence.js';
import type { RedditAccountsService } from '../services/reddit-accounts.js'; import type { RedditAccountsService } from '../services/reddit-accounts.js';
import type { RedditAccountRuntimeManager } from '../services/reddit-account-runtime.js'; import type { RedditAccountRuntimeManager } from '../services/reddit-account-runtime.js';
import type { AiUsageRepository } from '../database/ai-usage-repository.js';
/** Bag of everything the route modules might need. */ /** Bag of everything the route modules might need. */
export interface WebContext { export interface WebContext {
@@ -35,6 +36,7 @@ export interface WebContext {
posts: PostsRepository; posts: PostsRepository;
messages: MessagesRepository; messages: MessagesRepository;
successful: SuccessfulMessagesRepository; successful: SuccessfulMessagesRepository;
aiUsage?: AiUsageRepository;
}; };
services: { services: {
+9
View File
@@ -37,5 +37,14 @@ export function createAnalyticsRouter(ctx: WebContext): Router {
res.json(ctx.repos.messages.getResponseRateTrend(days)); res.json(ctx.repos.messages.getResponseRateTrend(days));
}); });
router.get('/analytics/ai-usage', (req, res) => {
if (!ctx.repos.aiUsage) {
res.status(503).json({ error: 'AI usage accounting is not initialized' });
return;
}
const days = parseInt(req.query.days as string) || 30;
res.json(ctx.repos.aiUsage.getSummary(days));
});
return router; return router;
} }
+103
View File
@@ -0,0 +1,103 @@
import { Router } from 'express';
import type { WebContext } from '../context.js';
interface AttentionItem {
id: string;
kind: 'human-required' | 'boundary' | 'failed-delivery' | 'account-login-required' | 'runtime-failed' | 'invitation-cooldown';
severity: 'warning' | 'critical';
title: string;
detail: string;
occurredAt: number;
owner?: { accountId: string; redditUsername: string };
conversationId?: string;
username?: string;
}
export function createAttentionRouter(ctx: WebContext): Router {
const router = Router();
router.get('/attention', async (_req, res) => {
const items: AttentionItem[] = [];
const accounts = ctx.services.redditAccounts?.list() ?? [];
for (const account of accounts) {
if (account.status !== 'login-required') continue;
items.push({
id: `account:${account.accountId}`,
kind: 'account-login-required',
severity: 'critical',
title: `u/${account.redditUsername} needs a new session`,
detail: 'Import a fresh extension session before this account can resume.',
occurredAt: account.updatedAt,
owner: { accountId: account.accountId, redditUsername: account.redditUsername },
});
}
const snapshots = ctx.services.accountRuntimes?.list() ?? [];
for (const snapshot of snapshots) {
const owner = { accountId: snapshot.accountId, redditUsername: snapshot.redditUsername };
if (snapshot.state === 'failed') {
items.push({
id: `runtime:${snapshot.accountId}`,
kind: 'runtime-failed',
severity: 'critical',
title: `u/${snapshot.redditUsername} runtime failed`,
detail: snapshot.failure ?? 'The account runtime stopped unexpectedly.',
occurredAt: snapshot.updatedAt,
owner,
});
}
const runtime = ctx.services.accountRuntimes?.get(snapshot.accountId);
if (!runtime) continue;
const conversations = await runtime.resources.chatService.getCachedConversations();
const usernames = new Map(conversations.flatMap((conversation) => conversation.roomId
? [[conversation.roomId, conversation.username] as const]
: []));
for (const state of runtime.resources.conversationStateRepo.listAttention()) {
const username = usernames.get(state.conversationId);
items.push({
id: `conversation:${snapshot.accountId}:${state.conversationId}:${state.status}`,
kind: state.status,
severity: state.status === 'boundary' ? 'critical' : 'warning',
title: state.status === 'boundary'
? `Boundary detected${username ? ` from u/${username}` : ''}`
: `Human follow-up required${username ? ` for u/${username}` : ''}`,
detail: state.reason ?? (state.status === 'boundary'
? 'Automation stopped after the recipient set a boundary.'
: 'Automated conversation turns are exhausted.'),
occurredAt: state.updatedAt,
owner,
conversationId: `${snapshot.accountId}~${state.conversationId}`,
...(username ? { username } : {}),
});
}
for (const delivery of runtime.resources.deliveryOutboxRepo.getFailed()) {
items.push({
id: `delivery:${snapshot.accountId}:${delivery.id}`,
kind: 'failed-delivery',
severity: 'warning',
title: `Message to u/${delivery.recipient} failed`,
detail: delivery.lastError ?? `Delivery failed after ${delivery.attemptCount} attempt(s).`,
occurredAt: delivery.updatedAt,
owner,
username: delivery.recipient,
});
}
const cooldown = runtime.resources.chatService.getInvitationCooldown();
if (cooldown) {
items.push({
id: `cooldown:${snapshot.accountId}:${cooldown.until}`,
kind: 'invitation-cooldown',
severity: 'warning',
title: `u/${snapshot.redditUsername} invitation cooldown`,
detail: cooldown.reason,
occurredAt: Date.now(),
owner,
});
}
}
res.json({ generatedAt: Date.now(), items: items.sort((left, right) => right.occurredAt - left.occurredAt) });
});
return router;
}
+5
View File
@@ -41,6 +41,8 @@ import { createLoginRouter } from './routes/login.js';
import { createDashboardAuth, createDashboardSessionRouter } from './auth.js'; import { createDashboardAuth, createDashboardSessionRouter } from './auth.js';
import { resolveDashboardAssets } from './dashboard-assets.js'; import { resolveDashboardAssets } from './dashboard-assets.js';
import { createAgentRouter, createHealthRouter } from './routes/agent.js'; import { createAgentRouter, createHealthRouter } from './routes/agent.js';
import { createAttentionRouter } from './routes/attention.js';
import type { AiUsageRepository } from '../database/ai-usage-repository.js';
export type { BotStats, ActivityItem, LogEntry, RuntimeConfig } from './types.js'; export type { BotStats, ActivityItem, LogEntry, RuntimeConfig } from './types.js';
@@ -74,6 +76,7 @@ export class WebServer {
private logger: Logger, private logger: Logger,
broadcaster?: LogBroadcaster, broadcaster?: LogBroadcaster,
private workspaceDirectory = process.cwd(), private workspaceDirectory = process.cwd(),
private aiUsageRepo?: AiUsageRepository,
) { ) {
this.broadcaster = broadcaster ?? new LogBroadcaster(); this.broadcaster = broadcaster ?? new LogBroadcaster();
this.activity = new ActivityLog(activityLogRepo); this.activity = new ActivityLog(activityLogRepo);
@@ -104,6 +107,7 @@ export class WebServer {
posts: this.postsRepo, posts: this.postsRepo,
messages: this.messagesRepo, messages: this.messagesRepo,
successful: this.successfulMessagesRepo, successful: this.successfulMessagesRepo,
aiUsage: this.aiUsageRepo,
}, },
services: this.services, services: this.services,
getBotRunning: () => this.botRunning, getBotRunning: () => this.botRunning,
@@ -147,6 +151,7 @@ export class WebServer {
const ctx = this.context(); const ctx = this.context();
this.app.use('/api', createAgentRouter(ctx)); this.app.use('/api', createAgentRouter(ctx));
this.app.use('/api', createStatsRouter(ctx)); this.app.use('/api', createStatsRouter(ctx));
this.app.use('/api', createAttentionRouter(ctx));
this.app.use('/api', createChatsRouter(ctx)); this.app.use('/api', createChatsRouter(ctx));
this.app.use('/api', createConversationsRouter(ctx)); this.app.use('/api', createConversationsRouter(ctx));
this.app.use('/api', createMediaRouter(ctx)); this.app.use('/api', createMediaRouter(ctx));
+32
View File
@@ -0,0 +1,32 @@
import { describe, expect, it } from 'vitest';
import { AiUsageRepository } from '../src/database/ai-usage-repository.js';
import { createTestDb } from './testUtils.js';
describe('AiUsageRepository', () => {
it('aggregates cost, tokens, outcomes, and latency by actual model', () => {
const db = createTestDb();
const repo = new AiUsageRepository(db);
const now = Date.parse('2026-08-09T10:00:00Z');
repo.record({ operation: 'generateReply', requestedModel: 'router', actualModel: 'model-a', outcome: 'success', promptTokens: 100, completionTokens: 20, totalTokens: 120, costUsd: 0.0012, latencyMs: 400, createdAt: now });
repo.record({ operation: 'generateReply', requestedModel: 'model-a', actualModel: 'model-a', outcome: 'refusal', promptTokens: 40, completionTokens: 5, totalTokens: 45, costUsd: 0.0003, latencyMs: 200, createdAt: now });
repo.record({ operation: 'analyzeImage', requestedModel: 'model-b', outcome: 'error', latencyMs: 900, error: 'provider unavailable', createdAt: now });
expect(repo.getSummary(30, now)).toEqual({
days: 30,
requests: 3,
successes: 1,
refusals: 1,
errors: 1,
promptTokens: 140,
completionTokens: 25,
totalTokens: 165,
costUsd: 0.0015,
averageLatencyMs: 500,
models: [
{ model: 'model-a', requests: 2, successes: 1, refusals: 1, errors: 0, promptTokens: 140, completionTokens: 25, totalTokens: 165, costUsd: 0.0015, averageLatencyMs: 300 },
{ model: 'model-b', requests: 1, successes: 0, refusals: 0, errors: 1, promptTokens: 0, completionTokens: 0, totalTokens: 0, costUsd: 0, averageLatencyMs: 900 },
],
});
db.close();
});
});
+19 -1
View File
@@ -79,7 +79,7 @@ describe('dashboard foundation', () => {
const { router } = renderDashboard('/'); const { router } = renderDashboard('/');
const navigation = screen.getByRole('navigation', { name: 'Primary navigation' }); const navigation = screen.getByRole('navigation', { name: 'Primary navigation' });
for (const label of ['Overview', 'Conversations', 'Media', 'Performance', 'Operations', 'Settings']) { for (const label of ['Overview', 'Needs attention', 'Conversations', 'Media', 'Performance', 'Operations', 'Settings']) {
expect(within(navigation).getByRole('link', { name: label })).toBeInTheDocument(); expect(within(navigation).getByRole('link', { name: label })).toBeInTheDocument();
} }
@@ -91,6 +91,24 @@ describe('dashboard foundation', () => {
expect(screen.getByRole('heading', { name: 'Operations' })).toBeInTheDocument(); expect(screen.getByRole('heading', { name: 'Operations' })).toBeInTheDocument();
}); });
it('shows actionable attention items and links handoffs to their conversation', async () => {
installSuccessfulApi();
server.use(http.get('/api/attention', () => HttpResponse.json({
generatedAt: Date.parse('2026-08-09T08:00:00Z'),
items: [{
id: 'conversation:one', kind: 'human-required', severity: 'warning',
title: 'Human follow-up required for u/alice', detail: 'Automated turns are exhausted.',
occurredAt: Date.parse('2026-08-09T07:00:00Z'),
owner: { accountId: '@one:reddit.com', redditUsername: 'operator' },
conversationId: '@one:reddit.com~!room:reddit.com', username: 'alice',
}],
})));
renderDashboard('/attention');
expect(await screen.findByRole('heading', { name: 'Needs attention' })).toBeInTheDocument();
expect(screen.getByText('Human follow-up required for u/alice')).toBeInTheDocument();
expect(screen.getByRole('link', { name: 'Open conversation' })).toHaveAttribute('href', '/conversations/%40one%3Areddit.com~!room%3Areddit.com');
});
it('hydrates media filters from the routed URL', async () => { it('hydrates media filters from the routed URL', async () => {
installSuccessfulApi(); installSuccessfulApi();
server.use( server.use(
@@ -83,12 +83,24 @@ describe('PerformancePage', () => {
upsell: 3, upsell: 3,
images: 2, images: 2,
})), })),
http.get('/api/analytics/ai-usage', ({ request }) => {
expect(new URL(request.url).searchParams.get('days')).toBe('30');
return HttpResponse.json({
days: 30, requests: 12, successes: 9, refusals: 2, errors: 1,
promptTokens: 1200, completionTokens: 300, totalTokens: 1500,
costUsd: 0.0123, averageLatencyMs: 420,
models: [{ model: 'model-a', requests: 12, successes: 9, refusals: 2, errors: 1, promptTokens: 1200, completionTokens: 300, totalTokens: 1500, costUsd: 0.0123, averageLatencyMs: 420 }],
});
}),
); );
renderPage(<PerformancePage />); renderPage(<PerformancePage />);
expect(await screen.findByRole('heading', { name: 'Performance' })).toBeInTheDocument(); expect(await screen.findByRole('heading', { name: 'Performance' })).toBeInTheDocument();
expect(screen.getByRole('heading', { name: 'Outcome funnel' })).toBeInTheDocument(); expect(screen.getByRole('heading', { name: 'Outcome funnel' })).toBeInTheDocument();
expect(screen.getByRole('heading', { name: 'AI cost and reliability' })).toBeInTheDocument();
expect(screen.getByRole('table', { name: 'AI usage by model' })).toHaveTextContent('model-a');
expect(screen.getByLabelText('AI usage totals')).toHaveTextContent('$0.0123');
expect(screen.getByText('5 awaiting maturity')).toBeInTheDocument(); expect(screen.getByText('5 awaiting maturity')).toBeInTheDocument();
const subredditTable = screen.getByRole('table', { name: 'Subreddit performance' }); const subredditTable = screen.getByRole('table', { name: 'Subreddit performance' });
expect(within(subredditTable).getByText('n = 40')).toBeInTheDocument(); expect(within(subredditTable).getByText('n = 40')).toBeInTheDocument();
+3 -2
View File
@@ -107,7 +107,7 @@ describe('Database Client', () => {
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'") .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
.all() as { name: string }[]; .all() as { name: string }[];
// Existing domain/config tables plus account, conversation, sync and media ownership stores. // Existing domain/config tables plus account, conversation, sync and media ownership stores.
expect(tables.length).toBe(15); expect(tables.length).toBe(16);
}); });
it('tracks schema version and migrates a legacy sent_messages table transactionally', () => { it('tracks schema version and migrates a legacy sent_messages table transactionally', () => {
@@ -154,8 +154,9 @@ describe('Database Client', () => {
db.exec(` db.exec(`
DELETE FROM recipient_suppressions; DELETE FROM recipient_suppressions;
DROP TABLE conversation_automation_claims; DROP TABLE conversation_automation_claims;
DROP TABLE ai_usage;
`); `);
db.pragma(`user_version = ${latestVersion - 1}`); db.pragma(`user_version = ${latestVersion - 2}`);
runMigrations(db); runMigrations(db);
expect(db.prepare('SELECT recipient, reason FROM recipient_suppressions ORDER BY recipient COLLATE NOCASE').all()) expect(db.prepare('SELECT recipient, reason FROM recipient_suppressions ORDER BY recipient COLLATE NOCASE').all())
+30
View File
@@ -703,6 +703,36 @@ describe('OpenRouterClient', () => {
).rejects.toThrow('network'); ).rejects.toThrow('network');
}); });
}); });
it('records model, token, cost, latency, and outcome for each request attempt', async () => {
const usageRecorder = vi.fn();
const config = getTestConfig();
const localClient = new OpenRouterClient(
config,
logger,
{ chat: { completions: { create: mockCreate } } } as unknown as OpenAI,
usageRecorder,
);
mockCreate.mockResolvedValueOnce({
model: 'provider/model-a',
choices: [{ message: { content: 'A useful reply' } }],
usage: { prompt_tokens: 100, completion_tokens: 12, total_tokens: 112, cost: 0.0017 },
});
await localClient.generateReply([{ sender: 'them', text: 'hello?', isSelf: false }], 'them');
expect(usageRecorder).toHaveBeenCalledWith(expect.objectContaining({
operation: 'generateReply',
requestedModel: config.openrouter.chatModel,
actualModel: 'provider/model-a',
outcome: 'success',
promptTokens: 100,
completionTokens: 12,
totalTokens: 112,
costUsd: 0.0017,
latencyMs: expect.any(Number),
}));
});
}); });
describe('stripDashes', () => { describe('stripDashes', () => {
+13
View File
@@ -70,4 +70,17 @@ describe('DeliveryOutboxRepository', () => {
expect(outbox.markSent(delivery.id, 'worker-a', sentMessageId, 130)).toBe(true); expect(outbox.markSent(delivery.id, 'worker-a', sentMessageId, 130)).toBe(true);
expect(outbox.get(delivery.id)).toMatchObject({ status: 'sent', sentMessageId, leaseOwner: null }); expect(outbox.get(delivery.id)).toMatchObject({ status: 'sent', sentMessageId, leaseOwner: null });
}); });
it('surfaces failed deliveries for the attention inbox', () => {
const delivery = outbox.discover({
postId: 'post-1', recipient: 'recipient', subject: 'subject', message: 'message', idempotencyKey: 'post-1:v1', now: 100,
});
outbox.markPrepared(delivery.id, 110);
outbox.claim(delivery.id, 'worker-a', 50, 120);
outbox.markFailed(delivery.id, 'worker-a', 'Reddit rejected the invitation', 130);
expect(outbox.getFailed()).toEqual([
expect.objectContaining({ id: delivery.id, status: 'failed', lastError: 'Reddit rejected the invitation' }),
]);
});
}); });
+15
View File
@@ -607,6 +607,21 @@ describe('ConversationStateRepository', () => {
expect(repo.get('!room:reddit.com')?.persona.age).toBe(34); expect(repo.get('!room:reddit.com')?.persona.age).toBe(34);
expect(repo.get('missing')).toBeUndefined(); expect(repo.get('missing')).toBeUndefined();
}); });
it('lists only durable boundary and human-handoff states for operator attention', () => {
const db = createTestDb();
const repo = new ConversationStateRepository(db);
const active = buildConversationState([]);
const boundary = buildConversationState([{ sender: 'them', text: 'Stop messaging me.', isSelf: false }]);
boundary.automation.status = 'boundary';
boundary.automation.handoffReason = 'Recipient asked us to stop.';
repo.save('active-room', active);
repo.save('boundary-room', boundary);
expect(repo.listAttention()).toEqual([
expect.objectContaining({ conversationId: 'boundary-room', status: 'boundary', reason: 'Recipient asked us to stop.' }),
]);
});
}); });
describe('Mature performance and guardrails', () => { describe('Mature performance and guardrails', () => {