Add attention inbox and AI usage observability
This commit is contained in:
1 parent
ca59ba0f1f
commit
83b5f1f5b6
30 files changed
+702
-28
No files matched your search
@@ -6,6 +6,7 @@ export interface NavigationItem {
|
||||
|
||||
export const navigationItems: readonly NavigationItem[] = [
|
||||
{ label: 'Overview', path: '/', description: 'System health and current activity' },
|
||||
{ label: 'Needs attention', path: '/attention', description: 'Handoffs, failures, and account risks' },
|
||||
{ label: 'Conversations', path: '/conversations', description: 'Inbox and message history' },
|
||||
{ label: 'Media', path: '/media', description: 'Private media and sync controls' },
|
||||
{ label: 'Performance', path: '/performance', description: 'Response and strategy trends' },
|
||||
|
||||
@@ -7,6 +7,7 @@ import { OperationsPage } from '../features/operations/operations-page.js';
|
||||
import { PerformancePage } from '../features/performance/performance-page.js';
|
||||
import { SettingsRoute } from '../features/settings/settings-route.js';
|
||||
import { RouteErrorPage } from './route-error-page.js';
|
||||
import { AttentionPage } from '../features/attention/attention-page.js';
|
||||
|
||||
export function createDashboardRoutes(): RouteObject[] {
|
||||
return [
|
||||
@@ -16,6 +17,7 @@ export function createDashboardRoutes(): RouteObject[] {
|
||||
errorElement: <RouteErrorPage />,
|
||||
children: [
|
||||
{ index: true, element: <OverviewPage /> },
|
||||
{ path: 'attention', element: <AttentionPage /> },
|
||||
{ path: 'conversations', element: <ConversationRoute /> },
|
||||
{ path: 'conversations/:conversationId', element: <ConversationRoute /> },
|
||||
{ path: 'media', element: <MediaRoute /> },
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { AttentionResponseSchema } from '../../../../../packages/api-contracts/src/index.js';
|
||||
import { createDashboardRequest } from '../../api/request.js';
|
||||
|
||||
export function createAttentionApi(fetchImplementation?: typeof fetch) {
|
||||
const request = createDashboardRequest(fetchImplementation);
|
||||
return {
|
||||
async getAttention() {
|
||||
const response = await request('/api/attention');
|
||||
const payload: unknown = await response.json();
|
||||
if (!response.ok) throw new Error(`Attention inbox unavailable (${response.status})`);
|
||||
return AttentionResponseSchema.parse(payload);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const attentionApi = createAttentionApi();
|
||||
@@ -0,0 +1,33 @@
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { Link } from 'react-router-dom';
|
||||
import type { AttentionItem } from '../../../../../packages/api-contracts/src/index.js';
|
||||
import { AccountOwnerBadge } from '../../components/account-owner-badge.js';
|
||||
import { attentionApi } from './attention-api.js';
|
||||
import './attention.css';
|
||||
|
||||
function formatTime(timestamp: number): string {
|
||||
return new Intl.DateTimeFormat(undefined, { dateStyle: 'medium', timeStyle: 'short' }).format(timestamp);
|
||||
}
|
||||
|
||||
function itemAction(item: AttentionItem) {
|
||||
if (!item.conversationId) return null;
|
||||
return <Link className="button secondary" to={`/conversations/${encodeURIComponent(item.conversationId)}`}>Open conversation</Link>;
|
||||
}
|
||||
|
||||
export function AttentionPage() {
|
||||
const query = useQuery({ queryKey: ['attention'], queryFn: attentionApi.getAttention, refetchInterval: 30_000 });
|
||||
if (query.isPending) return <div className="page-state" role="status">Loading attention inbox…</div>;
|
||||
if (query.isError) return <div className="page-state error-state" role="alert"><strong>Attention inbox unavailable</strong><span>{query.error.message}</span><button className="button secondary" onClick={() => void query.refetch()} type="button">Retry</button></div>;
|
||||
const critical = query.data.items.filter((item) => item.severity === 'critical').length;
|
||||
return (
|
||||
<div className="page attention-page">
|
||||
<header className="page-header"><div><span className="eyebrow">Operator queue</span><h1>Needs attention</h1><p>Boundaries, human handoffs, delivery failures, account problems, and Reddit cooldowns in one actionable inbox.</p></div><button className="button secondary" onClick={() => void query.refetch()} type="button">Refresh</button></header>
|
||||
<section aria-label="Attention totals" className="status-strip"><div><span className={`status-dot ${query.data.items.length === 0 ? 'running' : 'stopped'}`} aria-hidden="true" /><div><small>Open items</small><strong>{query.data.items.length}</strong></div></div><span className={critical > 0 ? 'pill warning' : 'pill success'}>{critical} critical</span><span className="status-note">Updated {formatTime(query.data.generatedAt)}</span></section>
|
||||
{query.data.items.length === 0 ? <div className="empty-state panel"><strong>Nothing needs attention.</strong><span>Automation and account health have no outstanding operator items.</span></div> : (
|
||||
<ol aria-label="Attention items" className="attention-list">{query.data.items.map((item) => (
|
||||
<li className={`attention-item is-${item.severity}`} key={item.id}><div className="attention-item-heading"><div><span className={`pill ${item.severity === 'critical' ? 'warning' : 'neutral'}`}>{item.kind.replaceAll('-', ' ')}</span><h2>{item.title}</h2></div><time dateTime={new Date(item.occurredAt).toISOString()}>{formatTime(item.occurredAt)}</time></div><p>{item.detail}</p>{item.owner ? <AccountOwnerBadge owner={item.owner} /> : null}{itemAction(item)}</li>
|
||||
))}</ol>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
.attention-list { display: grid; gap: .8rem; padding: 0; margin: 0; list-style: none; }
|
||||
.attention-item { display: grid; gap: .8rem; padding: 1rem; background: var(--surface); border: 1px solid var(--border); border-left: .3rem solid var(--warning); border-radius: .9rem; }
|
||||
.attention-item.is-critical { border-left-color: var(--danger); }
|
||||
.attention-item-heading { display: flex; align-items: flex-start; justify-content: space-between; gap: 1rem; }
|
||||
.attention-item-heading > div { display: grid; gap: .55rem; }
|
||||
.attention-item-heading time { color: var(--muted); font-size: .8rem; text-align: right; }
|
||||
.attention-item p { margin: 0; color: var(--muted); }
|
||||
.attention-item .button { justify-self: start; text-decoration: none; }
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
SubredditPerformanceResponseSchema,
|
||||
SuccessfulExamplesResponseSchema,
|
||||
FunnelResponseSchema,
|
||||
AiUsageSummaryResponseSchema,
|
||||
} from '../../../../../packages/api-contracts/src/index.js';
|
||||
import type { z } from 'zod';
|
||||
import { createDashboardRequest } from '../../api/request.js';
|
||||
@@ -13,6 +14,7 @@ type StrategyPerformanceResponse = z.infer<typeof StrategyPerformanceResponseSch
|
||||
type DailyTrendResponse = z.infer<typeof DailyTrendResponseSchema>;
|
||||
type SuccessfulExamplesResponse = z.infer<typeof SuccessfulExamplesResponseSchema>;
|
||||
type FunnelResponse = z.infer<typeof FunnelResponseSchema>;
|
||||
type AiUsageSummaryResponse = z.infer<typeof AiUsageSummaryResponseSchema>;
|
||||
|
||||
async function parseResponse<T>(
|
||||
response: Response,
|
||||
@@ -38,6 +40,7 @@ export interface PerformanceApi {
|
||||
getDailyTrend(days: number): Promise<DailyTrendResponse>;
|
||||
getSuccessfulExamples(): Promise<SuccessfulExamplesResponse>;
|
||||
getFunnel(): Promise<FunnelResponse>;
|
||||
getAiUsage(days: number): Promise<AiUsageSummaryResponse>;
|
||||
}
|
||||
|
||||
export function createPerformanceApi(
|
||||
@@ -75,6 +78,12 @@ export function createPerformanceApi(
|
||||
FunnelResponseSchema,
|
||||
);
|
||||
},
|
||||
async getAiUsage(days: number) {
|
||||
return parseResponse(
|
||||
await request(`/api/analytics/ai-usage?days=${days}`),
|
||||
AiUsageSummaryResponseSchema,
|
||||
);
|
||||
},
|
||||
} satisfies PerformanceApi;
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import type {
|
||||
SubredditPerformanceResponseSchema,
|
||||
SuccessfulExample,
|
||||
FunnelResponseSchema,
|
||||
AiUsageSummaryResponseSchema,
|
||||
} from '../../../../../packages/api-contracts/src/index.js';
|
||||
import {
|
||||
useDailyTrendQuery,
|
||||
@@ -12,6 +13,7 @@ import {
|
||||
useSubredditPerformanceQuery,
|
||||
useSuccessfulExamplesQuery,
|
||||
useFunnelQuery,
|
||||
useAiUsageQuery,
|
||||
} from './performance-queries.js';
|
||||
import './performance.css';
|
||||
|
||||
@@ -20,6 +22,33 @@ type SubredditPerformance = z.infer<typeof SubredditPerformanceResponseSchema>[n
|
||||
type StrategyPerformance = z.infer<typeof StrategyPerformanceResponseSchema>[number];
|
||||
type DailyTrend = z.infer<typeof DailyTrendResponseSchema>[number];
|
||||
type Funnel = z.infer<typeof FunnelResponseSchema>;
|
||||
type AiUsageSummary = z.infer<typeof AiUsageSummaryResponseSchema>;
|
||||
|
||||
function formatUsd(value: number): string {
|
||||
return new Intl.NumberFormat(undefined, {
|
||||
style: 'currency', currency: 'USD', minimumFractionDigits: 2, maximumFractionDigits: 4,
|
||||
}).format(value);
|
||||
}
|
||||
|
||||
function AiUsagePanel({ usage }: { usage: AiUsageSummary }) {
|
||||
const successRate = usage.requests > 0 ? Math.round((usage.successes / usage.requests) * 100) : 0;
|
||||
return (
|
||||
<section aria-labelledby="ai-usage-heading" className="panel feature-panel">
|
||||
<div className="panel-heading"><div><span className="eyebrow">OpenRouter · last {usage.days} days</span><h2 id="ai-usage-heading">AI cost and reliability</h2></div><span className="sample-size">{usage.requests} requests</span></div>
|
||||
<div aria-label="AI usage totals" className="metrics-grid performance-metrics">
|
||||
<article className="metric-card"><span>Reported cost</span><strong>{formatUsd(usage.costUsd)}</strong><small>OpenRouter usage accounting</small></article>
|
||||
<article className="metric-card"><span>Tokens</span><strong>{usage.totalTokens.toLocaleString()}</strong><small>{usage.promptTokens.toLocaleString()} prompt · {usage.completionTokens.toLocaleString()} completion</small></article>
|
||||
<article className="metric-card"><span>Success rate</span><strong>{successRate}%</strong><small>{usage.refusals} refusals · {usage.errors} errors/empty</small></article>
|
||||
<article className="metric-card"><span>Average latency</span><strong>{usage.averageLatencyMs.toLocaleString()} ms</strong><small>All recorded attempts</small></article>
|
||||
</div>
|
||||
{usage.models.length === 0 ? <p className="feature-empty">No AI usage has been recorded yet. New requests will appear here.</p> : (
|
||||
<div className="table-scroll"><table aria-label="AI usage by model" className="performance-table"><thead><tr><th scope="col">Model</th><th scope="col">Requests</th><th scope="col">Success</th><th scope="col">Refusals</th><th scope="col">Errors</th><th scope="col">Tokens</th><th scope="col">Cost</th><th scope="col">Latency</th></tr></thead><tbody>{usage.models.map((model) => (
|
||||
<tr key={model.model}><th scope="row">{model.model}</th><td>{model.requests}</td><td>{model.successes}</td><td>{model.refusals}</td><td>{model.errors}</td><td>{model.totalTokens.toLocaleString()}</td><td>{formatUsd(model.costUsd)}</td><td>{model.averageLatencyMs.toLocaleString()} ms</td></tr>
|
||||
))}</tbody></table></div>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function SampleSize({ totalSent }: { totalSent: number }) {
|
||||
const isLowConfidence = totalSent < LOW_CONFIDENCE_SAMPLE_SIZE;
|
||||
@@ -129,7 +158,8 @@ export function PerformancePage() {
|
||||
const trend = useDailyTrendQuery(30);
|
||||
const examples = useSuccessfulExamplesQuery();
|
||||
const funnel = useFunnelQuery();
|
||||
const queries = [subreddits, strategies, trend, examples, funnel];
|
||||
const aiUsage = useAiUsageQuery(30);
|
||||
const queries = [subreddits, strategies, trend, examples, funnel, aiUsage];
|
||||
if (queries.some((query) => query.isPending)) {
|
||||
return <div className="page-state" role="status">Loading performance…</div>;
|
||||
}
|
||||
@@ -142,6 +172,7 @@ export function PerformancePage() {
|
||||
<div className="page performance-page">
|
||||
<header className="page-header"><div><span className="eyebrow">Decision support</span><h1>Performance</h1><p>Compare response and received-image outcomes without hiding the sample sizes behind the percentages.</p></div></header>
|
||||
{funnel.data ? <FunnelPanel funnel={funnel.data} /> : null}
|
||||
{aiUsage.data ? <AiUsagePanel usage={aiUsage.data} /> : null}
|
||||
<section aria-labelledby="trend-heading" className="panel feature-panel"><div className="panel-heading"><div><span className="eyebrow">Last 30 days</span><h2 id="trend-heading">Response trend</h2></div></div><Trend rows={trend.data ?? []} /></section>
|
||||
<div className="performance-grid">
|
||||
<section aria-labelledby="subreddit-heading" className="panel feature-panel"><div className="panel-heading"><h2 id="subreddit-heading">By subreddit</h2></div><PerformanceTable label="Subreddit performance" rows={subreddits.data ?? []} rowLabel={(row) => 'subreddit' in row ? row.subreddit : row.strategyId} /></section>
|
||||
|
||||
@@ -7,6 +7,7 @@ export const performanceQueryKeys = {
|
||||
trend: (days: number) => ['performance', 'trend', days] as const,
|
||||
successfulExamples: ['performance', 'successful-examples'] as const,
|
||||
funnel: ['performance', 'funnel'] as const,
|
||||
aiUsage: (days: number) => ['performance', 'ai-usage', days] as const,
|
||||
};
|
||||
|
||||
export function useSubredditPerformanceQuery() {
|
||||
@@ -43,3 +44,10 @@ export function useFunnelQuery() {
|
||||
queryFn: performanceApi.getFunnel,
|
||||
});
|
||||
}
|
||||
|
||||
export function useAiUsageQuery(days: number) {
|
||||
return useQuery({
|
||||
queryKey: performanceQueryKeys.aiUsage(days),
|
||||
queryFn: () => performanceApi.getAiUsage(days),
|
||||
});
|
||||
}
|
||||
@@ -86,6 +86,8 @@ an administrator credential.
|
||||
| `GET` | `/api/analytics/by-strategy` | Get performance by strategy | read | no |
|
||||
| `GET` | `/api/analytics/funnel` | Get mature message outcome funnel | read | no |
|
||||
| `GET` | `/api/analytics/trend` | Get daily performance trend | read | no |
|
||||
| `GET` | `/api/analytics/ai-usage` | Get OpenRouter cost and model reliability | read | no |
|
||||
| `GET` | `/api/attention` | Get operator attention items | read | no |
|
||||
| `GET` | `/api/config` | Get non-secret application configuration | read | no |
|
||||
| `PUT` | `/api/config` | Update application configuration | admin | yes |
|
||||
| `GET` | `/api/login/status` | Get Reddit session state | read | no |
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { z } from 'zod';
|
||||
import { RedditAccountOwnerSchema } from './session.js';
|
||||
|
||||
export const AttentionKindSchema = z.enum([
|
||||
'human-required',
|
||||
'boundary',
|
||||
'failed-delivery',
|
||||
'account-login-required',
|
||||
'runtime-failed',
|
||||
'invitation-cooldown',
|
||||
]);
|
||||
|
||||
export const AttentionItemSchema = z.strictObject({
|
||||
id: z.string(),
|
||||
kind: AttentionKindSchema,
|
||||
severity: z.enum(['warning', 'critical']),
|
||||
title: z.string(),
|
||||
detail: z.string(),
|
||||
occurredAt: z.number(),
|
||||
owner: RedditAccountOwnerSchema.optional(),
|
||||
conversationId: z.string().optional(),
|
||||
username: z.string().optional(),
|
||||
});
|
||||
|
||||
export const AttentionResponseSchema = z.strictObject({
|
||||
generatedAt: z.number(),
|
||||
items: z.array(AttentionItemSchema),
|
||||
});
|
||||
|
||||
export type AttentionItem = z.infer<typeof AttentionItemSchema>;
|
||||
export type AttentionResponse = z.infer<typeof AttentionResponseSchema>;
|
||||
@@ -1,4 +1,5 @@
|
||||
export * from './activity.js';
|
||||
export * from './attention.js';
|
||||
export * from './chats.js';
|
||||
export * from './common.js';
|
||||
export * from './config.js';
|
||||
|
||||
@@ -74,5 +74,24 @@ export const FunnelResponseSchema = z.strictObject({
|
||||
images: z.number(),
|
||||
});
|
||||
|
||||
const AiUsageFields = {
|
||||
requests: z.number().int().nonnegative(),
|
||||
successes: z.number().int().nonnegative(),
|
||||
refusals: z.number().int().nonnegative(),
|
||||
errors: z.number().int().nonnegative(),
|
||||
promptTokens: z.number().int().nonnegative(),
|
||||
completionTokens: z.number().int().nonnegative(),
|
||||
totalTokens: z.number().int().nonnegative(),
|
||||
costUsd: z.number().nonnegative(),
|
||||
averageLatencyMs: z.number().nonnegative(),
|
||||
};
|
||||
|
||||
export const AiUsageSummaryResponseSchema = z.strictObject({
|
||||
days: z.number().int().positive(),
|
||||
...AiUsageFields,
|
||||
models: z.array(z.strictObject({ model: z.string(), ...AiUsageFields })),
|
||||
});
|
||||
|
||||
export type StatsResponse = z.infer<typeof StatsResponseSchema>;
|
||||
export type TimersResponse = z.infer<typeof TimersResponseSchema>;
|
||||
export type AiUsageSummaryResponse = z.infer<typeof AiUsageSummaryResponseSchema>;
|
||||
+96
-22
@@ -1,6 +1,7 @@
|
||||
import OpenAI from 'openai';
|
||||
import type { Config } from '../config/index.js';
|
||||
import type { Logger } from '../utils/logger.js';
|
||||
import type { AiUsageRecordInput, AiUsageOutcome } from '../database/ai-usage-repository.js';
|
||||
import {
|
||||
formatConversationAsUntrustedData,
|
||||
isSafeConversationReply,
|
||||
@@ -29,6 +30,8 @@ interface ProtectedTextPrompt {
|
||||
user: string;
|
||||
}
|
||||
|
||||
export type AiUsageRecorder = (usage: AiUsageRecordInput) => void;
|
||||
|
||||
export class OpenRouterClient {
|
||||
private client: OpenAI;
|
||||
|
||||
@@ -36,6 +39,7 @@ export class OpenRouterClient {
|
||||
private config: Config,
|
||||
private logger: Logger,
|
||||
client?: OpenAI,
|
||||
private readonly usageRecorder?: AiUsageRecorder,
|
||||
) {
|
||||
this.client = client ?? new OpenAI({
|
||||
baseURL: 'https://openrouter.ai/api/v1',
|
||||
@@ -48,6 +52,7 @@ export class OpenRouterClient {
|
||||
}
|
||||
|
||||
async detectGenderOnly(imageBase64: string, mimeType: string): Promise<DetectedGender> {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
this.logger.debug('Quick gender detection...');
|
||||
|
||||
@@ -74,12 +79,14 @@ export class OpenRouterClient {
|
||||
});
|
||||
|
||||
const result = response.choices[0]?.message?.content?.toLowerCase().trim() || 'unknown';
|
||||
this.recordUsage('detectGender', this.config.openrouter.imageAnalysisModel, startedAt, response, result ? 'success' : 'empty');
|
||||
|
||||
if (result.includes('female')) return 'female';
|
||||
if (result.includes('male') && !result.includes('female')) return 'male';
|
||||
if (result.includes('other')) return 'other';
|
||||
return 'unknown';
|
||||
} catch (error) {
|
||||
this.recordError('detectGender', this.config.openrouter.imageAnalysisModel, startedAt, error);
|
||||
this.logger.error('Gender detection failed:', error);
|
||||
return 'unknown';
|
||||
}
|
||||
@@ -134,33 +141,40 @@ export class OpenRouterClient {
|
||||
}
|
||||
|
||||
private async tryAnalyzeWithModel(model: string, imageBase64: string, mimeType: string): Promise<ImageAnalysisResult | null> {
|
||||
const startedAt = Date.now();
|
||||
const genderPrompt = `First, identify the gender of the person in this image. Respond with EXACTLY one of these on the first line: GENDER:male, GENDER:female, GENDER:other, or GENDER:unknown
|
||||
|
||||
Then provide your analysis:
|
||||
|
||||
${this.config.openrouter.imageAnalysisPrompt}`;
|
||||
|
||||
const response = await this.client.chat.completions.create({
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: genderPrompt,
|
||||
},
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: `data:${mimeType};base64,${imageBase64}`,
|
||||
let response;
|
||||
try {
|
||||
response = await this.client.chat.completions.create({
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: genderPrompt,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
max_tokens: 600,
|
||||
});
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: `data:${mimeType};base64,${imageBase64}`,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
max_tokens: 600,
|
||||
});
|
||||
} catch (error) {
|
||||
this.recordError('analyzeImage', model, startedAt, error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
const fullResponse = response.choices[0]?.message?.content || '';
|
||||
|
||||
@@ -190,9 +204,12 @@ ${this.config.openrouter.imageAnalysisPrompt}`;
|
||||
|
||||
// Detect AI refusal - return null to signal fallback should be tried
|
||||
if (this.isRefusal(description)) {
|
||||
this.recordUsage('analyzeImage', model, startedAt, response, 'refusal');
|
||||
return null;
|
||||
}
|
||||
|
||||
this.recordUsage('analyzeImage', model, startedAt, response, description ? 'success' : 'empty');
|
||||
|
||||
this.logger.debug(`Image analysis complete (${model}) - Gender: ${detectedGender}, Rating: ${attractivenessScore ?? 'n/a'}`);
|
||||
this.logger.debug(`Description: ${description}`);
|
||||
|
||||
@@ -480,7 +497,10 @@ Just write the message, nothing else.`;
|
||||
if (isFallback) {
|
||||
this.logger.info(`[AI] ${label} trying fallback model: ${model}`);
|
||||
}
|
||||
const response = await this.client.chat.completions.create({
|
||||
const startedAt = Date.now();
|
||||
let response;
|
||||
try {
|
||||
response = await this.client.chat.completions.create({
|
||||
model,
|
||||
messages: typeof prompt === 'string'
|
||||
? [{ role: 'user', content: prompt }]
|
||||
@@ -490,7 +510,11 @@ Just write the message, nothing else.`;
|
||||
],
|
||||
max_tokens: opts.max_tokens,
|
||||
temperature: opts.temperature,
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
this.recordError(label, model, startedAt, error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
const raw = response.choices[0]?.message?.content || '';
|
||||
const choice = response.choices[0] as unknown as Record<string, unknown> | undefined;
|
||||
@@ -508,14 +532,18 @@ Just write the message, nothing else.`;
|
||||
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) {
|
||||
this.recordUsage(label, model, startedAt, response, 'empty');
|
||||
this.logger.warn(`[AI] ${label}: ${model} returned empty content${i < models.length - 1 ? ', trying fallback...' : ''}`);
|
||||
continue;
|
||||
}
|
||||
if (this.isRefusal(trimmed)) {
|
||||
this.recordUsage(label, model, startedAt, response, 'refusal');
|
||||
this.logger.warn(`[AI] ${label}: ${model} returned a refusal${i < models.length - 1 ? ', trying fallback...' : ''}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
this.recordUsage(label, model, startedAt, response, 'success');
|
||||
|
||||
if (isFallback) {
|
||||
this.logger.info(`[AI] ${label}: fallback model ${model} succeeded`);
|
||||
}
|
||||
@@ -532,6 +560,48 @@ Just write the message, nothing else.`;
|
||||
return '';
|
||||
}
|
||||
|
||||
private recordUsage(
|
||||
operation: string,
|
||||
requestedModel: string,
|
||||
startedAt: number,
|
||||
response: unknown,
|
||||
outcome: AiUsageOutcome,
|
||||
): void {
|
||||
if (!this.usageRecorder) return;
|
||||
const record = response as Record<string, unknown>;
|
||||
const usage = (record.usage ?? {}) as Record<string, unknown>;
|
||||
try {
|
||||
this.usageRecorder({
|
||||
operation,
|
||||
requestedModel,
|
||||
actualModel: typeof record.model === 'string' ? record.model : requestedModel,
|
||||
outcome,
|
||||
promptTokens: numericUsage(usage.prompt_tokens),
|
||||
completionTokens: numericUsage(usage.completion_tokens),
|
||||
totalTokens: numericUsage(usage.total_tokens),
|
||||
costUsd: typeof usage.cost === 'number' && Number.isFinite(usage.cost) ? usage.cost : null,
|
||||
latencyMs: Date.now() - startedAt,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.warn('Failed to persist AI usage accounting:', error);
|
||||
}
|
||||
}
|
||||
|
||||
private recordError(operation: string, requestedModel: string, startedAt: number, error: unknown): void {
|
||||
if (!this.usageRecorder) return;
|
||||
try {
|
||||
this.usageRecorder({
|
||||
operation,
|
||||
requestedModel,
|
||||
outcome: 'error',
|
||||
latencyMs: Date.now() - startedAt,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
} catch (recordingError) {
|
||||
this.logger.warn('Failed to persist AI usage error:', recordingError);
|
||||
}
|
||||
}
|
||||
|
||||
async analyzeAndGenerateMessage(
|
||||
imageBase64: string,
|
||||
mimeType: string,
|
||||
@@ -547,3 +617,7 @@ Just write the message, nothing else.`;
|
||||
return { analysis, message };
|
||||
}
|
||||
}
|
||||
|
||||
function numericUsage(value: unknown): number {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0;
|
||||
}
|
||||
+7
-1
@@ -47,6 +47,7 @@ import type { Cookie } from 'playwright';
|
||||
import { ChatCache } from './services/chat/cache.js';
|
||||
import { RedditAccountRuntimeManager } from './services/reddit-account-runtime.js';
|
||||
import { createConcreteRedditAccountRuntimeFactory } from './services/reddit-account-runtime-builder.js';
|
||||
import { AiUsageRepository } from './database/ai-usage-repository.js';
|
||||
import { importLegacyAccountContent } from './database/legacy-content-import.js';
|
||||
import { RecipientDeduplicator } from './services/recipient-deduplicator.js';
|
||||
import { createConfigBackedSubredditPollingControl } from './services/subreddit-polling-control.js';
|
||||
@@ -127,6 +128,7 @@ export async function bootstrap(
|
||||
const recipientDeduplicator = new RecipientDeduplicator(new RecipientHistoryRepository(db));
|
||||
const redditAccountsRepo = bootstrapAccountsRepo;
|
||||
const accountContentRepo = new AccountContentRepository(db);
|
||||
const aiUsageRepo = new AiUsageRepository(db);
|
||||
const legacyImport = importLegacyAccountContent(db);
|
||||
if (legacyImport.conversations > 0 || legacyImport.media > 0) {
|
||||
logger.info(`Imported legacy unassigned content into SQLite (${legacyImport.conversations} conversations, ${legacyImport.media} media items)`);
|
||||
@@ -168,7 +170,9 @@ export async function bootstrap(
|
||||
const chatPage = await browserClient.getContext()!.newPage();
|
||||
|
||||
// AI + services
|
||||
const aiClient = new OpenRouterClient(config, logger);
|
||||
const aiClient = new OpenRouterClient(config, logger, undefined, (usage) => {
|
||||
aiUsageRepo.record({ ...usage, accountId: accountAuth.getSnapshot().accountId });
|
||||
});
|
||||
const monitorService = new MonitorService(
|
||||
redditApi, postsRepo, recipientDeduplicator, config, logger, 1000,
|
||||
createConfigBackedSubredditPollingControl(config, new ConfigStore(db), logger),
|
||||
@@ -344,6 +348,8 @@ export async function bootstrap(
|
||||
activityLogRepo,
|
||||
logger,
|
||||
broadcaster,
|
||||
process.cwd(),
|
||||
aiUsageRepo,
|
||||
);
|
||||
|
||||
webServer.setChatService(chatService);
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
import type Database from 'better-sqlite3';
|
||||
|
||||
export type AiUsageOutcome = 'success' | 'refusal' | 'empty' | 'error';
|
||||
|
||||
export interface AiUsageRecordInput {
|
||||
accountId?: string | null;
|
||||
operation: string;
|
||||
requestedModel: string;
|
||||
actualModel?: string | null;
|
||||
outcome: AiUsageOutcome;
|
||||
promptTokens?: number;
|
||||
completionTokens?: number;
|
||||
totalTokens?: number;
|
||||
costUsd?: number | null;
|
||||
latencyMs: number;
|
||||
error?: string | null;
|
||||
createdAt?: number;
|
||||
}
|
||||
|
||||
export interface AiUsageModelSummary {
|
||||
model: string;
|
||||
requests: number;
|
||||
successes: number;
|
||||
refusals: number;
|
||||
errors: number;
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
totalTokens: number;
|
||||
costUsd: number;
|
||||
averageLatencyMs: number;
|
||||
}
|
||||
|
||||
export interface AiUsageSummary {
|
||||
days: number;
|
||||
requests: number;
|
||||
successes: number;
|
||||
refusals: number;
|
||||
errors: number;
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
totalTokens: number;
|
||||
costUsd: number;
|
||||
averageLatencyMs: number;
|
||||
models: AiUsageModelSummary[];
|
||||
}
|
||||
|
||||
type SummaryRow = {
|
||||
model?: string;
|
||||
requests: number;
|
||||
successes: number;
|
||||
refusals: number;
|
||||
errors: number;
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
total_tokens: number;
|
||||
cost_usd: number;
|
||||
average_latency_ms: number;
|
||||
};
|
||||
|
||||
/** Durable, prompt-free accounting for OpenRouter requests. */
|
||||
export class AiUsageRepository {
|
||||
constructor(private readonly db: Database.Database) {}
|
||||
|
||||
record(input: AiUsageRecordInput): void {
|
||||
this.db.prepare(`
|
||||
INSERT INTO ai_usage (
|
||||
account_id, operation, requested_model, actual_model, outcome,
|
||||
prompt_tokens, completion_tokens, total_tokens, cost_usd,
|
||||
latency_ms, error, created_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
input.accountId ?? null,
|
||||
input.operation,
|
||||
input.requestedModel,
|
||||
input.actualModel ?? null,
|
||||
input.outcome,
|
||||
input.promptTokens ?? 0,
|
||||
input.completionTokens ?? 0,
|
||||
input.totalTokens ?? 0,
|
||||
input.costUsd ?? null,
|
||||
Math.max(0, Math.round(input.latencyMs)),
|
||||
input.error?.slice(0, 500) ?? null,
|
||||
input.createdAt ?? Date.now(),
|
||||
);
|
||||
}
|
||||
|
||||
getSummary(days = 30, now = Date.now()): AiUsageSummary {
|
||||
const safeDays = Math.max(1, Math.min(365, Math.trunc(days)));
|
||||
const since = now - safeDays * 86_400_000;
|
||||
const fields = `
|
||||
COUNT(*) AS requests,
|
||||
SUM(CASE WHEN outcome = 'success' THEN 1 ELSE 0 END) AS successes,
|
||||
SUM(CASE WHEN outcome = 'refusal' THEN 1 ELSE 0 END) AS refusals,
|
||||
SUM(CASE WHEN outcome IN ('empty', 'error') THEN 1 ELSE 0 END) AS errors,
|
||||
COALESCE(SUM(prompt_tokens), 0) AS prompt_tokens,
|
||||
COALESCE(SUM(completion_tokens), 0) AS completion_tokens,
|
||||
COALESCE(SUM(total_tokens), 0) AS total_tokens,
|
||||
COALESCE(SUM(cost_usd), 0) AS cost_usd,
|
||||
COALESCE(ROUND(AVG(latency_ms)), 0) AS average_latency_ms
|
||||
`;
|
||||
const total = this.db.prepare(`SELECT ${fields} FROM ai_usage WHERE created_at >= ?`)
|
||||
.get(since) as SummaryRow;
|
||||
const models = this.db.prepare(`
|
||||
SELECT COALESCE(actual_model, requested_model) AS model, ${fields}
|
||||
FROM ai_usage
|
||||
WHERE created_at >= ?
|
||||
GROUP BY COALESCE(actual_model, requested_model)
|
||||
ORDER BY cost_usd DESC, requests DESC, model ASC
|
||||
`).all(since) as SummaryRow[];
|
||||
return {
|
||||
days: safeDays,
|
||||
...mapSummary(total),
|
||||
models: models.map((row) => ({ model: row.model ?? 'unknown', ...mapSummary(row) })),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function mapSummary(row: SummaryRow): Omit<AiUsageModelSummary, 'model'> {
|
||||
return {
|
||||
requests: Number(row.requests ?? 0),
|
||||
successes: Number(row.successes ?? 0),
|
||||
refusals: Number(row.refusals ?? 0),
|
||||
errors: Number(row.errors ?? 0),
|
||||
promptTokens: Number(row.prompt_tokens ?? 0),
|
||||
completionTokens: Number(row.completion_tokens ?? 0),
|
||||
totalTokens: Number(row.total_tokens ?? 0),
|
||||
costUsd: Math.round(Number(row.cost_usd ?? 0) * 1_000_000_000_000) / 1_000_000_000_000,
|
||||
averageLatencyMs: Number(row.average_latency_ms ?? 0),
|
||||
};
|
||||
}
|
||||
@@ -251,6 +251,27 @@ export function runMigrations(db: Database.Database): void {
|
||||
CREATE INDEX idx_conversation_automation_claims_status
|
||||
ON conversation_automation_claims(account_id, status, claimed_at);
|
||||
`),
|
||||
() => db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS ai_usage (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
account_id TEXT,
|
||||
operation TEXT NOT NULL,
|
||||
requested_model TEXT NOT NULL,
|
||||
actual_model TEXT,
|
||||
outcome TEXT NOT NULL CHECK(outcome IN ('success', 'refusal', 'empty', 'error')),
|
||||
prompt_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
completion_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
total_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
cost_usd REAL,
|
||||
latency_ms INTEGER NOT NULL,
|
||||
error TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
FOREIGN KEY (account_id) REFERENCES reddit_accounts(account_id) ON DELETE SET NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ai_usage_created_at ON ai_usage(created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_ai_usage_model ON ai_usage(actual_model, requested_model, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_ai_usage_account ON ai_usage(account_id, created_at DESC);
|
||||
`),
|
||||
];
|
||||
|
||||
let currentVersion = db.pragma('user_version', { simple: true }) as number;
|
||||
|
||||
@@ -227,6 +227,16 @@ export class DeliveryOutboxRepository {
|
||||
.get(this.accountId) as { count: number }).count;
|
||||
}
|
||||
|
||||
getFailed(limit = 50): DeliveryOutboxRecord[] {
|
||||
const rows = this.db.prepare(`
|
||||
SELECT * FROM delivery_outbox
|
||||
WHERE account_id = ? AND status = 'failed'
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT ?
|
||||
`).all(this.accountId, limit) as Record<string, unknown>[];
|
||||
return rows.map((row) => this.mapRow(row));
|
||||
}
|
||||
|
||||
markPrepared(id: number, now: number = Date.now()): boolean {
|
||||
return this.db.prepare(`
|
||||
UPDATE delivery_outbox SET status = 'prepared', updated_at = ?, last_error = NULL
|
||||
@@ -1081,6 +1091,32 @@ export class ConversationStateRepository {
|
||||
const row = this.getStmt.get(this.accountId, conversationId) as { state_json: string } | undefined;
|
||||
return row ? parseConversationState(row.state_json) ?? undefined : undefined;
|
||||
}
|
||||
|
||||
listAttention(): Array<{
|
||||
conversationId: string;
|
||||
status: 'human-required' | 'boundary';
|
||||
reason: string | null;
|
||||
updatedAt: number;
|
||||
}> {
|
||||
const rows = this.db.prepare(`
|
||||
SELECT conversation_id, state_json, updated_at
|
||||
FROM conversation_states
|
||||
WHERE account_id = ?
|
||||
ORDER BY updated_at DESC
|
||||
`).all(this.accountId) as Array<{ conversation_id: string; state_json: string; updated_at: number }>;
|
||||
return rows.flatMap((row) => {
|
||||
const state = parseConversationState(row.state_json);
|
||||
if (!state) return [];
|
||||
const status = state.automation.status;
|
||||
if (status !== 'human-required' && status !== 'boundary') return [];
|
||||
return [{
|
||||
conversationId: row.conversation_id,
|
||||
status,
|
||||
reason: state.automation.handoffReason,
|
||||
updatedAt: row.updated_at,
|
||||
}];
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Durable idempotency claims for AI replies to individual inbound events. */
|
||||
|
||||
@@ -43,6 +43,7 @@ import { RecipientDeduplicator } from './recipient-deduplicator.js';
|
||||
import { ConfigStore } from '../config/persistence.js';
|
||||
import { createConfigBackedSubredditPollingControl } from './subreddit-polling-control.js';
|
||||
import { ConversationAutomationService } from './conversation-automation.js';
|
||||
import { AiUsageRepository } from '../database/ai-usage-repository.js';
|
||||
|
||||
export interface ConcreteRuntimeFactoryOptions {
|
||||
config: Config;
|
||||
@@ -118,7 +119,10 @@ async function buildRuntime(
|
||||
const redditApi = new RedditApiClient(accountConfig, logger, redditRequestCoordinator);
|
||||
redditApi.setBrowserContext(browserClient.getContext()!);
|
||||
const chatPage = await browserClient.getContext()!.newPage();
|
||||
const aiClient = new OpenRouterClient(accountConfig, logger);
|
||||
const aiUsageRepo = new AiUsageRepository(db);
|
||||
const aiClient = new OpenRouterClient(accountConfig, logger, undefined, (usage) => {
|
||||
aiUsageRepo.record({ ...usage, accountId: session.accountId });
|
||||
});
|
||||
const monitorService = new MonitorService(
|
||||
redditApi, postsRepo, recipientDeduplicator, accountConfig, logger,
|
||||
1000, createConfigBackedSubredditPollingControl(config, new ConfigStore(db), logger),
|
||||
|
||||
@@ -94,6 +94,8 @@ export const API_OPERATIONS: readonly ApiOperation[] = [
|
||||
operation('getStrategyAnalytics', 'GET', '/api/analytics/by-strategy', 'Get performance by strategy', ['Analytics']),
|
||||
operation('getFunnelAnalytics', 'GET', '/api/analytics/funnel', 'Get mature message outcome funnel', ['Analytics']),
|
||||
operation('getAnalyticsTrend', 'GET', '/api/analytics/trend', 'Get daily performance trend', ['Analytics']),
|
||||
operation('getAiUsageAnalytics', 'GET', '/api/analytics/ai-usage', 'Get OpenRouter cost and model reliability', ['Analytics']),
|
||||
operation('getAttention', 'GET', '/api/attention', 'Get operator attention items', ['Operations']),
|
||||
operation('getConfig', 'GET', '/api/config', 'Get non-secret application configuration', ['Configuration']),
|
||||
operation('updateConfig', 'PUT', '/api/config', 'Update application configuration', ['Configuration']),
|
||||
operation('getLoginStatus', 'GET', '/api/login/status', 'Get Reddit session state', ['Accounts']),
|
||||
|
||||
@@ -22,6 +22,7 @@ import type { AccountAuthContext } from '../services/account-auth-context.js';
|
||||
import type { ConfigStore } from '../config/persistence.js';
|
||||
import type { RedditAccountsService } from '../services/reddit-accounts.js';
|
||||
import type { RedditAccountRuntimeManager } from '../services/reddit-account-runtime.js';
|
||||
import type { AiUsageRepository } from '../database/ai-usage-repository.js';
|
||||
|
||||
/** Bag of everything the route modules might need. */
|
||||
export interface WebContext {
|
||||
@@ -35,6 +36,7 @@ export interface WebContext {
|
||||
posts: PostsRepository;
|
||||
messages: MessagesRepository;
|
||||
successful: SuccessfulMessagesRepository;
|
||||
aiUsage?: AiUsageRepository;
|
||||
};
|
||||
|
||||
services: {
|
||||
|
||||
@@ -37,5 +37,14 @@ export function createAnalyticsRouter(ctx: WebContext): Router {
|
||||
res.json(ctx.repos.messages.getResponseRateTrend(days));
|
||||
});
|
||||
|
||||
router.get('/analytics/ai-usage', (req, res) => {
|
||||
if (!ctx.repos.aiUsage) {
|
||||
res.status(503).json({ error: 'AI usage accounting is not initialized' });
|
||||
return;
|
||||
}
|
||||
const days = parseInt(req.query.days as string) || 30;
|
||||
res.json(ctx.repos.aiUsage.getSummary(days));
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import { Router } from 'express';
|
||||
import type { WebContext } from '../context.js';
|
||||
|
||||
interface AttentionItem {
|
||||
id: string;
|
||||
kind: 'human-required' | 'boundary' | 'failed-delivery' | 'account-login-required' | 'runtime-failed' | 'invitation-cooldown';
|
||||
severity: 'warning' | 'critical';
|
||||
title: string;
|
||||
detail: string;
|
||||
occurredAt: number;
|
||||
owner?: { accountId: string; redditUsername: string };
|
||||
conversationId?: string;
|
||||
username?: string;
|
||||
}
|
||||
|
||||
export function createAttentionRouter(ctx: WebContext): Router {
|
||||
const router = Router();
|
||||
|
||||
router.get('/attention', async (_req, res) => {
|
||||
const items: AttentionItem[] = [];
|
||||
const accounts = ctx.services.redditAccounts?.list() ?? [];
|
||||
for (const account of accounts) {
|
||||
if (account.status !== 'login-required') continue;
|
||||
items.push({
|
||||
id: `account:${account.accountId}`,
|
||||
kind: 'account-login-required',
|
||||
severity: 'critical',
|
||||
title: `u/${account.redditUsername} needs a new session`,
|
||||
detail: 'Import a fresh extension session before this account can resume.',
|
||||
occurredAt: account.updatedAt,
|
||||
owner: { accountId: account.accountId, redditUsername: account.redditUsername },
|
||||
});
|
||||
}
|
||||
|
||||
const snapshots = ctx.services.accountRuntimes?.list() ?? [];
|
||||
for (const snapshot of snapshots) {
|
||||
const owner = { accountId: snapshot.accountId, redditUsername: snapshot.redditUsername };
|
||||
if (snapshot.state === 'failed') {
|
||||
items.push({
|
||||
id: `runtime:${snapshot.accountId}`,
|
||||
kind: 'runtime-failed',
|
||||
severity: 'critical',
|
||||
title: `u/${snapshot.redditUsername} runtime failed`,
|
||||
detail: snapshot.failure ?? 'The account runtime stopped unexpectedly.',
|
||||
occurredAt: snapshot.updatedAt,
|
||||
owner,
|
||||
});
|
||||
}
|
||||
const runtime = ctx.services.accountRuntimes?.get(snapshot.accountId);
|
||||
if (!runtime) continue;
|
||||
const conversations = await runtime.resources.chatService.getCachedConversations();
|
||||
const usernames = new Map(conversations.flatMap((conversation) => conversation.roomId
|
||||
? [[conversation.roomId, conversation.username] as const]
|
||||
: []));
|
||||
for (const state of runtime.resources.conversationStateRepo.listAttention()) {
|
||||
const username = usernames.get(state.conversationId);
|
||||
items.push({
|
||||
id: `conversation:${snapshot.accountId}:${state.conversationId}:${state.status}`,
|
||||
kind: state.status,
|
||||
severity: state.status === 'boundary' ? 'critical' : 'warning',
|
||||
title: state.status === 'boundary'
|
||||
? `Boundary detected${username ? ` from u/${username}` : ''}`
|
||||
: `Human follow-up required${username ? ` for u/${username}` : ''}`,
|
||||
detail: state.reason ?? (state.status === 'boundary'
|
||||
? 'Automation stopped after the recipient set a boundary.'
|
||||
: 'Automated conversation turns are exhausted.'),
|
||||
occurredAt: state.updatedAt,
|
||||
owner,
|
||||
conversationId: `${snapshot.accountId}~${state.conversationId}`,
|
||||
...(username ? { username } : {}),
|
||||
});
|
||||
}
|
||||
for (const delivery of runtime.resources.deliveryOutboxRepo.getFailed()) {
|
||||
items.push({
|
||||
id: `delivery:${snapshot.accountId}:${delivery.id}`,
|
||||
kind: 'failed-delivery',
|
||||
severity: 'warning',
|
||||
title: `Message to u/${delivery.recipient} failed`,
|
||||
detail: delivery.lastError ?? `Delivery failed after ${delivery.attemptCount} attempt(s).`,
|
||||
occurredAt: delivery.updatedAt,
|
||||
owner,
|
||||
username: delivery.recipient,
|
||||
});
|
||||
}
|
||||
const cooldown = runtime.resources.chatService.getInvitationCooldown();
|
||||
if (cooldown) {
|
||||
items.push({
|
||||
id: `cooldown:${snapshot.accountId}:${cooldown.until}`,
|
||||
kind: 'invitation-cooldown',
|
||||
severity: 'warning',
|
||||
title: `u/${snapshot.redditUsername} invitation cooldown`,
|
||||
detail: cooldown.reason,
|
||||
occurredAt: Date.now(),
|
||||
owner,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
res.json({ generatedAt: Date.now(), items: items.sort((left, right) => right.occurredAt - left.occurredAt) });
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -41,6 +41,8 @@ import { createLoginRouter } from './routes/login.js';
|
||||
import { createDashboardAuth, createDashboardSessionRouter } from './auth.js';
|
||||
import { resolveDashboardAssets } from './dashboard-assets.js';
|
||||
import { createAgentRouter, createHealthRouter } from './routes/agent.js';
|
||||
import { createAttentionRouter } from './routes/attention.js';
|
||||
import type { AiUsageRepository } from '../database/ai-usage-repository.js';
|
||||
|
||||
export type { BotStats, ActivityItem, LogEntry, RuntimeConfig } from './types.js';
|
||||
|
||||
@@ -74,6 +76,7 @@ export class WebServer {
|
||||
private logger: Logger,
|
||||
broadcaster?: LogBroadcaster,
|
||||
private workspaceDirectory = process.cwd(),
|
||||
private aiUsageRepo?: AiUsageRepository,
|
||||
) {
|
||||
this.broadcaster = broadcaster ?? new LogBroadcaster();
|
||||
this.activity = new ActivityLog(activityLogRepo);
|
||||
@@ -104,6 +107,7 @@ export class WebServer {
|
||||
posts: this.postsRepo,
|
||||
messages: this.messagesRepo,
|
||||
successful: this.successfulMessagesRepo,
|
||||
aiUsage: this.aiUsageRepo,
|
||||
},
|
||||
services: this.services,
|
||||
getBotRunning: () => this.botRunning,
|
||||
@@ -147,6 +151,7 @@ export class WebServer {
|
||||
const ctx = this.context();
|
||||
this.app.use('/api', createAgentRouter(ctx));
|
||||
this.app.use('/api', createStatsRouter(ctx));
|
||||
this.app.use('/api', createAttentionRouter(ctx));
|
||||
this.app.use('/api', createChatsRouter(ctx));
|
||||
this.app.use('/api', createConversationsRouter(ctx));
|
||||
this.app.use('/api', createMediaRouter(ctx));
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { AiUsageRepository } from '../src/database/ai-usage-repository.js';
|
||||
import { createTestDb } from './testUtils.js';
|
||||
|
||||
describe('AiUsageRepository', () => {
|
||||
it('aggregates cost, tokens, outcomes, and latency by actual model', () => {
|
||||
const db = createTestDb();
|
||||
const repo = new AiUsageRepository(db);
|
||||
const now = Date.parse('2026-08-09T10:00:00Z');
|
||||
repo.record({ operation: 'generateReply', requestedModel: 'router', actualModel: 'model-a', outcome: 'success', promptTokens: 100, completionTokens: 20, totalTokens: 120, costUsd: 0.0012, latencyMs: 400, createdAt: now });
|
||||
repo.record({ operation: 'generateReply', requestedModel: 'model-a', actualModel: 'model-a', outcome: 'refusal', promptTokens: 40, completionTokens: 5, totalTokens: 45, costUsd: 0.0003, latencyMs: 200, createdAt: now });
|
||||
repo.record({ operation: 'analyzeImage', requestedModel: 'model-b', outcome: 'error', latencyMs: 900, error: 'provider unavailable', createdAt: now });
|
||||
|
||||
expect(repo.getSummary(30, now)).toEqual({
|
||||
days: 30,
|
||||
requests: 3,
|
||||
successes: 1,
|
||||
refusals: 1,
|
||||
errors: 1,
|
||||
promptTokens: 140,
|
||||
completionTokens: 25,
|
||||
totalTokens: 165,
|
||||
costUsd: 0.0015,
|
||||
averageLatencyMs: 500,
|
||||
models: [
|
||||
{ model: 'model-a', requests: 2, successes: 1, refusals: 1, errors: 0, promptTokens: 140, completionTokens: 25, totalTokens: 165, costUsd: 0.0015, averageLatencyMs: 300 },
|
||||
{ model: 'model-b', requests: 1, successes: 0, refusals: 0, errors: 1, promptTokens: 0, completionTokens: 0, totalTokens: 0, costUsd: 0, averageLatencyMs: 900 },
|
||||
],
|
||||
});
|
||||
db.close();
|
||||
});
|
||||
});
|
||||
@@ -79,7 +79,7 @@ describe('dashboard foundation', () => {
|
||||
const { router } = renderDashboard('/');
|
||||
|
||||
const navigation = screen.getByRole('navigation', { name: 'Primary navigation' });
|
||||
for (const label of ['Overview', 'Conversations', 'Media', 'Performance', 'Operations', 'Settings']) {
|
||||
for (const label of ['Overview', 'Needs attention', 'Conversations', 'Media', 'Performance', 'Operations', 'Settings']) {
|
||||
expect(within(navigation).getByRole('link', { name: label })).toBeInTheDocument();
|
||||
}
|
||||
|
||||
@@ -91,6 +91,24 @@ describe('dashboard foundation', () => {
|
||||
expect(screen.getByRole('heading', { name: 'Operations' })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows actionable attention items and links handoffs to their conversation', async () => {
|
||||
installSuccessfulApi();
|
||||
server.use(http.get('/api/attention', () => HttpResponse.json({
|
||||
generatedAt: Date.parse('2026-08-09T08:00:00Z'),
|
||||
items: [{
|
||||
id: 'conversation:one', kind: 'human-required', severity: 'warning',
|
||||
title: 'Human follow-up required for u/alice', detail: 'Automated turns are exhausted.',
|
||||
occurredAt: Date.parse('2026-08-09T07:00:00Z'),
|
||||
owner: { accountId: '@one:reddit.com', redditUsername: 'operator' },
|
||||
conversationId: '@one:reddit.com~!room:reddit.com', username: 'alice',
|
||||
}],
|
||||
})));
|
||||
renderDashboard('/attention');
|
||||
expect(await screen.findByRole('heading', { name: 'Needs attention' })).toBeInTheDocument();
|
||||
expect(screen.getByText('Human follow-up required for u/alice')).toBeInTheDocument();
|
||||
expect(screen.getByRole('link', { name: 'Open conversation' })).toHaveAttribute('href', '/conversations/%40one%3Areddit.com~!room%3Areddit.com');
|
||||
});
|
||||
|
||||
it('hydrates media filters from the routed URL', async () => {
|
||||
installSuccessfulApi();
|
||||
server.use(
|
||||
|
||||
@@ -83,12 +83,24 @@ describe('PerformancePage', () => {
|
||||
upsell: 3,
|
||||
images: 2,
|
||||
})),
|
||||
http.get('/api/analytics/ai-usage', ({ request }) => {
|
||||
expect(new URL(request.url).searchParams.get('days')).toBe('30');
|
||||
return HttpResponse.json({
|
||||
days: 30, requests: 12, successes: 9, refusals: 2, errors: 1,
|
||||
promptTokens: 1200, completionTokens: 300, totalTokens: 1500,
|
||||
costUsd: 0.0123, averageLatencyMs: 420,
|
||||
models: [{ model: 'model-a', requests: 12, successes: 9, refusals: 2, errors: 1, promptTokens: 1200, completionTokens: 300, totalTokens: 1500, costUsd: 0.0123, averageLatencyMs: 420 }],
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
renderPage(<PerformancePage />);
|
||||
|
||||
expect(await screen.findByRole('heading', { name: 'Performance' })).toBeInTheDocument();
|
||||
expect(screen.getByRole('heading', { name: 'Outcome funnel' })).toBeInTheDocument();
|
||||
expect(screen.getByRole('heading', { name: 'AI cost and reliability' })).toBeInTheDocument();
|
||||
expect(screen.getByRole('table', { name: 'AI usage by model' })).toHaveTextContent('model-a');
|
||||
expect(screen.getByLabelText('AI usage totals')).toHaveTextContent('$0.0123');
|
||||
expect(screen.getByText('5 awaiting maturity')).toBeInTheDocument();
|
||||
const subredditTable = screen.getByRole('table', { name: 'Subreddit performance' });
|
||||
expect(within(subredditTable).getByText('n = 40')).toBeInTheDocument();
|
||||
|
||||
@@ -107,7 +107,7 @@ describe('Database Client', () => {
|
||||
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
|
||||
.all() as { name: string }[];
|
||||
// Existing domain/config tables plus account, conversation, sync and media ownership stores.
|
||||
expect(tables.length).toBe(15);
|
||||
expect(tables.length).toBe(16);
|
||||
});
|
||||
|
||||
it('tracks schema version and migrates a legacy sent_messages table transactionally', () => {
|
||||
@@ -154,8 +154,9 @@ describe('Database Client', () => {
|
||||
db.exec(`
|
||||
DELETE FROM recipient_suppressions;
|
||||
DROP TABLE conversation_automation_claims;
|
||||
DROP TABLE ai_usage;
|
||||
`);
|
||||
db.pragma(`user_version = ${latestVersion - 1}`);
|
||||
db.pragma(`user_version = ${latestVersion - 2}`);
|
||||
runMigrations(db);
|
||||
|
||||
expect(db.prepare('SELECT recipient, reason FROM recipient_suppressions ORDER BY recipient COLLATE NOCASE').all())
|
||||
|
||||
@@ -703,6 +703,36 @@ describe('OpenRouterClient', () => {
|
||||
).rejects.toThrow('network');
|
||||
});
|
||||
});
|
||||
|
||||
it('records model, token, cost, latency, and outcome for each request attempt', async () => {
|
||||
const usageRecorder = vi.fn();
|
||||
const config = getTestConfig();
|
||||
const localClient = new OpenRouterClient(
|
||||
config,
|
||||
logger,
|
||||
{ chat: { completions: { create: mockCreate } } } as unknown as OpenAI,
|
||||
usageRecorder,
|
||||
);
|
||||
mockCreate.mockResolvedValueOnce({
|
||||
model: 'provider/model-a',
|
||||
choices: [{ message: { content: 'A useful reply' } }],
|
||||
usage: { prompt_tokens: 100, completion_tokens: 12, total_tokens: 112, cost: 0.0017 },
|
||||
});
|
||||
|
||||
await localClient.generateReply([{ sender: 'them', text: 'hello?', isSelf: false }], 'them');
|
||||
|
||||
expect(usageRecorder).toHaveBeenCalledWith(expect.objectContaining({
|
||||
operation: 'generateReply',
|
||||
requestedModel: config.openrouter.chatModel,
|
||||
actualModel: 'provider/model-a',
|
||||
outcome: 'success',
|
||||
promptTokens: 100,
|
||||
completionTokens: 12,
|
||||
totalTokens: 112,
|
||||
costUsd: 0.0017,
|
||||
latencyMs: expect.any(Number),
|
||||
}));
|
||||
});
|
||||
});
|
||||
|
||||
describe('stripDashes', () => {
|
||||
|
||||
@@ -70,4 +70,17 @@ describe('DeliveryOutboxRepository', () => {
|
||||
expect(outbox.markSent(delivery.id, 'worker-a', sentMessageId, 130)).toBe(true);
|
||||
expect(outbox.get(delivery.id)).toMatchObject({ status: 'sent', sentMessageId, leaseOwner: null });
|
||||
});
|
||||
|
||||
it('surfaces failed deliveries for the attention inbox', () => {
|
||||
const delivery = outbox.discover({
|
||||
postId: 'post-1', recipient: 'recipient', subject: 'subject', message: 'message', idempotencyKey: 'post-1:v1', now: 100,
|
||||
});
|
||||
outbox.markPrepared(delivery.id, 110);
|
||||
outbox.claim(delivery.id, 'worker-a', 50, 120);
|
||||
outbox.markFailed(delivery.id, 'worker-a', 'Reddit rejected the invitation', 130);
|
||||
|
||||
expect(outbox.getFailed()).toEqual([
|
||||
expect.objectContaining({ id: delivery.id, status: 'failed', lastError: 'Reddit rejected the invitation' }),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -607,6 +607,21 @@ describe('ConversationStateRepository', () => {
|
||||
expect(repo.get('!room:reddit.com')?.persona.age).toBe(34);
|
||||
expect(repo.get('missing')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('lists only durable boundary and human-handoff states for operator attention', () => {
|
||||
const db = createTestDb();
|
||||
const repo = new ConversationStateRepository(db);
|
||||
const active = buildConversationState([]);
|
||||
const boundary = buildConversationState([{ sender: 'them', text: 'Stop messaging me.', isSelf: false }]);
|
||||
boundary.automation.status = 'boundary';
|
||||
boundary.automation.handoffReason = 'Recipient asked us to stop.';
|
||||
repo.save('active-room', active);
|
||||
repo.save('boundary-room', boundary);
|
||||
|
||||
expect(repo.listAttention()).toEqual([
|
||||
expect.objectContaining({ conversationId: 'boundary-room', status: 'boundary', reason: 'Recipient asked us to stop.' }),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Mature performance and guardrails', () => {
|
||||
|
||||
Reference in new issue
Block a user