By subreddit
'subreddit' in row ? row.subreddit : row.strategyId} />
diff --git a/apps/dashboard/src/features/performance/performance-queries.ts b/apps/dashboard/src/features/performance/performance-queries.ts
index 376bcf5..0da4203 100644
--- a/apps/dashboard/src/features/performance/performance-queries.ts
+++ b/apps/dashboard/src/features/performance/performance-queries.ts
@@ -7,6 +7,7 @@ export const performanceQueryKeys = {
trend: (days: number) => ['performance', 'trend', days] as const,
successfulExamples: ['performance', 'successful-examples'] as const,
funnel: ['performance', 'funnel'] as const,
+ aiUsage: (days: number) => ['performance', 'ai-usage', days] as const,
};
export function useSubredditPerformanceQuery() {
@@ -43,3 +44,10 @@ export function useFunnelQuery() {
queryFn: performanceApi.getFunnel,
});
}
+
+export function useAiUsageQuery(days: number) {
+ return useQuery({
+ queryKey: performanceQueryKeys.aiUsage(days),
+ queryFn: () => performanceApi.getAiUsage(days),
+ });
+}
diff --git a/docs/api.md b/docs/api.md
index 9cec8a9..dcad456 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -86,6 +86,8 @@ an administrator credential.
| `GET` | `/api/analytics/by-strategy` | Get performance by strategy | read | no |
| `GET` | `/api/analytics/funnel` | Get mature message outcome funnel | read | no |
| `GET` | `/api/analytics/trend` | Get daily performance trend | read | no |
+| `GET` | `/api/analytics/ai-usage` | Get OpenRouter cost and model reliability | read | no |
+| `GET` | `/api/attention` | Get operator attention items | read | no |
| `GET` | `/api/config` | Get non-secret application configuration | read | no |
| `PUT` | `/api/config` | Update application configuration | admin | yes |
| `GET` | `/api/login/status` | Get Reddit session state | read | no |
diff --git a/packages/api-contracts/src/attention.ts b/packages/api-contracts/src/attention.ts
new file mode 100644
index 0000000..9aac0c7
--- /dev/null
+++ b/packages/api-contracts/src/attention.ts
@@ -0,0 +1,31 @@
+import { z } from 'zod';
+import { RedditAccountOwnerSchema } from './session.js';
+
+export const AttentionKindSchema = z.enum([
+ 'human-required',
+ 'boundary',
+ 'failed-delivery',
+ 'account-login-required',
+ 'runtime-failed',
+ 'invitation-cooldown',
+]);
+
+export const AttentionItemSchema = z.strictObject({
+ id: z.string(),
+ kind: AttentionKindSchema,
+ severity: z.enum(['warning', 'critical']),
+ title: z.string(),
+ detail: z.string(),
+ occurredAt: z.number(),
+ owner: RedditAccountOwnerSchema.optional(),
+ conversationId: z.string().optional(),
+ username: z.string().optional(),
+});
+
+export const AttentionResponseSchema = z.strictObject({
+ generatedAt: z.number(),
+ items: z.array(AttentionItemSchema),
+});
+
+export type AttentionItem = z.infer
;
+export type AttentionResponse = z.infer;
diff --git a/packages/api-contracts/src/index.ts b/packages/api-contracts/src/index.ts
index 9e2b99c..e10ed60 100644
--- a/packages/api-contracts/src/index.ts
+++ b/packages/api-contracts/src/index.ts
@@ -1,4 +1,5 @@
export * from './activity.js';
+export * from './attention.js';
export * from './chats.js';
export * from './common.js';
export * from './config.js';
diff --git a/packages/api-contracts/src/stats.ts b/packages/api-contracts/src/stats.ts
index e2ccfe6..d717117 100644
--- a/packages/api-contracts/src/stats.ts
+++ b/packages/api-contracts/src/stats.ts
@@ -74,5 +74,24 @@ export const FunnelResponseSchema = z.strictObject({
images: z.number(),
});
+const AiUsageFields = {
+ requests: z.number().int().nonnegative(),
+ successes: z.number().int().nonnegative(),
+ refusals: z.number().int().nonnegative(),
+ errors: z.number().int().nonnegative(),
+ promptTokens: z.number().int().nonnegative(),
+ completionTokens: z.number().int().nonnegative(),
+ totalTokens: z.number().int().nonnegative(),
+ costUsd: z.number().nonnegative(),
+ averageLatencyMs: z.number().nonnegative(),
+};
+
+export const AiUsageSummaryResponseSchema = z.strictObject({
+ days: z.number().int().positive(),
+ ...AiUsageFields,
+ models: z.array(z.strictObject({ model: z.string(), ...AiUsageFields })),
+});
+
export type StatsResponse = z.infer;
export type TimersResponse = z.infer;
+export type AiUsageSummaryResponse = z.infer;
diff --git a/src/ai/openrouter.ts b/src/ai/openrouter.ts
index e880f7d..c33d627 100644
--- a/src/ai/openrouter.ts
+++ b/src/ai/openrouter.ts
@@ -1,6 +1,7 @@
import OpenAI from 'openai';
import type { Config } from '../config/index.js';
import type { Logger } from '../utils/logger.js';
+import type { AiUsageRecordInput, AiUsageOutcome } from '../database/ai-usage-repository.js';
import {
formatConversationAsUntrustedData,
isSafeConversationReply,
@@ -29,6 +30,8 @@ interface ProtectedTextPrompt {
user: string;
}
+export type AiUsageRecorder = (usage: AiUsageRecordInput) => void;
+
export class OpenRouterClient {
private client: OpenAI;
@@ -36,6 +39,7 @@ export class OpenRouterClient {
private config: Config,
private logger: Logger,
client?: OpenAI,
+ private readonly usageRecorder?: AiUsageRecorder,
) {
this.client = client ?? new OpenAI({
baseURL: 'https://openrouter.ai/api/v1',
@@ -48,6 +52,7 @@ export class OpenRouterClient {
}
async detectGenderOnly(imageBase64: string, mimeType: string): Promise {
+ const startedAt = Date.now();
try {
this.logger.debug('Quick gender detection...');
@@ -74,12 +79,14 @@ export class OpenRouterClient {
});
const result = response.choices[0]?.message?.content?.toLowerCase().trim() || 'unknown';
+ this.recordUsage('detectGender', this.config.openrouter.imageAnalysisModel, startedAt, response, result ? 'success' : 'empty');
if (result.includes('female')) return 'female';
if (result.includes('male') && !result.includes('female')) return 'male';
if (result.includes('other')) return 'other';
return 'unknown';
} catch (error) {
+ this.recordError('detectGender', this.config.openrouter.imageAnalysisModel, startedAt, error);
this.logger.error('Gender detection failed:', error);
return 'unknown';
}
@@ -134,33 +141,40 @@ export class OpenRouterClient {
}
private async tryAnalyzeWithModel(model: string, imageBase64: string, mimeType: string): Promise {
+ const startedAt = Date.now();
const genderPrompt = `First, identify the gender of the person in this image. Respond with EXACTLY one of these on the first line: GENDER:male, GENDER:female, GENDER:other, or GENDER:unknown
Then provide your analysis:
${this.config.openrouter.imageAnalysisPrompt}`;
- const response = await this.client.chat.completions.create({
- model,
- messages: [
- {
- role: 'user',
- content: [
- {
- type: 'text',
- text: genderPrompt,
- },
- {
- type: 'image_url',
- image_url: {
- url: `data:${mimeType};base64,${imageBase64}`,
+ let response;
+ try {
+ response = await this.client.chat.completions.create({
+ model,
+ messages: [
+ {
+ role: 'user',
+ content: [
+ {
+ type: 'text',
+ text: genderPrompt,
},
- },
- ],
- },
- ],
- max_tokens: 600,
- });
+ {
+ type: 'image_url',
+ image_url: {
+ url: `data:${mimeType};base64,${imageBase64}`,
+ },
+ },
+ ],
+ },
+ ],
+ max_tokens: 600,
+ });
+ } catch (error) {
+ this.recordError('analyzeImage', model, startedAt, error);
+ throw error;
+ }
const fullResponse = response.choices[0]?.message?.content || '';
@@ -190,9 +204,12 @@ ${this.config.openrouter.imageAnalysisPrompt}`;
// Detect AI refusal - return null to signal fallback should be tried
if (this.isRefusal(description)) {
+ this.recordUsage('analyzeImage', model, startedAt, response, 'refusal');
return null;
}
+ this.recordUsage('analyzeImage', model, startedAt, response, description ? 'success' : 'empty');
+
this.logger.debug(`Image analysis complete (${model}) - Gender: ${detectedGender}, Rating: ${attractivenessScore ?? 'n/a'}`);
this.logger.debug(`Description: ${description}`);
@@ -480,7 +497,10 @@ Just write the message, nothing else.`;
if (isFallback) {
this.logger.info(`[AI] ${label} trying fallback model: ${model}`);
}
- const response = await this.client.chat.completions.create({
+ const startedAt = Date.now();
+ let response;
+ try {
+ response = await this.client.chat.completions.create({
model,
messages: typeof prompt === 'string'
? [{ role: 'user', content: prompt }]
@@ -490,7 +510,11 @@ Just write the message, nothing else.`;
],
max_tokens: opts.max_tokens,
temperature: opts.temperature,
- });
+ });
+ } catch (error) {
+ this.recordError(label, model, startedAt, error);
+ throw error;
+ }
const raw = response.choices[0]?.message?.content || '';
const choice = response.choices[0] as unknown as Record | undefined;
@@ -508,14 +532,18 @@ Just write the message, nothing else.`;
const trimmed = raw.trim();
if (!trimmed) {
+ this.recordUsage(label, model, startedAt, response, 'empty');
this.logger.warn(`[AI] ${label}: ${model} returned empty content${i < models.length - 1 ? ', trying fallback...' : ''}`);
continue;
}
if (this.isRefusal(trimmed)) {
+ this.recordUsage(label, model, startedAt, response, 'refusal');
this.logger.warn(`[AI] ${label}: ${model} returned a refusal${i < models.length - 1 ? ', trying fallback...' : ''}`);
continue;
}
+ this.recordUsage(label, model, startedAt, response, 'success');
+
if (isFallback) {
this.logger.info(`[AI] ${label}: fallback model ${model} succeeded`);
}
@@ -532,6 +560,48 @@ Just write the message, nothing else.`;
return '';
}
+ private recordUsage(
+ operation: string,
+ requestedModel: string,
+ startedAt: number,
+ response: unknown,
+ outcome: AiUsageOutcome,
+ ): void {
+ if (!this.usageRecorder) return;
+ const record = response as Record;
+ const usage = (record.usage ?? {}) as Record;
+ try {
+ this.usageRecorder({
+ operation,
+ requestedModel,
+ actualModel: typeof record.model === 'string' ? record.model : requestedModel,
+ outcome,
+ promptTokens: numericUsage(usage.prompt_tokens),
+ completionTokens: numericUsage(usage.completion_tokens),
+ totalTokens: numericUsage(usage.total_tokens),
+ costUsd: typeof usage.cost === 'number' && Number.isFinite(usage.cost) ? usage.cost : null,
+ latencyMs: Date.now() - startedAt,
+ });
+ } catch (error) {
+ this.logger.warn('Failed to persist AI usage accounting:', error);
+ }
+ }
+
+ private recordError(operation: string, requestedModel: string, startedAt: number, error: unknown): void {
+ if (!this.usageRecorder) return;
+ try {
+ this.usageRecorder({
+ operation,
+ requestedModel,
+ outcome: 'error',
+ latencyMs: Date.now() - startedAt,
+ error: error instanceof Error ? error.message : String(error),
+ });
+ } catch (recordingError) {
+ this.logger.warn('Failed to persist AI usage error:', recordingError);
+ }
+ }
+
async analyzeAndGenerateMessage(
imageBase64: string,
mimeType: string,
@@ -547,3 +617,7 @@ Just write the message, nothing else.`;
return { analysis, message };
}
}
+
+function numericUsage(value: unknown): number {
+ return typeof value === 'number' && Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0;
+}
diff --git a/src/bootstrap.ts b/src/bootstrap.ts
index f308945..e9a8b48 100644
--- a/src/bootstrap.ts
+++ b/src/bootstrap.ts
@@ -47,6 +47,7 @@ import type { Cookie } from 'playwright';
import { ChatCache } from './services/chat/cache.js';
import { RedditAccountRuntimeManager } from './services/reddit-account-runtime.js';
import { createConcreteRedditAccountRuntimeFactory } from './services/reddit-account-runtime-builder.js';
+import { AiUsageRepository } from './database/ai-usage-repository.js';
import { importLegacyAccountContent } from './database/legacy-content-import.js';
import { RecipientDeduplicator } from './services/recipient-deduplicator.js';
import { createConfigBackedSubredditPollingControl } from './services/subreddit-polling-control.js';
@@ -127,6 +128,7 @@ export async function bootstrap(
const recipientDeduplicator = new RecipientDeduplicator(new RecipientHistoryRepository(db));
const redditAccountsRepo = bootstrapAccountsRepo;
const accountContentRepo = new AccountContentRepository(db);
+ const aiUsageRepo = new AiUsageRepository(db);
const legacyImport = importLegacyAccountContent(db);
if (legacyImport.conversations > 0 || legacyImport.media > 0) {
logger.info(`Imported legacy unassigned content into SQLite (${legacyImport.conversations} conversations, ${legacyImport.media} media items)`);
@@ -168,7 +170,9 @@ export async function bootstrap(
const chatPage = await browserClient.getContext()!.newPage();
// AI + services
- const aiClient = new OpenRouterClient(config, logger);
+ const aiClient = new OpenRouterClient(config, logger, undefined, (usage) => {
+ aiUsageRepo.record({ ...usage, accountId: accountAuth.getSnapshot().accountId });
+ });
const monitorService = new MonitorService(
redditApi, postsRepo, recipientDeduplicator, config, logger, 1000,
createConfigBackedSubredditPollingControl(config, new ConfigStore(db), logger),
@@ -344,6 +348,8 @@ export async function bootstrap(
activityLogRepo,
logger,
broadcaster,
+ process.cwd(),
+ aiUsageRepo,
);
webServer.setChatService(chatService);
diff --git a/src/database/ai-usage-repository.ts b/src/database/ai-usage-repository.ts
new file mode 100644
index 0000000..b9196a7
--- /dev/null
+++ b/src/database/ai-usage-repository.ts
@@ -0,0 +1,130 @@
+import type Database from 'better-sqlite3';
+
+export type AiUsageOutcome = 'success' | 'refusal' | 'empty' | 'error';
+
+export interface AiUsageRecordInput {
+ accountId?: string | null;
+ operation: string;
+ requestedModel: string;
+ actualModel?: string | null;
+ outcome: AiUsageOutcome;
+ promptTokens?: number;
+ completionTokens?: number;
+ totalTokens?: number;
+ costUsd?: number | null;
+ latencyMs: number;
+ error?: string | null;
+ createdAt?: number;
+}
+
+export interface AiUsageModelSummary {
+ model: string;
+ requests: number;
+ successes: number;
+ refusals: number;
+ errors: number;
+ promptTokens: number;
+ completionTokens: number;
+ totalTokens: number;
+ costUsd: number;
+ averageLatencyMs: number;
+}
+
+export interface AiUsageSummary {
+ days: number;
+ requests: number;
+ successes: number;
+ refusals: number;
+ errors: number;
+ promptTokens: number;
+ completionTokens: number;
+ totalTokens: number;
+ costUsd: number;
+ averageLatencyMs: number;
+ models: AiUsageModelSummary[];
+}
+
+type SummaryRow = {
+ model?: string;
+ requests: number;
+ successes: number;
+ refusals: number;
+ errors: number;
+ prompt_tokens: number;
+ completion_tokens: number;
+ total_tokens: number;
+ cost_usd: number;
+ average_latency_ms: number;
+};
+
+/** Durable, prompt-free accounting for OpenRouter requests. */
+export class AiUsageRepository {
+ constructor(private readonly db: Database.Database) {}
+
+ record(input: AiUsageRecordInput): void {
+ this.db.prepare(`
+ INSERT INTO ai_usage (
+ account_id, operation, requested_model, actual_model, outcome,
+ prompt_tokens, completion_tokens, total_tokens, cost_usd,
+ latency_ms, error, created_at
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ `).run(
+ input.accountId ?? null,
+ input.operation,
+ input.requestedModel,
+ input.actualModel ?? null,
+ input.outcome,
+ input.promptTokens ?? 0,
+ input.completionTokens ?? 0,
+ input.totalTokens ?? 0,
+ input.costUsd ?? null,
+ Math.max(0, Math.round(input.latencyMs)),
+ input.error?.slice(0, 500) ?? null,
+ input.createdAt ?? Date.now(),
+ );
+ }
+
+ getSummary(days = 30, now = Date.now()): AiUsageSummary {
+ const safeDays = Math.max(1, Math.min(365, Math.trunc(days)));
+ const since = now - safeDays * 86_400_000;
+ const fields = `
+ COUNT(*) AS requests,
+ SUM(CASE WHEN outcome = 'success' THEN 1 ELSE 0 END) AS successes,
+ SUM(CASE WHEN outcome = 'refusal' THEN 1 ELSE 0 END) AS refusals,
+ SUM(CASE WHEN outcome IN ('empty', 'error') THEN 1 ELSE 0 END) AS errors,
+ COALESCE(SUM(prompt_tokens), 0) AS prompt_tokens,
+ COALESCE(SUM(completion_tokens), 0) AS completion_tokens,
+ COALESCE(SUM(total_tokens), 0) AS total_tokens,
+ COALESCE(SUM(cost_usd), 0) AS cost_usd,
+ COALESCE(ROUND(AVG(latency_ms)), 0) AS average_latency_ms
+ `;
+ const total = this.db.prepare(`SELECT ${fields} FROM ai_usage WHERE created_at >= ?`)
+ .get(since) as SummaryRow;
+ const models = this.db.prepare(`
+ SELECT COALESCE(actual_model, requested_model) AS model, ${fields}
+ FROM ai_usage
+ WHERE created_at >= ?
+ GROUP BY COALESCE(actual_model, requested_model)
+ ORDER BY cost_usd DESC, requests DESC, model ASC
+ `).all(since) as SummaryRow[];
+ return {
+ days: safeDays,
+ ...mapSummary(total),
+ models: models.map((row) => ({ model: row.model ?? 'unknown', ...mapSummary(row) })),
+ };
+ }
+}
+
+function mapSummary(row: SummaryRow): Omit {
+ return {
+ requests: Number(row.requests ?? 0),
+ successes: Number(row.successes ?? 0),
+ refusals: Number(row.refusals ?? 0),
+ errors: Number(row.errors ?? 0),
+ promptTokens: Number(row.prompt_tokens ?? 0),
+ completionTokens: Number(row.completion_tokens ?? 0),
+ totalTokens: Number(row.total_tokens ?? 0),
+ costUsd: Math.round(Number(row.cost_usd ?? 0) * 1_000_000_000_000) / 1_000_000_000_000,
+ averageLatencyMs: Number(row.average_latency_ms ?? 0),
+ };
+}
diff --git a/src/database/client.ts b/src/database/client.ts
index aeb5770..897c256 100644
--- a/src/database/client.ts
+++ b/src/database/client.ts
@@ -251,6 +251,27 @@ export function runMigrations(db: Database.Database): void {
CREATE INDEX idx_conversation_automation_claims_status
ON conversation_automation_claims(account_id, status, claimed_at);
`),
+ () => db.exec(`
+ CREATE TABLE IF NOT EXISTS ai_usage (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ account_id TEXT,
+ operation TEXT NOT NULL,
+ requested_model TEXT NOT NULL,
+ actual_model TEXT,
+ outcome TEXT NOT NULL CHECK(outcome IN ('success', 'refusal', 'empty', 'error')),
+ prompt_tokens INTEGER NOT NULL DEFAULT 0,
+ completion_tokens INTEGER NOT NULL DEFAULT 0,
+ total_tokens INTEGER NOT NULL DEFAULT 0,
+ cost_usd REAL,
+ latency_ms INTEGER NOT NULL,
+ error TEXT,
+ created_at INTEGER NOT NULL,
+ FOREIGN KEY (account_id) REFERENCES reddit_accounts(account_id) ON DELETE SET NULL
+ );
+ CREATE INDEX IF NOT EXISTS idx_ai_usage_created_at ON ai_usage(created_at DESC);
+ CREATE INDEX IF NOT EXISTS idx_ai_usage_model ON ai_usage(actual_model, requested_model, created_at DESC);
+ CREATE INDEX IF NOT EXISTS idx_ai_usage_account ON ai_usage(account_id, created_at DESC);
+ `),
];
let currentVersion = db.pragma('user_version', { simple: true }) as number;
diff --git a/src/database/repositories.ts b/src/database/repositories.ts
index 5dfafb7..1b2671b 100644
--- a/src/database/repositories.ts
+++ b/src/database/repositories.ts
@@ -227,6 +227,16 @@ export class DeliveryOutboxRepository {
.get(this.accountId) as { count: number }).count;
}
+ getFailed(limit = 50): DeliveryOutboxRecord[] {
+ const rows = this.db.prepare(`
+ SELECT * FROM delivery_outbox
+ WHERE account_id = ? AND status = 'failed'
+ ORDER BY updated_at DESC
+ LIMIT ?
+ `).all(this.accountId, limit) as Record[];
+ return rows.map((row) => this.mapRow(row));
+ }
+
markPrepared(id: number, now: number = Date.now()): boolean {
return this.db.prepare(`
UPDATE delivery_outbox SET status = 'prepared', updated_at = ?, last_error = NULL
@@ -1081,6 +1091,32 @@ export class ConversationStateRepository {
const row = this.getStmt.get(this.accountId, conversationId) as { state_json: string } | undefined;
return row ? parseConversationState(row.state_json) ?? undefined : undefined;
}
+
+ listAttention(): Array<{
+ conversationId: string;
+ status: 'human-required' | 'boundary';
+ reason: string | null;
+ updatedAt: number;
+ }> {
+ const rows = this.db.prepare(`
+ SELECT conversation_id, state_json, updated_at
+ FROM conversation_states
+ WHERE account_id = ?
+ ORDER BY updated_at DESC
+ `).all(this.accountId) as Array<{ conversation_id: string; state_json: string; updated_at: number }>;
+ return rows.flatMap((row) => {
+ const state = parseConversationState(row.state_json);
+ if (!state) return [];
+ const status = state.automation.status;
+ if (status !== 'human-required' && status !== 'boundary') return [];
+ return [{
+ conversationId: row.conversation_id,
+ status,
+ reason: state.automation.handoffReason,
+ updatedAt: row.updated_at,
+ }];
+ });
+ }
}
/** Durable idempotency claims for AI replies to individual inbound events. */
diff --git a/src/services/reddit-account-runtime-builder.ts b/src/services/reddit-account-runtime-builder.ts
index 5046abb..f79d446 100644
--- a/src/services/reddit-account-runtime-builder.ts
+++ b/src/services/reddit-account-runtime-builder.ts
@@ -43,6 +43,7 @@ import { RecipientDeduplicator } from './recipient-deduplicator.js';
import { ConfigStore } from '../config/persistence.js';
import { createConfigBackedSubredditPollingControl } from './subreddit-polling-control.js';
import { ConversationAutomationService } from './conversation-automation.js';
+import { AiUsageRepository } from '../database/ai-usage-repository.js';
export interface ConcreteRuntimeFactoryOptions {
config: Config;
@@ -118,7 +119,10 @@ async function buildRuntime(
const redditApi = new RedditApiClient(accountConfig, logger, redditRequestCoordinator);
redditApi.setBrowserContext(browserClient.getContext()!);
const chatPage = await browserClient.getContext()!.newPage();
- const aiClient = new OpenRouterClient(accountConfig, logger);
+ const aiUsageRepo = new AiUsageRepository(db);
+ const aiClient = new OpenRouterClient(accountConfig, logger, undefined, (usage) => {
+ aiUsageRepo.record({ ...usage, accountId: session.accountId });
+ });
const monitorService = new MonitorService(
redditApi, postsRepo, recipientDeduplicator, accountConfig, logger,
1000, createConfigBackedSubredditPollingControl(config, new ConfigStore(db), logger),
diff --git a/src/web/api-catalog.ts b/src/web/api-catalog.ts
index f27cded..69896bf 100644
--- a/src/web/api-catalog.ts
+++ b/src/web/api-catalog.ts
@@ -94,6 +94,8 @@ export const API_OPERATIONS: readonly ApiOperation[] = [
operation('getStrategyAnalytics', 'GET', '/api/analytics/by-strategy', 'Get performance by strategy', ['Analytics']),
operation('getFunnelAnalytics', 'GET', '/api/analytics/funnel', 'Get mature message outcome funnel', ['Analytics']),
operation('getAnalyticsTrend', 'GET', '/api/analytics/trend', 'Get daily performance trend', ['Analytics']),
+ operation('getAiUsageAnalytics', 'GET', '/api/analytics/ai-usage', 'Get OpenRouter cost and model reliability', ['Analytics']),
+ operation('getAttention', 'GET', '/api/attention', 'Get operator attention items', ['Operations']),
operation('getConfig', 'GET', '/api/config', 'Get non-secret application configuration', ['Configuration']),
operation('updateConfig', 'PUT', '/api/config', 'Update application configuration', ['Configuration']),
operation('getLoginStatus', 'GET', '/api/login/status', 'Get Reddit session state', ['Accounts']),
diff --git a/src/web/context.ts b/src/web/context.ts
index 3c51dc8..935c762 100644
--- a/src/web/context.ts
+++ b/src/web/context.ts
@@ -22,6 +22,7 @@ import type { AccountAuthContext } from '../services/account-auth-context.js';
import type { ConfigStore } from '../config/persistence.js';
import type { RedditAccountsService } from '../services/reddit-accounts.js';
import type { RedditAccountRuntimeManager } from '../services/reddit-account-runtime.js';
+import type { AiUsageRepository } from '../database/ai-usage-repository.js';
/** Bag of everything the route modules might need. */
export interface WebContext {
@@ -35,6 +36,7 @@ export interface WebContext {
posts: PostsRepository;
messages: MessagesRepository;
successful: SuccessfulMessagesRepository;
+ aiUsage?: AiUsageRepository;
};
services: {
diff --git a/src/web/routes/analytics.ts b/src/web/routes/analytics.ts
index a520f4a..d432678 100644
--- a/src/web/routes/analytics.ts
+++ b/src/web/routes/analytics.ts
@@ -37,5 +37,14 @@ export function createAnalyticsRouter(ctx: WebContext): Router {
res.json(ctx.repos.messages.getResponseRateTrend(days));
});
+ router.get('/analytics/ai-usage', (req, res) => {
+ if (!ctx.repos.aiUsage) {
+ res.status(503).json({ error: 'AI usage accounting is not initialized' });
+ return;
+ }
+ const days = parseInt(req.query.days as string) || 30;
+ res.json(ctx.repos.aiUsage.getSummary(days));
+ });
+
return router;
}
diff --git a/src/web/routes/attention.ts b/src/web/routes/attention.ts
new file mode 100644
index 0000000..348b1f6
--- /dev/null
+++ b/src/web/routes/attention.ts
@@ -0,0 +1,103 @@
+import { Router } from 'express';
+import type { WebContext } from '../context.js';
+
+interface AttentionItem {
+ id: string;
+ kind: 'human-required' | 'boundary' | 'failed-delivery' | 'account-login-required' | 'runtime-failed' | 'invitation-cooldown';
+ severity: 'warning' | 'critical';
+ title: string;
+ detail: string;
+ occurredAt: number;
+ owner?: { accountId: string; redditUsername: string };
+ conversationId?: string;
+ username?: string;
+}
+
+export function createAttentionRouter(ctx: WebContext): Router {
+ const router = Router();
+
+ router.get('/attention', async (_req, res) => {
+ const items: AttentionItem[] = [];
+ const accounts = ctx.services.redditAccounts?.list() ?? [];
+ for (const account of accounts) {
+ if (account.status !== 'login-required') continue;
+ items.push({
+ id: `account:${account.accountId}`,
+ kind: 'account-login-required',
+ severity: 'critical',
+ title: `u/${account.redditUsername} needs a new session`,
+ detail: 'Import a fresh extension session before this account can resume.',
+ occurredAt: account.updatedAt,
+ owner: { accountId: account.accountId, redditUsername: account.redditUsername },
+ });
+ }
+
+ const snapshots = ctx.services.accountRuntimes?.list() ?? [];
+ for (const snapshot of snapshots) {
+ const owner = { accountId: snapshot.accountId, redditUsername: snapshot.redditUsername };
+ if (snapshot.state === 'failed') {
+ items.push({
+ id: `runtime:${snapshot.accountId}`,
+ kind: 'runtime-failed',
+ severity: 'critical',
+ title: `u/${snapshot.redditUsername} runtime failed`,
+ detail: snapshot.failure ?? 'The account runtime stopped unexpectedly.',
+ occurredAt: snapshot.updatedAt,
+ owner,
+ });
+ }
+ const runtime = ctx.services.accountRuntimes?.get(snapshot.accountId);
+ if (!runtime) continue;
+ const conversations = await runtime.resources.chatService.getCachedConversations();
+ const usernames = new Map(conversations.flatMap((conversation) => conversation.roomId
+ ? [[conversation.roomId, conversation.username] as const]
+ : []));
+ for (const state of runtime.resources.conversationStateRepo.listAttention()) {
+ const username = usernames.get(state.conversationId);
+ items.push({
+ id: `conversation:${snapshot.accountId}:${state.conversationId}:${state.status}`,
+ kind: state.status,
+ severity: state.status === 'boundary' ? 'critical' : 'warning',
+ title: state.status === 'boundary'
+ ? `Boundary detected${username ? ` from u/${username}` : ''}`
+ : `Human follow-up required${username ? ` for u/${username}` : ''}`,
+ detail: state.reason ?? (state.status === 'boundary'
+ ? 'Automation stopped after the recipient set a boundary.'
+ : 'Automated conversation turns are exhausted.'),
+ occurredAt: state.updatedAt,
+ owner,
+ conversationId: `${snapshot.accountId}~${state.conversationId}`,
+ ...(username ? { username } : {}),
+ });
+ }
+ for (const delivery of runtime.resources.deliveryOutboxRepo.getFailed()) {
+ items.push({
+ id: `delivery:${snapshot.accountId}:${delivery.id}`,
+ kind: 'failed-delivery',
+ severity: 'warning',
+ title: `Message to u/${delivery.recipient} failed`,
+ detail: delivery.lastError ?? `Delivery failed after ${delivery.attemptCount} attempt(s).`,
+ occurredAt: delivery.updatedAt,
+ owner,
+ username: delivery.recipient,
+ });
+ }
+ const cooldown = runtime.resources.chatService.getInvitationCooldown();
+ if (cooldown) {
+ items.push({
+ id: `cooldown:${snapshot.accountId}:${cooldown.until}`,
+ kind: 'invitation-cooldown',
+ severity: 'warning',
+ title: `u/${snapshot.redditUsername} invitation cooldown`,
+ detail: cooldown.reason,
+ occurredAt: Date.now(),
+ owner,
+ });
+ }
+ }
+
+ res.json({ generatedAt: Date.now(), items: items.sort((left, right) => right.occurredAt - left.occurredAt) });
+ });
+
+ return router;
+}
diff --git a/src/web/server.ts b/src/web/server.ts
index 600ed1d..18b3b10 100644
--- a/src/web/server.ts
+++ b/src/web/server.ts
@@ -41,6 +41,8 @@ import { createLoginRouter } from './routes/login.js';
import { createDashboardAuth, createDashboardSessionRouter } from './auth.js';
import { resolveDashboardAssets } from './dashboard-assets.js';
import { createAgentRouter, createHealthRouter } from './routes/agent.js';
+import { createAttentionRouter } from './routes/attention.js';
+import type { AiUsageRepository } from '../database/ai-usage-repository.js';
export type { BotStats, ActivityItem, LogEntry, RuntimeConfig } from './types.js';
@@ -74,6 +76,7 @@ export class WebServer {
private logger: Logger,
broadcaster?: LogBroadcaster,
private workspaceDirectory = process.cwd(),
+ private aiUsageRepo?: AiUsageRepository,
) {
this.broadcaster = broadcaster ?? new LogBroadcaster();
this.activity = new ActivityLog(activityLogRepo);
@@ -104,6 +107,7 @@ export class WebServer {
posts: this.postsRepo,
messages: this.messagesRepo,
successful: this.successfulMessagesRepo,
+ aiUsage: this.aiUsageRepo,
},
services: this.services,
getBotRunning: () => this.botRunning,
@@ -147,6 +151,7 @@ export class WebServer {
const ctx = this.context();
this.app.use('/api', createAgentRouter(ctx));
this.app.use('/api', createStatsRouter(ctx));
+ this.app.use('/api', createAttentionRouter(ctx));
this.app.use('/api', createChatsRouter(ctx));
this.app.use('/api', createConversationsRouter(ctx));
this.app.use('/api', createMediaRouter(ctx));
diff --git a/tests/ai-usage-repository.test.ts b/tests/ai-usage-repository.test.ts
new file mode 100644
index 0000000..c6c6488
--- /dev/null
+++ b/tests/ai-usage-repository.test.ts
@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'vitest';
+import { AiUsageRepository } from '../src/database/ai-usage-repository.js';
+import { createTestDb } from './testUtils.js';
+
+describe('AiUsageRepository', () => {
+ it('aggregates cost, tokens, outcomes, and latency by actual model', () => {
+ const db = createTestDb();
+ const repo = new AiUsageRepository(db);
+ const now = Date.parse('2026-08-09T10:00:00Z');
+ repo.record({ operation: 'generateReply', requestedModel: 'router', actualModel: 'model-a', outcome: 'success', promptTokens: 100, completionTokens: 20, totalTokens: 120, costUsd: 0.0012, latencyMs: 400, createdAt: now });
+ repo.record({ operation: 'generateReply', requestedModel: 'model-a', actualModel: 'model-a', outcome: 'refusal', promptTokens: 40, completionTokens: 5, totalTokens: 45, costUsd: 0.0003, latencyMs: 200, createdAt: now });
+ repo.record({ operation: 'analyzeImage', requestedModel: 'model-b', outcome: 'error', latencyMs: 900, error: 'provider unavailable', createdAt: now });
+
+ expect(repo.getSummary(30, now)).toEqual({
+ days: 30,
+ requests: 3,
+ successes: 1,
+ refusals: 1,
+ errors: 1,
+ promptTokens: 140,
+ completionTokens: 25,
+ totalTokens: 165,
+ costUsd: 0.0015,
+ averageLatencyMs: 500,
+ models: [
+ { model: 'model-a', requests: 2, successes: 1, refusals: 1, errors: 0, promptTokens: 140, completionTokens: 25, totalTokens: 165, costUsd: 0.0015, averageLatencyMs: 300 },
+ { model: 'model-b', requests: 1, successes: 0, refusals: 0, errors: 1, promptTokens: 0, completionTokens: 0, totalTokens: 0, costUsd: 0, averageLatencyMs: 900 },
+ ],
+ });
+ db.close();
+ });
+});
diff --git a/tests/dashboard/app.test.tsx b/tests/dashboard/app.test.tsx
index c136224..2dee6d0 100644
--- a/tests/dashboard/app.test.tsx
+++ b/tests/dashboard/app.test.tsx
@@ -79,7 +79,7 @@ describe('dashboard foundation', () => {
const { router } = renderDashboard('/');
const navigation = screen.getByRole('navigation', { name: 'Primary navigation' });
- for (const label of ['Overview', 'Conversations', 'Media', 'Performance', 'Operations', 'Settings']) {
+ for (const label of ['Overview', 'Needs attention', 'Conversations', 'Media', 'Performance', 'Operations', 'Settings']) {
expect(within(navigation).getByRole('link', { name: label })).toBeInTheDocument();
}
@@ -91,6 +91,24 @@ describe('dashboard foundation', () => {
expect(screen.getByRole('heading', { name: 'Operations' })).toBeInTheDocument();
});
+ it('shows actionable attention items and links handoffs to their conversation', async () => {
+ installSuccessfulApi();
+ server.use(http.get('/api/attention', () => HttpResponse.json({
+ generatedAt: Date.parse('2026-08-09T08:00:00Z'),
+ items: [{
+ id: 'conversation:one', kind: 'human-required', severity: 'warning',
+ title: 'Human follow-up required for u/alice', detail: 'Automated turns are exhausted.',
+ occurredAt: Date.parse('2026-08-09T07:00:00Z'),
+ owner: { accountId: '@one:reddit.com', redditUsername: 'operator' },
+ conversationId: '@one:reddit.com~!room:reddit.com', username: 'alice',
+ }],
+ })));
+ renderDashboard('/attention');
+ expect(await screen.findByRole('heading', { name: 'Needs attention' })).toBeInTheDocument();
+ expect(screen.getByText('Human follow-up required for u/alice')).toBeInTheDocument();
+ expect(screen.getByRole('link', { name: 'Open conversation' })).toHaveAttribute('href', '/conversations/%40one%3Areddit.com~!room%3Areddit.com');
+ });
+
it('hydrates media filters from the routed URL', async () => {
installSuccessfulApi();
server.use(
diff --git a/tests/dashboard/performance-operations.test.tsx b/tests/dashboard/performance-operations.test.tsx
index c0da0db..47c219d 100644
--- a/tests/dashboard/performance-operations.test.tsx
+++ b/tests/dashboard/performance-operations.test.tsx
@@ -83,12 +83,24 @@ describe('PerformancePage', () => {
upsell: 3,
images: 2,
})),
+ http.get('/api/analytics/ai-usage', ({ request }) => {
+ expect(new URL(request.url).searchParams.get('days')).toBe('30');
+ return HttpResponse.json({
+ days: 30, requests: 12, successes: 9, refusals: 2, errors: 1,
+ promptTokens: 1200, completionTokens: 300, totalTokens: 1500,
+ costUsd: 0.0123, averageLatencyMs: 420,
+ models: [{ model: 'model-a', requests: 12, successes: 9, refusals: 2, errors: 1, promptTokens: 1200, completionTokens: 300, totalTokens: 1500, costUsd: 0.0123, averageLatencyMs: 420 }],
+ });
+ }),
);
renderPage();
expect(await screen.findByRole('heading', { name: 'Performance' })).toBeInTheDocument();
expect(screen.getByRole('heading', { name: 'Outcome funnel' })).toBeInTheDocument();
+ expect(screen.getByRole('heading', { name: 'AI cost and reliability' })).toBeInTheDocument();
+ expect(screen.getByRole('table', { name: 'AI usage by model' })).toHaveTextContent('model-a');
+ expect(screen.getByLabelText('AI usage totals')).toHaveTextContent('$0.0123');
expect(screen.getByText('5 awaiting maturity')).toBeInTheDocument();
const subredditTable = screen.getByRole('table', { name: 'Subreddit performance' });
expect(within(subredditTable).getByText('n = 40')).toBeInTheDocument();
diff --git a/tests/database.test.ts b/tests/database.test.ts
index 55ae259..6ebfae7 100644
--- a/tests/database.test.ts
+++ b/tests/database.test.ts
@@ -107,7 +107,7 @@ describe('Database Client', () => {
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
.all() as { name: string }[];
// Existing domain/config tables plus account, conversation, sync and media ownership stores.
- expect(tables.length).toBe(15);
+ expect(tables.length).toBe(16);
});
it('tracks schema version and migrates a legacy sent_messages table transactionally', () => {
@@ -154,8 +154,9 @@ describe('Database Client', () => {
db.exec(`
DELETE FROM recipient_suppressions;
DROP TABLE conversation_automation_claims;
+ DROP TABLE ai_usage;
`);
- db.pragma(`user_version = ${latestVersion - 1}`);
+ db.pragma(`user_version = ${latestVersion - 2}`);
runMigrations(db);
expect(db.prepare('SELECT recipient, reason FROM recipient_suppressions ORDER BY recipient COLLATE NOCASE').all())
diff --git a/tests/openrouter-client.test.ts b/tests/openrouter-client.test.ts
index c11cfe4..f37f9f0 100644
--- a/tests/openrouter-client.test.ts
+++ b/tests/openrouter-client.test.ts
@@ -703,6 +703,36 @@ describe('OpenRouterClient', () => {
).rejects.toThrow('network');
});
});
+
+ it('records model, token, cost, latency, and outcome for each request attempt', async () => {
+ const usageRecorder = vi.fn();
+ const config = getTestConfig();
+ const localClient = new OpenRouterClient(
+ config,
+ logger,
+ { chat: { completions: { create: mockCreate } } } as unknown as OpenAI,
+ usageRecorder,
+ );
+ mockCreate.mockResolvedValueOnce({
+ model: 'provider/model-a',
+ choices: [{ message: { content: 'A useful reply' } }],
+ usage: { prompt_tokens: 100, completion_tokens: 12, total_tokens: 112, cost: 0.0017 },
+ });
+
+ await localClient.generateReply([{ sender: 'them', text: 'hello?', isSelf: false }], 'them');
+
+ expect(usageRecorder).toHaveBeenCalledWith(expect.objectContaining({
+ operation: 'generateReply',
+ requestedModel: config.openrouter.chatModel,
+ actualModel: 'provider/model-a',
+ outcome: 'success',
+ promptTokens: 100,
+ completionTokens: 12,
+ totalTokens: 112,
+ costUsd: 0.0017,
+ latencyMs: expect.any(Number),
+ }));
+ });
});
describe('stripDashes', () => {
diff --git a/tests/outbox.test.ts b/tests/outbox.test.ts
index cd6ee45..a083d05 100644
--- a/tests/outbox.test.ts
+++ b/tests/outbox.test.ts
@@ -70,4 +70,17 @@ describe('DeliveryOutboxRepository', () => {
expect(outbox.markSent(delivery.id, 'worker-a', sentMessageId, 130)).toBe(true);
expect(outbox.get(delivery.id)).toMatchObject({ status: 'sent', sentMessageId, leaseOwner: null });
});
+
+ it('surfaces failed deliveries for the attention inbox', () => {
+ const delivery = outbox.discover({
+ postId: 'post-1', recipient: 'recipient', subject: 'subject', message: 'message', idempotencyKey: 'post-1:v1', now: 100,
+ });
+ outbox.markPrepared(delivery.id, 110);
+ outbox.claim(delivery.id, 'worker-a', 50, 120);
+ outbox.markFailed(delivery.id, 'worker-a', 'Reddit rejected the invitation', 130);
+
+ expect(outbox.getFailed()).toEqual([
+ expect.objectContaining({ id: delivery.id, status: 'failed', lastError: 'Reddit rejected the invitation' }),
+ ]);
+ });
});
diff --git a/tests/repositories.test.ts b/tests/repositories.test.ts
index c7943b3..90624cd 100644
--- a/tests/repositories.test.ts
+++ b/tests/repositories.test.ts
@@ -607,6 +607,21 @@ describe('ConversationStateRepository', () => {
expect(repo.get('!room:reddit.com')?.persona.age).toBe(34);
expect(repo.get('missing')).toBeUndefined();
});
+
+ it('lists only durable boundary and human-handoff states for operator attention', () => {
+ const db = createTestDb();
+ const repo = new ConversationStateRepository(db);
+ const active = buildConversationState([]);
+ const boundary = buildConversationState([{ sender: 'them', text: 'Stop messaging me.', isSelf: false }]);
+ boundary.automation.status = 'boundary';
+ boundary.automation.handoffReason = 'Recipient asked us to stop.';
+ repo.save('active-room', active);
+ repo.save('boundary-room', boundary);
+
+ expect(repo.listAttention()).toEqual([
+ expect.objectContaining({ conversationId: 'boundary-room', status: 'boundary', reason: 'Recipient asked us to stop.' }),
+ ]);
+ });
});
describe('Mature performance and guardrails', () => {