Update source code and remove runtime data from tracking

Add gitignore rules for browser session data, SQLite databases,
media downloads, and build artifacts. Remove previously tracked
runtime files from the repository.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jjbruntonandClaude Opus 4.6 committed 2026-05-20 18:54:07 +01:00
1 parent e90213d1ff
commit dfa0580859
23 files changed
+5324 -347

No files matched your search

+600 -5
View File
@@ -1,11 +1,18 @@
import express from 'express';
import path from 'path';
import type { Server } from 'http';
import type { Response } from 'express';
import type { Logger } from '../utils/logger.js';
import type { PostsRepository, MessagesRepository, SuccessfulMessagesRepository } from '../database/repositories.js';
import type { Config } from '../config/index.js';
import type { ChatService, ActiveChat } from '../services/chat.js';
import type { MediaService } from '../services/media.js';
import type { MediaSyncService } from '../services/media-sync.js';
import type { ResponseCheckerService } from '../services/response-checker.js';
import type { RateLimiter } from '../utils/rate-limiter.js';
import type { OpenRouterClient } from '../ai/openrouter.js';
import type { AutoReplyService } from '../services/auto-reply.js';
import type { StrategySelector } from '../services/strategy-selector.js';
export interface BotStats {
isRunning: boolean;
@@ -23,19 +30,48 @@ export interface BotStats {
export interface ActivityItem {
timestamp: number;
type: 'sent' | 'skipped' | 'response' | 'error';
type: 'sent' | 'skipped' | 'response' | 'error' | 'info' | 'debug';
message: string;
recipient?: string;
imageUrl?: string;
}
export interface LogEntry {
timestamp: number;
level: 'info' | 'warn' | 'error' | 'debug';
message: string;
data?: unknown;
}
export interface RuntimeConfig {
rateLimit: {
minDelayBetweenMessagesMs: number;
maxMessagesPerHour: number;
jitterMs: number;
};
browser: {
dryRun: boolean;
};
}
export class WebServer {
private app: express.Application;
private server: Server | null = null;
private activityLog: ActivityItem[] = [];
private realtimeLog: LogEntry[] = [];
private sseClients: Set<Response> = new Set();
private botRunning = false; // Start stopped
private messagesThisHour = 0;
private chatService: ChatService | null = null;
private mediaService: MediaService | null = null;
private mediaSyncService: MediaSyncService | null = null;
private responseChecker: ResponseCheckerService | null = null;
private rateLimiter: RateLimiter | null = null;
private aiClient: OpenRouterClient | null = null;
private autoReplyService: AutoReplyService | null = null;
private strategySelector: StrategySelector | null = null;
private runtimeConfig: RuntimeConfig;
private onConfigChange: ((config: RuntimeConfig) => void) | null = null;
constructor(
private port: number,
@@ -46,9 +82,21 @@ export class WebServer {
private logger: Logger
) {
this.app = express();
// Initialize runtime config from loaded config
this.runtimeConfig = {
rateLimit: {
minDelayBetweenMessagesMs: config.rateLimit.minDelayBetweenMessagesMs,
maxMessagesPerHour: config.rateLimit.maxMessagesPerHour,
jitterMs: config.rateLimit.jitterMs,
},
browser: {
dryRun: config.browser.dryRun,
},
};
this.setupRoutes();
}
private setupRoutes(): void {
this.app.use('/media', express.static(path.join(process.cwd(), 'data', 'media')));
// Serve static files
this.app.use(express.static(path.join(process.cwd(), 'public')));
this.app.use(express.json());
@@ -64,6 +112,12 @@ export class WebServer {
res.json(this.activityLog.slice(-50).reverse());
});
// API: Look up posts by author username
this.app.get('/api/posts/by-author/:username', (req, res) => {
const posts = this.postsRepo.getByAuthor(req.params.username);
res.json(posts);
});
// API: Get recent messages
this.app.get('/api/messages', (req, res) => {
const messages = this.messagesRepo.getRecent(20);
@@ -76,21 +130,347 @@ export class WebServer {
res.json(successful);
});
// API: Get active chats (last 48 hours)
// API: Get active chats (from cache - last 48 hours)
// Does NOT fetch from Reddit - use /api/chats/fetch-all to refresh from Reddit
this.app.get('/api/chats', async (req, res) => {
if (!this.chatService) {
res.json([]);
return;
}
try {
const chats = await this.chatService.getActiveChats();
res.json(chats);
const allChats = await this.chatService.getCachedConversations();
// Filter to last 48 hours for "active" view
const cutoffTime = Date.now() - (48 * 60 * 60 * 1000);
const activeChats = allChats.filter(chat => chat.lastMessageTime >= cutoffTime);
res.json(activeChats);
} catch (error) {
this.logger.error('Failed to fetch chats:', error);
res.json([]);
}
});
// API: Fetch all chats/messages and refresh cache
this.app.post('/api/chats/fetch-all', async (req, res) => {
if (!this.chatService) {
res.status(500).json({ error: 'Chat service not initialized' });
return;
}
try {
const chats = await this.chatService.fetchAllMessages();
res.json({ success: true, count: chats.length });
} catch (error) {
this.logger.error('Failed to fetch all chats:', error);
res.status(500).json({ error: String(error) });
}
});
// API: Get cached conversations with full message history
this.app.get('/api/conversations', async (req, res) => {
if (!this.chatService) {
res.json([]);
return;
}
try {
const conversations = await this.chatService.getCachedConversations();
res.json(conversations);
} catch (error) {
this.logger.error('Failed to fetch conversations:', error);
res.json([]);
}
});
// API: Get a single cached conversation by room ID or username
this.app.get('/api/conversations/:id', async (req, res) => {
if (!this.chatService) {
res.status(404).json({ error: 'Chat service not initialized' });
return;
}
try {
const conversations = await this.chatService.getCachedConversations();
const id = decodeURIComponent(req.params.id);
const conversation = conversations.find(
(c: ActiveChat) => c.roomId === id || c.username === id || c.roomName === id
);
if (!conversation) {
res.status(404).json({ error: 'Conversation not found' });
return;
}
res.json(conversation);
} catch (error) {
this.logger.error('Failed to fetch conversation:', error);
res.status(500).json({ error: String(error) });
}
});
// API: Suggest a reply for a conversation using AI
this.app.post('/api/conversations/:id/suggest-reply', async (req, res) => {
if (!this.chatService) {
res.status(500).json({ error: 'Chat service not initialized' });
return;
}
if (!this.aiClient) {
res.status(500).json({ error: 'AI client not initialized' });
return;
}
try {
const conversations = await this.chatService.getCachedConversations();
const id = decodeURIComponent(req.params.id);
const conversation = conversations.find(
(c: ActiveChat) => c.roomId === id || c.username === id || c.roomName === id
);
if (!conversation) {
res.status(404).json({ error: 'Conversation not found' });
return;
}
if (!conversation.messages || conversation.messages.length === 0) {
res.status(400).json({ error: 'No messages in conversation to generate a reply from' });
return;
}
const hasReply = conversation.messages.some(m => !m.isSelf);
if (!hasReply) {
res.status(400).json({ error: 'No reply from user yet - nothing to respond to' });
return;
}
const suggestedReply = await this.aiClient.generateReply(
conversation.messages.map(m => ({ sender: m.sender, text: m.text, isSelf: m.isSelf })),
conversation.username
);
res.json({ suggestedReply });
} catch (error) {
this.logger.error('Failed to generate suggested reply:', error);
res.status(500).json({ error: String(error) });
}
});
// API: Send a reply in a chat conversation
this.app.post('/api/conversations/:id/send-reply', async (req, res) => {
if (!this.chatService) {
res.status(500).json({ error: 'Chat service not initialized' });
return;
}
try {
const { message } = req.body;
if (!message || typeof message !== 'string' || message.trim().length === 0) {
res.status(400).json({ error: 'Message is required' });
return;
}
const conversations = await this.chatService.getCachedConversations();
const id = decodeURIComponent(req.params.id);
const conversation = conversations.find(
(c: ActiveChat) => c.roomId === id || c.username === id || c.roomName === id
);
if (!conversation) {
res.status(404).json({ error: 'Conversation not found' });
return;
}
if (!conversation.url) {
res.status(400).json({ error: 'No chat URL for this conversation' });
return;
}
const success = await this.chatService.sendChatMessage(conversation.url, message.trim());
if (success) {
this.logActivity('sent', `Sent chat reply to u/${conversation.username}`, conversation.username);
this.pushLog('info', `Chat reply sent to u/${conversation.username}`);
res.json({ success: true });
// Refresh chats in background so cache reflects the sent message
this.chatService.refreshTopChats(5).catch(err =>
this.logger.error('Failed to refresh chats after send-reply:', err)
);
} else {
res.status(500).json({ error: 'Failed to send chat message' });
}
} catch (error) {
this.logger.error('Failed to send chat reply:', error);
res.status(500).json({ error: String(error) });
}
});
// API: Approve a pending auto-reply
this.app.post('/api/conversations/:id/approve-reply', async (req, res) => {
if (!this.chatService) {
res.status(500).json({ error: 'Chat service not initialized' });
return;
}
try {
const conversations = await this.chatService.getCachedConversations();
const id = decodeURIComponent(req.params.id);
const conversation = conversations.find(
(c: ActiveChat) => c.roomId === id || c.username === id || c.roomName === id
);
if (!conversation) {
res.status(404).json({ error: 'Conversation not found' });
return;
}
if (conversation.replyStatus !== 'pending' || !conversation.pendingReply) {
res.status(400).json({ error: 'No pending reply to approve' });
return;
}
if (!conversation.url) {
res.status(400).json({ error: 'No chat URL for this conversation' });
return;
}
const message = conversation.pendingReply;
const success = await this.chatService.sendChatMessage(conversation.url, message);
if (success) {
conversation.replyStatus = 'sent';
conversation.pendingReply = undefined;
await this.chatService.saveCacheExternal();
this.logActivity('sent', `Approved & sent auto-reply to u/${conversation.username}`, conversation.username);
this.pushLog('info', `Auto-reply approved and sent to u/${conversation.username}`);
res.json({ success: true });
// Refresh chats in background so cache reflects the sent message
this.chatService.refreshTopChats(5).catch(err =>
this.logger.error('Failed to refresh chats after approve-reply:', err)
);
} else {
res.status(500).json({ error: 'Failed to send message' });
}
} catch (error) {
this.logger.error('Failed to approve reply:', error);
res.status(500).json({ error: String(error) });
}
});
// API: Reject a pending auto-reply
this.app.post('/api/conversations/:id/reject-reply', async (req, res) => {
if (!this.chatService) {
res.status(500).json({ error: 'Chat service not initialized' });
return;
}
try {
const conversations = await this.chatService.getCachedConversations();
const id = decodeURIComponent(req.params.id);
const conversation = conversations.find(
(c: ActiveChat) => c.roomId === id || c.username === id || c.roomName === id
);
if (!conversation) {
res.status(404).json({ error: 'Conversation not found' });
return;
}
if (conversation.replyStatus !== 'pending') {
res.status(400).json({ error: 'No pending reply to reject' });
return;
}
conversation.pendingReply = undefined;
conversation.replyStatus = undefined;
// Clear processed state so it can be re-evaluated later if needed
if (this.autoReplyService) {
this.autoReplyService.clearProcessedState(conversation.username);
}
await this.chatService.saveCacheExternal();
this.logActivity('info', `Rejected auto-reply for u/${conversation.username}`, conversation.username);
this.pushLog('info', `Auto-reply rejected for u/${conversation.username}`);
res.json({ success: true });
} catch (error) {
this.logger.error('Failed to reject reply:', error);
res.status(500).json({ error: String(error) });
}
});
// API: Dismiss upsell detection flag
this.app.post('/api/conversations/:id/dismiss-upsell', async (req, res) => {
if (!this.chatService) {
res.status(500).json({ error: 'Chat service not initialized' });
return;
}
try {
const conversations = await this.chatService.getCachedConversations();
const id = decodeURIComponent(req.params.id);
const conversation = conversations.find(
(c: ActiveChat) => c.roomId === id || c.username === id || c.roomName === id
);
if (!conversation) {
res.status(404).json({ error: 'Conversation not found' });
return;
}
conversation.replyStatus = undefined;
conversation.pendingReply = undefined;
await this.chatService.saveCacheExternal();
res.json({ success: true });
} catch (error) {
this.logger.error('Failed to dismiss upsell flag:', error);
res.status(500).json({ error: String(error) });
}
});
// API: Get downloaded chat media
this.app.get('/api/media', async (req, res) => {
if (!this.mediaService) {
res.json([]);
return;
}
try {
const items = await this.mediaService.getMediaItems();
res.json(items.map((item) => ({
...item,
urlPath: `/media/${encodeURIComponent(item.relativePath)}`,
})));
} catch (error) {
this.logger.error('Failed to fetch media index:', error);
res.status(500).json({ error: String(error) });
}
});
// API: Get media sync status
this.app.get('/api/media/sync', (req, res) => {
if (!this.mediaSyncService) {
res.status(503).json({ error: 'Media sync service not initialized' });
return;
}
res.json(this.mediaSyncService.getStatus());
});
// API: Trigger media sync from dashboard
this.app.post('/api/media/sync', async (req, res) => {
if (!this.mediaSyncService) {
res.status(503).json({ error: 'Media sync service not initialized' });
return;
}
try {
const full = req.body?.full === true;
const started = await this.mediaSyncService.triggerSync(full);
if (!started) {
res.json({ success: false, reason: 'Sync already in progress' });
return;
}
this.pushLog('info', `Media sync triggered from dashboard (${full ? 'full' : 'incremental'})`);
res.json({ success: true });
} catch (error) {
this.logger.error('Media sync trigger failed:', error);
res.status(500).json({ error: String(error) });
}
});
// API: Full media sync - walks ALL conversations
this.app.post('/api/media/sync/full', async (req, res) => {
if (!this.mediaSyncService) {
res.status(503).json({ error: 'Media sync service not initialized' });
return;
}
try {
const started = await this.mediaSyncService.triggerSync(true);
if (!started) {
res.json({ success: false, reason: 'Sync already in progress' });
return;
}
this.pushLog('info', 'Full media sync triggered from dashboard (all conversations)');
res.json({ success: true });
} catch (error) {
this.logger.error('Full media sync trigger failed:', error);
res.status(500).json({ error: String(error) });
}
});
// API: Start bot
this.app.post('/api/start', (req, res) => {
this.botRunning = true;
@@ -107,6 +487,26 @@ export class WebServer {
res.json({ success: true, running: false });
});
// API: Debug inspect chat DOM
this.app.post('/api/debug/inspect-dom', async (req, res) => {
if (!this.chatService) {
res.status(500).json({ error: 'Chat service not initialized' });
return;
}
try {
const chatUrl = req.body?.url;
if (!chatUrl) {
res.status(400).json({ error: 'url is required' });
return;
}
const result = await this.chatService.debugInspectDom(chatUrl);
res.json(result);
} catch (error) {
this.logger.error('DOM inspection failed:', error);
res.status(500).json({ error: String(error) });
}
});
// API: Debug check responses
this.app.post('/api/debug/check-responses', async (req, res) => {
if (!this.responseChecker) {
@@ -123,6 +523,121 @@ export class WebServer {
}
});
// API: SSE endpoint for realtime logs
this.app.get('/api/logs/stream', (req, res) => {
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
res.setHeader('X-Accel-Buffering', 'no'); // Disable nginx buffering
// Send initial connection message
res.write(`data: ${JSON.stringify({ type: 'connected', timestamp: Date.now() })}\n\n`);
// Add client to SSE set
this.sseClients.add(res);
// Send recent logs as initial data
this.realtimeLog.slice(-50).forEach(log => {
res.write(`data: ${JSON.stringify({ type: 'log', ...log })}\n\n`);
});
// Remove client on disconnect
req.on('close', () => {
this.sseClients.delete(res);
});
});
// API: Get recent logs (for initial load)
this.app.get('/api/logs', (req, res) => {
const limit = parseInt(req.query.limit as string) || 100;
res.json(this.realtimeLog.slice(-limit));
});
// API: Get active timers
this.app.get('/api/timers', (req, res) => {
if (!this.rateLimiter) {
res.status(503).json({ error: 'Rate limiter not initialized' });
return;
}
res.json(this.rateLimiter.getTimers());
});
// API: Analytics - response rate by subreddit
this.app.get('/api/analytics/by-subreddit', (req, res) => {
res.json(this.messagesRepo.getSubredditStats());
});
// API: Analytics - response rate by strategy
this.app.get('/api/analytics/by-strategy', (req, res) => {
if (this.strategySelector) {
res.json(this.strategySelector.getStats());
} else {
res.json(this.messagesRepo.getStrategyStats());
}
});
// API: Analytics - daily response rate trend
this.app.get('/api/analytics/trend', (req, res) => {
const days = parseInt(req.query.days as string) || 30;
res.json(this.messagesRepo.getResponseRateTrend(days));
});
// API: Get runtime config
this.app.get('/api/config', (req, res) => {
res.json({
rateLimit: this.runtimeConfig.rateLimit,
browser: {
dryRun: this.runtimeConfig.browser.dryRun,
},
});
});
// API: Update runtime config
this.app.put('/api/config', (req, res) => {
try {
const updates = req.body;
// Validate and apply rate limit updates
if (updates.rateLimit) {
if (typeof updates.rateLimit.minDelayBetweenMessagesMs === 'number') {
this.runtimeConfig.rateLimit.minDelayBetweenMessagesMs = Math.max(0, updates.rateLimit.minDelayBetweenMessagesMs);
}
if (typeof updates.rateLimit.maxMessagesPerHour === 'number') {
this.runtimeConfig.rateLimit.maxMessagesPerHour = Math.max(1, updates.rateLimit.maxMessagesPerHour);
}
if (typeof updates.rateLimit.jitterMs === 'number') {
this.runtimeConfig.rateLimit.jitterMs = Math.max(0, updates.rateLimit.jitterMs);
}
}
// Validate and apply browser updates
if (updates.browser) {
if (typeof updates.browser.dryRun === 'boolean') {
this.runtimeConfig.browser.dryRun = updates.browser.dryRun;
}
}
// Also update the original config object so getStats reflects new values
this.config.rateLimit.minDelayBetweenMessagesMs = this.runtimeConfig.rateLimit.minDelayBetweenMessagesMs;
this.config.rateLimit.maxMessagesPerHour = this.runtimeConfig.rateLimit.maxMessagesPerHour;
this.config.rateLimit.jitterMs = this.runtimeConfig.rateLimit.jitterMs;
this.config.browser.dryRun = this.runtimeConfig.browser.dryRun;
// Notify listeners of config change
if (this.onConfigChange) {
this.onConfigChange(this.runtimeConfig);
}
this.logActivity('info', `Config updated via dashboard`);
this.pushLog('info', 'Configuration updated via dashboard', updates);
res.json({ success: true, config: this.runtimeConfig });
} catch (error) {
this.logger.error('Failed to update config:', error);
res.status(400).json({ error: String(error) });
}
});
// Serve dashboard for all other routes
this.app.get(/.*/, (req, res) => {
res.sendFile(path.join(process.cwd(), 'public', 'index.html'));
@@ -136,10 +651,34 @@ export class WebServer {
this.chatService = chatService;
}
setMediaService(mediaService: MediaService): void {
this.mediaService = mediaService;
}
setMediaSyncService(mediaSyncService: MediaSyncService): void {
this.mediaSyncService = mediaSyncService;
}
setResponseChecker(responseChecker: ResponseCheckerService): void {
this.responseChecker = responseChecker;
}
setRateLimiter(rateLimiter: RateLimiter): void {
this.rateLimiter = rateLimiter;
}
setAiClient(aiClient: OpenRouterClient): void {
this.aiClient = aiClient;
}
setStrategySelector(strategySelector: StrategySelector): void {
this.strategySelector = strategySelector;
}
setAutoReplyService(autoReplyService: AutoReplyService): void {
this.autoReplyService = autoReplyService;
}
private getStats(): BotStats {
const messages = this.messagesRepo.getRecent(1000);
const totalSent = messages.length;
@@ -165,12 +704,13 @@ export class WebServer {
return this.activityLog.filter((a) => a.type === 'sent' || a.type === 'skipped').length;
}
logActivity(type: ActivityItem['type'], message: string, recipient?: string): void {
logActivity(type: ActivityItem['type'], message: string, recipient?: string, imageUrl?: string): void {
const item: ActivityItem = {
timestamp: Date.now(),
type,
message,
recipient,
imageUrl,
};
this.activityLog.push(item);
@@ -180,6 +720,50 @@ export class WebServer {
}
}
/**
* Push a log entry to all connected SSE clients
*/
pushLog(level: LogEntry['level'], message: string, data?: unknown): void {
const entry: LogEntry = {
timestamp: Date.now(),
level,
message,
data,
};
this.realtimeLog.push(entry);
// Keep only last 500 log entries
if (this.realtimeLog.length > 500) {
this.realtimeLog = this.realtimeLog.slice(-500);
}
// Broadcast to all SSE clients
const eventData = JSON.stringify({ type: 'log', ...entry });
this.sseClients.forEach(client => {
try {
client.write(`data: ${eventData}\n\n`);
} catch {
// Client disconnected, will be cleaned up on close event
this.sseClients.delete(client);
}
});
}
/**
* Get the current runtime configuration
*/
getRuntimeConfig(): RuntimeConfig {
return this.runtimeConfig;
}
/**
* Set a callback to be notified when config changes
*/
setOnConfigChange(callback: (config: RuntimeConfig) => void): void {
this.onConfigChange = callback;
}
updateStats(messagesThisHour: number): void {
this.messagesThisHour = messagesThisHour;
}
@@ -187,10 +771,21 @@ export class WebServer {
start(): void {
this.server = this.app.listen(this.port, () => {
this.logger.info(`Web dashboard running at http://localhost:${this.port}`);
this.pushLog('info', `Web dashboard started on port ${this.port}`);
});
}
stop(): void {
// Close all SSE connections
this.sseClients.forEach(client => {
try {
client.end();
} catch {
// Ignore errors when closing
}
});
this.sseClients.clear();
if (this.server) {
this.server.close();
this.server = null;