Update source code and remove runtime data from tracking
Add gitignore rules for browser session data, SQLite databases, media downloads, and build artifacts. Remove previously tracked runtime files from the repository. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jjbruntonandClaude Opus 4.6 committed
1 parent
e90213d1ff
commit
dfa0580859
23 files changed
+5324
-347
No files matched your search
+714
-140
@@ -1,12 +1,22 @@
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import type { Page } from 'playwright';
|
||||
import type { Logger } from '../utils/logger.js';
|
||||
import type { SuccessfulMessagesRepository } from '../database/repositories.js';
|
||||
import type { MediaService } from './media.js';
|
||||
|
||||
export interface ChatMessage {
|
||||
sender: string;
|
||||
text: string;
|
||||
timestamp: number;
|
||||
isSelf: boolean;
|
||||
media?: ChatMedia[];
|
||||
}
|
||||
|
||||
export interface ChatMedia {
|
||||
originalUrl: string;
|
||||
localPath: string;
|
||||
fileName: string;
|
||||
}
|
||||
|
||||
export interface ActiveChat {
|
||||
@@ -20,64 +30,318 @@ export interface ActiveChat {
|
||||
roomId?: string;
|
||||
messages?: ChatMessage[];
|
||||
originMessage?: string; // The first message we sent that started the conversation
|
||||
pendingReply?: string; // AI-generated reply awaiting approval
|
||||
replyStatus?: 'pending' | 'upsell_detected' | 'sent'; // Status of auto-reply
|
||||
}
|
||||
|
||||
interface ChatCacheFile {
|
||||
version: number;
|
||||
savedAt: number;
|
||||
chats: ActiveChat[];
|
||||
}
|
||||
|
||||
type ScrapedChatMessage = Omit<ChatMessage, 'media'> & { mediaUrls: string[] };
|
||||
|
||||
export class ChatService {
|
||||
private cachedChats: ActiveChat[] = [];
|
||||
private lastFetchTime: number = 0;
|
||||
private readonly CACHE_DURATION_MS = 60 * 1000; // 1 minute cache
|
||||
private readonly cacheFilePath: string;
|
||||
|
||||
private queue: Promise<any> = Promise.resolve();
|
||||
|
||||
private enqueue<T>(task: () => Promise<T>): Promise<T> {
|
||||
const result = this.queue.then(() => task());
|
||||
this.queue = result.catch(() => { }); // Ensure queue never breaks on error
|
||||
return result;
|
||||
}
|
||||
|
||||
constructor(
|
||||
private page: Page,
|
||||
private logger: Logger,
|
||||
private successfulMessagesRepo?: SuccessfulMessagesRepository
|
||||
) { }
|
||||
private successfulMessagesRepo?: SuccessfulMessagesRepository,
|
||||
private mediaService?: MediaService
|
||||
) {
|
||||
this.cacheFilePath = path.join(process.cwd(), 'data', 'chat-cache.json');
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns cached conversations from memory or disk cache for dashboard browsing.
|
||||
* Does NOT fetch from Reddit - only returns locally cached data.
|
||||
*/
|
||||
async getCachedConversations(): Promise<ActiveChat[]> {
|
||||
// If we have cached chats in memory, return them
|
||||
if (this.cachedChats.length > 0) {
|
||||
return this.cachedChats;
|
||||
}
|
||||
// Otherwise, try to load from disk
|
||||
await this.loadCache();
|
||||
return this.cachedChats;
|
||||
}
|
||||
|
||||
async getActiveChats(): Promise<ActiveChat[]> {
|
||||
// Return cached data if fresh enough
|
||||
const now = Date.now();
|
||||
if (now - this.lastFetchTime < this.CACHE_DURATION_MS && this.cachedChats.length > 0) {
|
||||
return this.cachedChats;
|
||||
}
|
||||
|
||||
try {
|
||||
this.logger.debug('Fetching active chats from chat.reddit.com...');
|
||||
|
||||
// Navigate to chat.reddit.com
|
||||
await this.page.goto('https://chat.reddit.com/', {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 30000,
|
||||
});
|
||||
|
||||
// Wait for the chat app to initialize
|
||||
await this.page.waitForTimeout(5000);
|
||||
|
||||
const chats = await this.scrapeChats();
|
||||
|
||||
// Fetch full history for each chat
|
||||
this.logger.info(`Fetching full history for ${chats.length} chats...`);
|
||||
|
||||
for (const chat of chats) {
|
||||
try {
|
||||
await this.loadConversationDetails(chat);
|
||||
} catch (err) {
|
||||
this.logger.error(`Failed to load conversation for ${chat.roomName}`, err);
|
||||
}
|
||||
return this.enqueue(async () => {
|
||||
// Return cached data if fresh enough
|
||||
const now = Date.now();
|
||||
if (now - this.lastFetchTime < this.CACHE_DURATION_MS && this.cachedChats.length > 0) {
|
||||
return this.cachedChats;
|
||||
}
|
||||
|
||||
// Filter to last 96 hours for the "active" view
|
||||
const cutoffTime = now - (96 * 60 * 60 * 1000);
|
||||
this.cachedChats = chats.filter(chat => chat.lastMessageTime >= cutoffTime);
|
||||
this.lastFetchTime = now;
|
||||
try {
|
||||
this.logger.debug('Fetching active chats from chat.reddit.com...');
|
||||
|
||||
this.logger.info(`Found ${this.cachedChats.length} active chats`);
|
||||
return this.cachedChats;
|
||||
// Navigate to chat.reddit.com
|
||||
await this.page.goto('https://chat.reddit.com/', {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 30000,
|
||||
});
|
||||
|
||||
// Wait for the chat app to initialize
|
||||
await this.page.waitForTimeout(5000);
|
||||
|
||||
const chats = await this.scrapeChats();
|
||||
|
||||
// Fetch full history for each chat
|
||||
this.logger.info(`Fetching full history for ${chats.length} chats...`);
|
||||
|
||||
for (const chat of chats) {
|
||||
try {
|
||||
await this.loadConversationDetails(chat);
|
||||
} catch (err) {
|
||||
this.logger.error(`Failed to load conversation for ${chat.roomName}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
// Filter to last 96 hours for the "active" view
|
||||
const cutoffTime = now - (96 * 60 * 60 * 1000);
|
||||
this.cachedChats = chats.filter(chat => chat.lastMessageTime >= cutoffTime);
|
||||
this.lastFetchTime = now;
|
||||
|
||||
// Save to disk cache
|
||||
await this.saveCache(chats);
|
||||
|
||||
this.logger.info(`Found ${this.cachedChats.length} active chats`);
|
||||
return this.cachedChats;
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to fetch active chats:', error);
|
||||
return this.cachedChats;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async refreshTopChats(limit: number = 5): Promise<void> {
|
||||
await this.enqueue(async () => {
|
||||
try {
|
||||
this.logger.info(`Refreshing top ${limit} chats...`);
|
||||
|
||||
await this.page.goto('https://chat.reddit.com/', {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 30000,
|
||||
});
|
||||
await this.page.waitForTimeout(5000);
|
||||
|
||||
const chats = await this.scrapeChats();
|
||||
|
||||
// Load details for the top N chats
|
||||
const topChats = chats.slice(0, limit);
|
||||
this.logger.info(`Loading details for top ${topChats.length} chats...`);
|
||||
|
||||
for (const chat of topChats) {
|
||||
try {
|
||||
await this.loadConversationDetails(chat);
|
||||
} catch (err) {
|
||||
this.logger.error(`Failed to load conversation for ${chat.roomName}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
// Merge with existing cache to preserve history for chats we didn't fully load this time
|
||||
const mergedChats = chats.map(newChat => {
|
||||
const existing = this.cachedChats.find(c => c.url === newChat.url);
|
||||
// If we didn't load messages this time (check if messages array is missing or empty on newChat)
|
||||
// But wait, loadConversationDetails adds 'messages' to newChat.
|
||||
// If we SKIPPED loadConversationDetails, newChat.messages will be undefined.
|
||||
if (existing && existing.messages && !newChat.messages) {
|
||||
newChat.messages = existing.messages;
|
||||
newChat.originMessage = existing.originMessage;
|
||||
}
|
||||
return newChat;
|
||||
});
|
||||
|
||||
this.cachedChats = mergedChats;
|
||||
this.lastFetchTime = Date.now();
|
||||
await this.saveCache(this.cachedChats);
|
||||
this.logger.info(`Refreshed top ${limit} chats and updated cache.`);
|
||||
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to refresh top chats:', error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fetchAllMessages(): Promise<ActiveChat[]> {
|
||||
return this.enqueue(async () => {
|
||||
try {
|
||||
this.logger.info('Fetching all chats and messages from chat.reddit.com...');
|
||||
|
||||
await this.page.goto('https://chat.reddit.com/', {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 30000,
|
||||
});
|
||||
await this.page.waitForTimeout(5000);
|
||||
|
||||
const chats = await this.scrapeChats();
|
||||
this.logger.info(`Fetching full history for ${chats.length} chats (all)...`);
|
||||
|
||||
for (const chat of chats) {
|
||||
try {
|
||||
await this.loadConversationDetails(chat);
|
||||
} catch (err) {
|
||||
this.logger.error(`Failed to load conversation for ${chat.roomName}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
this.cachedChats = chats;
|
||||
this.lastFetchTime = Date.now();
|
||||
await this.saveCache(chats);
|
||||
|
||||
return chats;
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to fetch all messages:', error);
|
||||
return this.cachedChats;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async loadCache(): Promise<void> {
|
||||
try {
|
||||
const raw = await fs.readFile(this.cacheFilePath, 'utf-8');
|
||||
const parsed = JSON.parse(raw) as ChatCacheFile;
|
||||
if (!parsed || !Array.isArray(parsed.chats)) {
|
||||
this.logger.warn('Chat cache file is invalid, ignoring.');
|
||||
return;
|
||||
}
|
||||
this.cachedChats = parsed.chats;
|
||||
this.lastFetchTime = Date.now();
|
||||
this.logger.info(`Loaded ${parsed.chats.length} chats from cache`);
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to fetch active chats:', error);
|
||||
return this.cachedChats;
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
this.logger.info('No chat cache found to load');
|
||||
return;
|
||||
}
|
||||
this.logger.warn('Failed to load chat cache:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async debugInspectDom(chatUrl: string): Promise<unknown> {
|
||||
return this.enqueue(async () => {
|
||||
await this.page.goto(chatUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
|
||||
await this.page.waitForTimeout(3000);
|
||||
|
||||
return this.page.evaluate(`
|
||||
(function() {
|
||||
var result = { shadowChain: [], allImages: [], allMedia: [], messageElements: [], customElements: [] };
|
||||
|
||||
var rsApp = document.querySelector('rs-app');
|
||||
if (!rsApp || !rsApp.shadowRoot) { result.error = 'No rs-app shadow root'; return result; }
|
||||
|
||||
var rsRoom = rsApp.shadowRoot.querySelector('rs-room');
|
||||
if (!rsRoom) { result.error = 'No rs-room'; return result; }
|
||||
|
||||
var searchRoot = rsRoom.shadowRoot || rsRoom;
|
||||
|
||||
// Recursively map ALL elements through shadow DOMs
|
||||
function walk(root, depth, path) {
|
||||
if (!root || depth > 10) return;
|
||||
var els = root.querySelectorAll('*');
|
||||
for (var i = 0; i < els.length; i++) {
|
||||
var el = els[i];
|
||||
var tag = el.tagName.toLowerCase();
|
||||
|
||||
// Track images
|
||||
if (tag === 'img') {
|
||||
result.allImages.push({
|
||||
src: (el.getAttribute('src') || '').slice(0, 300),
|
||||
dataSrc: (el.getAttribute('data-src') || '').slice(0, 300),
|
||||
alt: (el.getAttribute('alt') || '').slice(0, 100),
|
||||
className: (el.className || '').toString().slice(0, 150),
|
||||
width: el.width, height: el.height,
|
||||
naturalWidth: el.naturalWidth, naturalHeight: el.naturalHeight,
|
||||
parentTag: el.parentElement ? el.parentElement.tagName.toLowerCase() : 'none',
|
||||
parentClass: el.parentElement ? (el.parentElement.className || '').toString().slice(0, 150) : '',
|
||||
path: path,
|
||||
depth: depth
|
||||
});
|
||||
}
|
||||
|
||||
// Track video, picture, source, canvas, iframe
|
||||
if (['video', 'picture', 'source', 'canvas', 'iframe', 'object', 'embed'].indexOf(tag) !== -1) {
|
||||
result.allMedia.push({
|
||||
tag: tag,
|
||||
src: (el.getAttribute('src') || el.getAttribute('data') || '').slice(0, 300),
|
||||
type: el.getAttribute('type') || '',
|
||||
className: (el.className || '').toString().slice(0, 150),
|
||||
path: path,
|
||||
depth: depth,
|
||||
outerHTML: el.outerHTML.slice(0, 500)
|
||||
});
|
||||
}
|
||||
|
||||
// Track custom elements (contain hyphens) or elements with shadow roots
|
||||
if (tag.indexOf('-') !== -1 || el.shadowRoot) {
|
||||
var text = (el.textContent || '').trim();
|
||||
result.customElements.push({
|
||||
tag: tag,
|
||||
hasShadow: !!el.shadowRoot,
|
||||
className: (el.className || '').toString().slice(0, 150),
|
||||
childCount: el.childElementCount,
|
||||
path: path,
|
||||
depth: depth,
|
||||
textSnippet: text.slice(0, 80),
|
||||
hasImgChild: el.querySelectorAll('img').length > 0,
|
||||
innerHTML: el.innerHTML ? el.innerHTML.slice(0, 200) : ''
|
||||
});
|
||||
}
|
||||
|
||||
// Look for elements with background-image CSS
|
||||
if (el.style && el.style.backgroundImage && el.style.backgroundImage.indexOf('url') !== -1) {
|
||||
result.allMedia.push({
|
||||
tag: tag + '[bg-image]',
|
||||
src: el.style.backgroundImage.slice(0, 300),
|
||||
type: 'css-background',
|
||||
className: (el.className || '').toString().slice(0, 150),
|
||||
path: path,
|
||||
depth: depth,
|
||||
outerHTML: ''
|
||||
});
|
||||
}
|
||||
|
||||
// Recurse into shadow roots
|
||||
if (el.shadowRoot) {
|
||||
walk(el.shadowRoot, depth + 1, path + ' > ' + tag + '[shadow]');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
walk(searchRoot, 0, 'rs-room');
|
||||
|
||||
// Also get direct children structure for context
|
||||
result.rootChildren = [];
|
||||
for (var c = 0; c < searchRoot.children.length; c++) {
|
||||
var child = searchRoot.children[c];
|
||||
result.rootChildren.push({
|
||||
tag: child.tagName.toLowerCase(),
|
||||
className: (child.className || '').toString().slice(0, 200),
|
||||
hasShadow: !!child.shadowRoot,
|
||||
childCount: child.childElementCount
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
})()
|
||||
`);
|
||||
});
|
||||
}
|
||||
|
||||
async loadConversationDetails(chat: ActiveChat): Promise<void> {
|
||||
if (!chat.url) {
|
||||
this.logger.warn(`No URL for chat ${chat.roomName}, skipping details.`);
|
||||
@@ -104,7 +368,10 @@ export class ChatService {
|
||||
}
|
||||
|
||||
// Now scrape all messages in the conversation
|
||||
const messages = await this.scrapeConversationMessages(chat.username);
|
||||
const scrapedMessages = await this.scrapeConversationMessages(chat.username);
|
||||
const messages = this.mediaService
|
||||
? await this.mediaService.attachMedia(chat, scrapedMessages)
|
||||
: this.stripMediaUrls(scrapedMessages);
|
||||
|
||||
if (messages.length > 0) {
|
||||
chat.messages = messages;
|
||||
@@ -136,6 +403,7 @@ export class ChatService {
|
||||
imageAnalysis: null, // We don't have image analysis for chat messages
|
||||
postTitle: `Chat with ${chat.username}`,
|
||||
responseSnippet: replySnippet,
|
||||
subreddit: null,
|
||||
});
|
||||
this.logger.info(`[SUCCESS] Stored successful chat example from ${chat.username}!`);
|
||||
} catch (err) {
|
||||
@@ -155,6 +423,116 @@ export class ChatService {
|
||||
}
|
||||
}
|
||||
|
||||
async sendChatMessage(chatUrl: string, message: string): Promise<boolean> {
|
||||
return this.enqueue(async () => {
|
||||
try {
|
||||
this.logger.info(`Sending chat message to ${chatUrl}...`);
|
||||
|
||||
await this.page.goto(chatUrl, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 30000,
|
||||
});
|
||||
await this.page.waitForTimeout(3000);
|
||||
|
||||
// Find and focus the chat input inside shadow DOM
|
||||
const inputFound = await this.page.evaluate(`
|
||||
(function() {
|
||||
function collectAllElements(root) {
|
||||
var elements = [];
|
||||
if (!root) return elements;
|
||||
var treeWalker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
|
||||
var current = treeWalker.currentNode;
|
||||
while (current) {
|
||||
elements.push(current);
|
||||
if (current.shadowRoot) {
|
||||
elements = elements.concat(collectAllElements(current.shadowRoot));
|
||||
}
|
||||
current = treeWalker.nextNode();
|
||||
}
|
||||
return elements;
|
||||
}
|
||||
|
||||
var elements = collectAllElements(document);
|
||||
|
||||
// Look for textarea or contenteditable input in the chat
|
||||
for (var i = 0; i < elements.length; i++) {
|
||||
var el = elements[i];
|
||||
if (!el.tagName) continue;
|
||||
var tag = el.tagName.toLowerCase();
|
||||
|
||||
// Match textarea or input with chat-related attributes
|
||||
if (tag === 'textarea' || (tag === 'input' && el.type === 'text')) {
|
||||
var placeholder = (el.getAttribute('placeholder') || '').toLowerCase();
|
||||
var ariaLabel = (el.getAttribute('aria-label') || '').toLowerCase();
|
||||
if (placeholder.includes('message') || ariaLabel.includes('message') ||
|
||||
placeholder.includes('chat') || ariaLabel.includes('chat') ||
|
||||
placeholder.includes('type') || ariaLabel.includes('type')) {
|
||||
el.focus();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Match contenteditable divs used as chat inputs
|
||||
if (el.getAttribute('contenteditable') === 'true') {
|
||||
var role = (el.getAttribute('role') || '').toLowerCase();
|
||||
var ariaLabel2 = (el.getAttribute('aria-label') || '').toLowerCase();
|
||||
if (role === 'textbox' || ariaLabel2.includes('message') || ariaLabel2.includes('chat')) {
|
||||
el.focus();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Broader fallback: any textarea or contenteditable inside rs-room
|
||||
var rsApp = document.querySelector('rs-app');
|
||||
if (rsApp && rsApp.shadowRoot) {
|
||||
var rsRoom = rsApp.shadowRoot.querySelector('rs-room');
|
||||
if (rsRoom) {
|
||||
var searchRoot = rsRoom.shadowRoot || rsRoom;
|
||||
var inputs = collectAllElements(searchRoot);
|
||||
for (var j = 0; j < inputs.length; j++) {
|
||||
var inp = inputs[j];
|
||||
if (!inp.tagName) continue;
|
||||
if (inp.tagName.toLowerCase() === 'textarea' ||
|
||||
inp.getAttribute('contenteditable') === 'true' ||
|
||||
(inp.tagName.toLowerCase() === 'input' && inp.type === 'text')) {
|
||||
inp.focus();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
})()
|
||||
`);
|
||||
|
||||
if (!inputFound) {
|
||||
this.logger.error('Could not find chat input field');
|
||||
return false;
|
||||
}
|
||||
|
||||
// Type the message with human-like delay
|
||||
for (const char of message) {
|
||||
await this.page.keyboard.type(char, { delay: 15 + Math.random() * 25 });
|
||||
}
|
||||
|
||||
await this.page.waitForTimeout(500);
|
||||
|
||||
// Press Enter to send
|
||||
await this.page.keyboard.press('Enter');
|
||||
|
||||
await this.page.waitForTimeout(2000);
|
||||
|
||||
this.logger.info('Chat message sent successfully');
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to send chat message:', error);
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async verifyMessageSent(recipient: string, messageText: string): Promise<boolean> {
|
||||
this.logger.info(`Verifying message to ${recipient} in chat...`);
|
||||
|
||||
@@ -283,8 +661,8 @@ export class ChatService {
|
||||
}
|
||||
}
|
||||
|
||||
private async scrapeConversationMessages(recipientUsername: string): Promise<ChatMessage[]> {
|
||||
const messages: ChatMessage[] = [];
|
||||
private async scrapeConversationMessages(recipientUsername: string): Promise<ScrapedChatMessage[]> {
|
||||
const messages: ScrapedChatMessage[] = [];
|
||||
|
||||
try {
|
||||
// Use raw JavaScript string to avoid TypeScript transformation issues
|
||||
@@ -292,127 +670,154 @@ export class ChatService {
|
||||
(function() {
|
||||
var myUsername = 'Upset_Marsupial1006';
|
||||
var results = [];
|
||||
|
||||
// Find the rs-room element which contains only the message area (not sidebar)
|
||||
function findRsRoom() {
|
||||
var rsApp = document.querySelector('rs-app');
|
||||
if (!rsApp || !rsApp.shadowRoot) return null;
|
||||
return rsApp.shadowRoot.querySelector('rs-room');
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
function getImageSrc(img) {
|
||||
return img.getAttribute('src') || img.getAttribute('data-src') || img.getAttribute('data-lazy-src') || '';
|
||||
}
|
||||
|
||||
function findAllSpans(root) {
|
||||
|
||||
function isAvatarImage(img) {
|
||||
var src = getImageSrc(img);
|
||||
if (!src || src.indexOf('http') !== 0) return true;
|
||||
|
||||
var srcLower = src.toLowerCase();
|
||||
if (srcLower.indexOf('styles.redditmedia.com') !== -1) return true;
|
||||
if (srcLower.indexOf('redditstatic.com/avatars') !== -1) return true;
|
||||
if (srcLower.indexOf('/avatar/') !== -1) return true;
|
||||
if (srcLower.indexOf('/avatars/') !== -1) return true;
|
||||
if (srcLower.indexOf('avatar_default') !== -1) return true;
|
||||
if (srcLower.indexOf('snoovatar') !== -1) return true;
|
||||
|
||||
var imgAlt = (img.getAttribute('alt') || '').toLowerCase();
|
||||
if (imgAlt.indexOf('avatar') !== -1) return true;
|
||||
|
||||
if (img.width > 0 && img.height > 0 && img.width <= 64 && img.height <= 64) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// Recursively find all images through shadow DOM boundaries
|
||||
function findAllImages(root) {
|
||||
if (!root) return [];
|
||||
var spans = Array.from(root.querySelectorAll('span'));
|
||||
var imgs = Array.from(root.querySelectorAll('img'));
|
||||
var allElements = root.querySelectorAll('*');
|
||||
for (var i = 0; i < allElements.length; i++) {
|
||||
var el = allElements[i];
|
||||
if (el.shadowRoot) {
|
||||
spans = spans.concat(findAllSpans(el.shadowRoot));
|
||||
imgs = imgs.concat(findAllImages(el.shadowRoot));
|
||||
}
|
||||
}
|
||||
return spans;
|
||||
return imgs;
|
||||
}
|
||||
|
||||
var rsRoom = findRsRoom();
|
||||
|
||||
// Recursively collect all text from a root, piercing shadow DOMs
|
||||
function collectText(root) {
|
||||
if (!root) return '';
|
||||
var text = '';
|
||||
var walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT, null);
|
||||
var node;
|
||||
while (node = walker.nextNode()) {
|
||||
text += node.textContent;
|
||||
}
|
||||
// Also collect from shadow roots of child elements
|
||||
var allElements = root.querySelectorAll('*');
|
||||
for (var i = 0; i < allElements.length; i++) {
|
||||
if (allElements[i].shadowRoot) {
|
||||
text += collectText(allElements[i].shadowRoot);
|
||||
}
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
// --- Navigate shadow DOM to find timeline events ---
|
||||
|
||||
var rsApp = document.querySelector('rs-app');
|
||||
if (!rsApp || !rsApp.shadowRoot) return [];
|
||||
|
||||
var rsRoom = rsApp.shadowRoot.querySelector('rs-room');
|
||||
if (!rsRoom) return [];
|
||||
|
||||
// Start from rs-room's shadow root if it has one
|
||||
var searchRoot = rsRoom.shadowRoot || rsRoom;
|
||||
var allSpans = findAllSpans(searchRoot);
|
||||
|
||||
// Find username spans
|
||||
var usernameSpans = allSpans.filter(function(span) {
|
||||
var text = (span.textContent || '').trim();
|
||||
if (text.length === 0 || text.length > 40) return false;
|
||||
if (!/^[A-Za-z0-9_-]+$/.test(text)) return false;
|
||||
if (span.className && (span.className.includes('user-name') || span.className.includes('username'))) {
|
||||
return true;
|
||||
var roomRoot = rsRoom.shadowRoot || rsRoom;
|
||||
|
||||
var rsTimeline = roomRoot.querySelector('rs-timeline');
|
||||
if (!rsTimeline || !rsTimeline.shadowRoot) return [];
|
||||
|
||||
var rsVirtualScroll = rsTimeline.shadowRoot.querySelector('rs-virtual-scroll-dynamic');
|
||||
if (!rsVirtualScroll || !rsVirtualScroll.shadowRoot) return [];
|
||||
|
||||
var scrollRoot = rsVirtualScroll.shadowRoot;
|
||||
|
||||
// Find all rs-timeline-event elements (each one = a message)
|
||||
var events = scrollRoot.querySelectorAll('rs-timeline-event');
|
||||
|
||||
var lastSender = null;
|
||||
|
||||
for (var i = 0; i < events.length; i++) {
|
||||
var event = events[i];
|
||||
var eventRoot = event.shadowRoot;
|
||||
if (!eventRoot) continue;
|
||||
|
||||
// 1. Extract sender from rs-timeline-event-hovercard elements
|
||||
var sender = null;
|
||||
var hovercards = eventRoot.querySelectorAll('rs-timeline-event-hovercard');
|
||||
for (var h = 0; h < hovercards.length; h++) {
|
||||
var hcText = collectText(hovercards[h]).trim();
|
||||
// The hovercard with the username has just alphanumeric text
|
||||
if (hcText.length > 0 && hcText.length < 40 && /^[A-Za-z0-9_-]+$/.test(hcText)) {
|
||||
sender = hcText;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (text.includes('_') || text.includes('-') || (text.length > 3 && text.length < 25)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
// For each username span, find associated message text
|
||||
for (var j = 0; j < usernameSpans.length; j++) {
|
||||
var userSpan = usernameSpans[j];
|
||||
var sender = (userSpan.textContent || '').trim();
|
||||
// If no sender found in this event, it's a continuation from the same sender
|
||||
if (!sender) sender = lastSender;
|
||||
if (!sender) continue;
|
||||
|
||||
// Walk up to find container with content
|
||||
var container = userSpan.parentElement;
|
||||
var attempts = 0;
|
||||
while (container && attempts < 10) {
|
||||
var containerText = container.innerText || container.textContent || '';
|
||||
var senderEscaped = sender.replace(/[-_]/g, '\\\\$&');
|
||||
var cleaned = containerText
|
||||
.replace(new RegExp(senderEscaped, 'g'), '')
|
||||
.replace(/\\d{1,2}:\\d{2}\\s*(AM|PM)?/gi, '')
|
||||
.replace(/\\d{1,2}\\/\\d{1,2}\\/\\d{2,4}/gi, '')
|
||||
.replace(/yesterday|today|just now/gi, '')
|
||||
.trim();
|
||||
|
||||
if (cleaned.length > 10 && cleaned.length < 3000) {
|
||||
break;
|
||||
}
|
||||
container = container.parentElement;
|
||||
attempts++;
|
||||
lastSender = sender;
|
||||
|
||||
// 2. Extract message text - get all text, then strip sender/timestamps
|
||||
var fullText = collectText(eventRoot);
|
||||
var senderEscaped = sender.replace(/[-_]/g, '\\\\$&');
|
||||
var messageText = fullText
|
||||
.replace(new RegExp(senderEscaped, 'g'), '')
|
||||
.replace(/\\d{1,2}:\\d{2}\\s*(AM|PM)?/gi, '')
|
||||
.replace(/\\d{1,2}\\/\\d{1,2}\\/\\d{2,4}/gi, '')
|
||||
.replace(/yesterday|today|just now/gi, '')
|
||||
.replace(/send\\s*message/gi, '')
|
||||
.replace(/\\s+/g, ' ')
|
||||
.replace(/^\\s*[:|-]\\s*/, '')
|
||||
.trim();
|
||||
|
||||
// 3. Extract media images from rs-image elements and any other non-avatar images
|
||||
var mediaUrls = [];
|
||||
var allImgs = findAllImages(eventRoot);
|
||||
for (var mi = 0; mi < allImgs.length; mi++) {
|
||||
if (isAvatarImage(allImgs[mi])) continue;
|
||||
var src = getImageSrc(allImgs[mi]);
|
||||
if (src) mediaUrls.push(src);
|
||||
}
|
||||
|
||||
if (container) {
|
||||
var fullText = container.innerText || container.textContent || '';
|
||||
var senderEscaped = sender.replace(/[-_]/g, '\\\\$&');
|
||||
var messageText = fullText
|
||||
.replace(new RegExp(senderEscaped, 'g'), '')
|
||||
.replace(/\\d{1,2}:\\d{2}\\s*(AM|PM)?/gi, '')
|
||||
.replace(/\\d{1,2}\\/\\d{1,2}\\/\\d{2,4}/gi, '')
|
||||
.replace(/yesterday|today|just now/gi, '')
|
||||
.replace(/\\s+/g, ' ')
|
||||
.replace(/^\\s*[:|-]\\s*/, '')
|
||||
.trim();
|
||||
|
||||
if (messageText.length > 5) {
|
||||
results.push({
|
||||
sender: sender,
|
||||
text: messageText,
|
||||
isSelf: sender === myUsername
|
||||
});
|
||||
}
|
||||
|
||||
if (messageText.length > 0 || mediaUrls.length > 0) {
|
||||
results.push({
|
||||
sender: sender,
|
||||
text: messageText,
|
||||
isSelf: sender === myUsername,
|
||||
mediaUrls: mediaUrls
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate
|
||||
var unique = [];
|
||||
for (var k = 0; k < results.length; k++) {
|
||||
var msg = results[k];
|
||||
var existingIdx = -1;
|
||||
for (var m = 0; m < unique.length; m++) {
|
||||
var u = unique[m];
|
||||
if (u.sender === msg.sender &&
|
||||
(u.text === msg.text || u.text.includes(msg.text) || msg.text.includes(u.text))) {
|
||||
existingIdx = m;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (existingIdx === -1) {
|
||||
unique.push(msg);
|
||||
} else if (msg.text.length > unique[existingIdx].text.length) {
|
||||
unique[existingIdx] = msg;
|
||||
}
|
||||
}
|
||||
|
||||
return unique;
|
||||
|
||||
return results;
|
||||
})()
|
||||
`) as { sender: string; text: string; isSelf: boolean }[];
|
||||
`) as { sender: string; text: string; isSelf: boolean; mediaUrls: string[] }[];
|
||||
|
||||
// Convert to ChatMessage format
|
||||
for (const msg of extractedMessages) {
|
||||
const normalizedText = msg.text || (msg.mediaUrls && msg.mediaUrls.length > 0 ? '[Image]' : '');
|
||||
messages.push({
|
||||
sender: msg.isSelf ? 'Me' : msg.sender,
|
||||
text: msg.text,
|
||||
text: normalizedText,
|
||||
timestamp: Date.now(),
|
||||
isSelf: msg.isSelf,
|
||||
mediaUrls: msg.mediaUrls || [],
|
||||
});
|
||||
}
|
||||
|
||||
@@ -425,11 +830,67 @@ export class ChatService {
|
||||
return messages;
|
||||
}
|
||||
|
||||
private stripMediaUrls(messages: ScrapedChatMessage[]): ChatMessage[] {
|
||||
return messages.map((message) => ({
|
||||
sender: message.sender,
|
||||
text: message.text,
|
||||
timestamp: message.timestamp,
|
||||
isSelf: message.isSelf,
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
private async scrapeChats(): Promise<ActiveChat[]> {
|
||||
const chats: ActiveChat[] = [];
|
||||
|
||||
try {
|
||||
const shadowRooms = await this.scrapeChatsViaShadowDom();
|
||||
if (shadowRooms.length > 0) {
|
||||
for (const room of shadowRooms) {
|
||||
if (!room.href) continue;
|
||||
if (room.href.includes('/create')) continue;
|
||||
|
||||
const url = room.href.startsWith('http')
|
||||
? room.href
|
||||
: `https://chat.reddit.com${room.href}`;
|
||||
const roomIdMatch = room.href.match(/\/room\/([^/]+)/);
|
||||
const roomId = roomIdMatch ? decodeURIComponent(roomIdMatch[1]) : undefined;
|
||||
|
||||
const roomName = (room.roomName || room.ariaLabel || 'Unknown').trim();
|
||||
let lastMessage = (room.lastMessage || '').trim();
|
||||
const isFromUs = /^you[:\s·-]/i.test(lastMessage);
|
||||
if (isFromUs && lastMessage) {
|
||||
lastMessage = lastMessage.replace(/^you[:\s·-]+/i, '').trim();
|
||||
}
|
||||
|
||||
const timestamp = room.timeText ? this.parseRelativeTime(room.timeText) : Date.now();
|
||||
|
||||
chats.push({
|
||||
username: roomName,
|
||||
roomName,
|
||||
lastMessageTime: timestamp,
|
||||
lastMessageSnippet: lastMessage,
|
||||
isFromUs,
|
||||
unread: room.unread,
|
||||
url,
|
||||
roomId,
|
||||
});
|
||||
}
|
||||
|
||||
const chatMap = new Map<string, ActiveChat>();
|
||||
for (const chat of chats) {
|
||||
if (chat.url && !chatMap.has(chat.url)) {
|
||||
chatMap.set(chat.url, chat);
|
||||
}
|
||||
}
|
||||
|
||||
const deduped = Array.from(chatMap.values())
|
||||
.sort((a, b) => b.lastMessageTime - a.lastMessageTime);
|
||||
|
||||
this.logger.info(`Scraped ${deduped.length} rooms via shadow DOM`);
|
||||
return deduped;
|
||||
}
|
||||
|
||||
// Look for room links in the sidebar
|
||||
this.logger.debug('Looking for chat room links...');
|
||||
const roomLinks = await this.page.$$('a[href^="/room/"]');
|
||||
@@ -537,6 +998,119 @@ export class ChatService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Public method to persist current cached chats to disk.
|
||||
* Used by AutoReplyService after updating pendingReply/replyStatus fields.
|
||||
*/
|
||||
async saveCacheExternal(): Promise<void> {
|
||||
await this.saveCache(this.cachedChats);
|
||||
}
|
||||
|
||||
private async saveCache(chats: ActiveChat[]): Promise<void> {
|
||||
const payload: ChatCacheFile = {
|
||||
version: 1,
|
||||
savedAt: Date.now(),
|
||||
chats,
|
||||
};
|
||||
try {
|
||||
await fs.mkdir(path.dirname(this.cacheFilePath), { recursive: true });
|
||||
await fs.writeFile(this.cacheFilePath, JSON.stringify(payload, null, 2), 'utf-8');
|
||||
this.logger.info(`Saved ${chats.length} chats to cache`);
|
||||
} catch (error) {
|
||||
this.logger.warn('Failed to save chat cache:', error);
|
||||
}
|
||||
}
|
||||
|
||||
private async scrapeChatsViaShadowDom(): Promise<Array<{
|
||||
href: string;
|
||||
ariaLabel: string | null;
|
||||
roomName: string | null;
|
||||
lastMessage: string | null;
|
||||
timeText: string | null;
|
||||
unread: boolean;
|
||||
}>> {
|
||||
try {
|
||||
return await this.page.evaluate(`
|
||||
(function() {
|
||||
function collectAllElements(root) {
|
||||
var elements = [];
|
||||
if (!root) return elements;
|
||||
var treeWalker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT, null);
|
||||
var current = treeWalker.currentNode;
|
||||
while (current) {
|
||||
elements.push(current);
|
||||
if (current.shadowRoot) {
|
||||
elements = elements.concat(collectAllElements(current.shadowRoot));
|
||||
}
|
||||
current = treeWalker.nextNode();
|
||||
}
|
||||
return elements;
|
||||
}
|
||||
|
||||
function findByClassParts(root, parts) {
|
||||
var elements = collectAllElements(root);
|
||||
for (var i = 0; i < elements.length; i++) {
|
||||
var el = elements[i];
|
||||
if (!el.className || typeof el.className !== 'string') continue;
|
||||
var className = el.className.toLowerCase();
|
||||
for (var j = 0; j < parts.length; j++) {
|
||||
if (className.includes(parts[j])) {
|
||||
var text = (el.textContent || '').trim();
|
||||
if (text.length > 0) return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function hasUnreadBadge(root) {
|
||||
var elements = collectAllElements(root);
|
||||
for (var i = 0; i < elements.length; i++) {
|
||||
var el = elements[i];
|
||||
var className = (el.className || '').toString().toLowerCase();
|
||||
var testId = (el.getAttribute && el.getAttribute('data-testid')) || '';
|
||||
var ariaLabel = (el.getAttribute && el.getAttribute('aria-label')) || '';
|
||||
if (className.includes('unread') || className.includes('badge')) return true;
|
||||
if (testId.toLowerCase().includes('unread')) return true;
|
||||
if (ariaLabel.toLowerCase().includes('unread')) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
var elements = collectAllElements(document);
|
||||
var anchors = elements.filter(function(el) {
|
||||
return el.tagName === 'A' && el.getAttribute('href') && el.getAttribute('href').startsWith('/room/');
|
||||
});
|
||||
|
||||
var rooms = [];
|
||||
for (var i = 0; i < anchors.length; i++) {
|
||||
var a = anchors[i];
|
||||
var href = a.getAttribute('href');
|
||||
if (!href) continue;
|
||||
|
||||
var roomName = findByClassParts(a, ['room-name', 'roomname', 'room_title', 'roomtitle']);
|
||||
var lastMessage = findByClassParts(a, ['last-message', 'lastmessage', 'preview', 'message-preview', 'messagepreview']);
|
||||
var timeText = findByClassParts(a, ['last-message-time', 'lastmessagetime', 'timestamp', 'time']);
|
||||
|
||||
rooms.push({
|
||||
href: href,
|
||||
ariaLabel: a.getAttribute('aria-label'),
|
||||
roomName: roomName,
|
||||
lastMessage: lastMessage,
|
||||
timeText: timeText,
|
||||
unread: hasUnreadBadge(a)
|
||||
});
|
||||
}
|
||||
|
||||
return rooms;
|
||||
})()
|
||||
`);
|
||||
} catch (error) {
|
||||
this.logger.warn('Failed to scrape chats via shadow DOM:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private parseRelativeTime(text: string): number {
|
||||
const now = Date.now();
|
||||
const clean = text.toLowerCase().trim();
|
||||
|
||||
Reference in new issue
Block a user