Merge remote-tracking branch 'origin/main'
# Conflicts: # src/browser/client.ts # src/services/reddit-session-importer.ts # src/web/routes/conversations.ts
This commit is contained in:
commit
48492d5238
30 files changed
+1041
-181
No files matched your search
+12
-12
@@ -37,7 +37,7 @@ import { RateLimiter } from './utils/rate-limiter.js';
|
||||
import { WebServer } from './web/server.js';
|
||||
import { reportReply } from './web/event-reporting.js';
|
||||
import { RedditSessionImporter } from './services/reddit-session-importer.js';
|
||||
import { AccountAuthContext } from './services/account-auth-context.js';
|
||||
import { AccountAuthContext, authenticateCurrentAccount } from './services/account-auth-context.js';
|
||||
import { ConfigStore } from './config/persistence.js';
|
||||
import { RedditAccountsRepository, AccountContentRepository } from './database/account-repositories.js';
|
||||
import { RedditAccountsService } from './services/reddit-accounts.js';
|
||||
@@ -199,17 +199,17 @@ export async function bootstrap(
|
||||
};
|
||||
}
|
||||
let activeMatrixClient = new MatrixClient(matrixCreds, logger);
|
||||
accountAuth.setAuthenticator(async () => {
|
||||
const redditUsername = await browserClient.getAuthenticatedUsername();
|
||||
if (!redditUsername) return null;
|
||||
try {
|
||||
const matrixUserId = await activeMatrixClient.whoami();
|
||||
if (!matrixUserId) return null;
|
||||
return { accountId: matrixUserId, redditUsername, matrixUserId };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
accountAuth.setAuthenticator(() => authenticateCurrentAccount({
|
||||
getRedditIdentity: () => browserClient.getAuthenticatedIdentity(),
|
||||
refreshMatrixCredentials: async () => {
|
||||
try {
|
||||
const userId = await activeMatrixClient.whoami();
|
||||
return userId ? { userId } : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
}));
|
||||
const fullyAuthenticated = redditLoggedIn && await accountAuth.ensureFullyAuthenticated();
|
||||
if (fullyAuthenticated) logger.info('Reddit and Matrix authentication verified');
|
||||
broadcaster.dashboardEvents.redditSession({
|
||||
|
||||
+26
-19
@@ -3,6 +3,18 @@ import type { Config } from '../config/index.js';
|
||||
import type { Logger } from '../utils/logger.js';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import { z } from 'zod';
|
||||
import type { RedditAccountIdentity } from '../services/account-auth-context.js';
|
||||
|
||||
const RedditAccountSchema = z.object({
|
||||
name: z.string().trim().min(1),
|
||||
id: z.string().trim().min(1),
|
||||
}).passthrough();
|
||||
|
||||
const RedditIdentityResponseSchema = z.union([
|
||||
RedditAccountSchema,
|
||||
z.object({ data: RedditAccountSchema }).passthrough().transform(({ data }) => data),
|
||||
]);
|
||||
|
||||
export interface BrowserRuntime {
|
||||
launchPersistentContext(
|
||||
@@ -115,17 +127,13 @@ export class BrowserClient {
|
||||
|
||||
/** Returns the account name from Reddit's authenticated identity endpoint. */
|
||||
async getAuthenticatedUsername(): Promise<string | null> {
|
||||
try {
|
||||
return (await this.getRedditIdentity(await this.getPage()))?.username ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return (await this.getAuthenticatedIdentity())?.username ?? null;
|
||||
}
|
||||
|
||||
async getAuthenticatedIdentity(): Promise<{ username: string; accountId: string } | null> {
|
||||
/** Returns the username and stable t2 account ID for identity binding. */
|
||||
async getAuthenticatedIdentity(): Promise<RedditAccountIdentity | null> {
|
||||
try {
|
||||
const identity = await this.getRedditIdentity(await this.getPage());
|
||||
return identity?.accountId ? { username: identity.username, accountId: identity.accountId } : null;
|
||||
return await this.getRedditIdentity(await this.getPage());
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
@@ -180,25 +188,24 @@ export class BrowserClient {
|
||||
return (await this.getRedditIdentity(page)) !== null;
|
||||
}
|
||||
|
||||
private async getRedditIdentity(page: Page): Promise<{ username: string; accountId: string | null } | null> {
|
||||
private async getRedditIdentity(page: Page): Promise<RedditAccountIdentity | null> {
|
||||
try {
|
||||
return await page.evaluate(async () => {
|
||||
const payload: unknown = await page.evaluate(async () => {
|
||||
try {
|
||||
const res = await fetch('https://www.reddit.com/api/me.json', { credentials: 'include' });
|
||||
if (!res.ok) return null;
|
||||
const data: unknown = await res.json();
|
||||
if (!data || typeof data !== 'object') return null;
|
||||
const account = 'data' in data && data.data && typeof data.data === 'object'
|
||||
? data.data
|
||||
: data;
|
||||
if (!('name' in account) || typeof account.name !== 'string') return null;
|
||||
const username = account.name.trim();
|
||||
const accountId = 'id' in account && typeof account.id === 'string' ? account.id.trim() : '';
|
||||
return username ? { username, accountId: accountId || null } : null;
|
||||
return await res.json();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
const parsed = RedditIdentityResponseSchema.safeParse(payload);
|
||||
if (!parsed.success) return null;
|
||||
const rawAccountId = parsed.data.id;
|
||||
return {
|
||||
username: parsed.data.name,
|
||||
accountId: rawAccountId.startsWith('t2_') ? rawAccountId : `t2_${rawAccountId}`,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -258,29 +258,6 @@ export async function runMainLoop(app: AppContainer): Promise<void> {
|
||||
processedAt: Date.now(),
|
||||
});
|
||||
|
||||
// Skip OF/seller accounts before spending on AI.
|
||||
if (
|
||||
config.reddit.filterSellers &&
|
||||
(config.reddit.sellerKeywords.length > 0 || config.reddit.sellerDomains.length > 0)
|
||||
) {
|
||||
const isSeller = await redditApi.isSellerProfile(
|
||||
post.author,
|
||||
config.reddit.sellerKeywords,
|
||||
config.reddit.sellerDomains
|
||||
);
|
||||
if (isSeller) {
|
||||
logger.info(`Skipping u/${post.author} - seller/OnlyFans profile detected`);
|
||||
webServer?.logActivity(
|
||||
'skipped',
|
||||
`Skipped post by u/${post.author} (seller/OnlyFans profile)`,
|
||||
post.author,
|
||||
);
|
||||
webServer?.pushLog('info', `🚫 Skipped u/${post.author} - seller/OnlyFans profile`);
|
||||
postsRepo.setError(post.id, 'Seller profile detected');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh examples weighted toward this post's subreddit.
|
||||
const subredditExamples = responseChecker.getSuccessfulExamples(3, post.subreddit);
|
||||
processorService.setSuccessfulExamples(subredditExamples);
|
||||
|
||||
+105
-29
@@ -48,6 +48,38 @@ export interface RedditPost {
|
||||
link_flair_text?: string | null;
|
||||
}
|
||||
|
||||
interface RedditListingChild {
|
||||
data?: {
|
||||
title?: unknown;
|
||||
selftext?: unknown;
|
||||
body?: unknown;
|
||||
url?: unknown;
|
||||
subreddit?: unknown;
|
||||
};
|
||||
}
|
||||
|
||||
function joinText(values: unknown[]): string {
|
||||
return values.filter((value): value is string => typeof value === 'string').join(' ');
|
||||
}
|
||||
|
||||
/** Match configured seller signals in user-controlled text. */
|
||||
export function findSellerSignal(
|
||||
text: string,
|
||||
keywords: string[],
|
||||
domains: string[],
|
||||
): { kind: 'keyword' | 'domain'; value: string } | null {
|
||||
const normalized = text.toLocaleLowerCase();
|
||||
for (const keyword of keywords) {
|
||||
const candidate = keyword.trim().toLocaleLowerCase();
|
||||
if (candidate && normalized.includes(candidate)) {
|
||||
return { kind: 'keyword', value: keyword };
|
||||
}
|
||||
}
|
||||
|
||||
const domain = findSellerDomain(text, domains);
|
||||
return domain ? { kind: 'domain', value: domain } : null;
|
||||
}
|
||||
|
||||
// Extracts URL-ish tokens (with or without scheme) and returns the first
|
||||
// domain match. Matches `host` or any subdomain (`*.host`), case-insensitive.
|
||||
export function findSellerDomain(text: string, domains: string[]): string | null {
|
||||
@@ -249,10 +281,28 @@ export class RedditApiClient {
|
||||
async isSellerProfile(
|
||||
username: string,
|
||||
sellerKeywords: string[],
|
||||
sellerDomains: string[] = []
|
||||
sellerDomains: string[] = [],
|
||||
currentPost?: RedditPost,
|
||||
): Promise<boolean> {
|
||||
if (sellerKeywords.length === 0 && sellerDomains.length === 0) return false;
|
||||
|
||||
// Check the post first. This catches direct promotional posts without an
|
||||
// extra request and also covers accounts whose profile bio is deliberately
|
||||
// vague.
|
||||
if (currentPost) {
|
||||
const postSignal = findSellerSignal(
|
||||
joinText([currentPost.title, currentPost.selftext, currentPost.url]),
|
||||
sellerKeywords,
|
||||
sellerDomains,
|
||||
);
|
||||
if (postSignal) {
|
||||
this.logger.info(
|
||||
`u/${username} flagged as seller from current post (matched ${postSignal.kind}: "${postSignal.value}")`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const cookieHeader = await this.getCookieHeader();
|
||||
const headers: Record<string, string> = {};
|
||||
@@ -260,51 +310,77 @@ export class RedditApiClient {
|
||||
headers['Cookie'] = cookieHeader;
|
||||
}
|
||||
|
||||
const response = await this.client.get(`/user/${username}/about.json`, {
|
||||
const response = await this.client.get(`/user/${encodeURIComponent(username)}/about.json`, {
|
||||
params: { raw_json: 1 },
|
||||
headers,
|
||||
});
|
||||
|
||||
const userData = response.data?.data;
|
||||
if (!userData) return false;
|
||||
|
||||
// Combine all text fields to scan
|
||||
const textToCheck = [
|
||||
userData.subreddit?.public_description || '',
|
||||
userData.subreddit?.title || '',
|
||||
userData.subreddit?.display_name_prefixed || '',
|
||||
]
|
||||
.join(' ')
|
||||
.toLowerCase();
|
||||
|
||||
for (const keyword of sellerKeywords) {
|
||||
if (textToCheck.includes(keyword.toLowerCase())) {
|
||||
if (userData) {
|
||||
const profileText = joinText([
|
||||
userData.subreddit?.public_description || '',
|
||||
userData.subreddit?.title || '',
|
||||
userData.subreddit?.display_name_prefixed || '',
|
||||
]);
|
||||
const profileSignal = findSellerSignal(profileText, sellerKeywords, sellerDomains);
|
||||
if (profileSignal) {
|
||||
this.logger.info(
|
||||
`u/${username} flagged as seller profile (matched keyword: "${keyword}")`
|
||||
`u/${username} flagged as seller profile (matched ${profileSignal.kind}: "${profileSignal.value}")`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (sellerDomains.length > 0) {
|
||||
const matchedDomain = findSellerDomain(textToCheck, sellerDomains);
|
||||
if (matchedDomain) {
|
||||
this.logger.info(
|
||||
`u/${username} flagged as seller profile (matched domain: "${matchedDomain}")`
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
if (axios.isAxiosError(error) && error.response?.status === 429) {
|
||||
const rateLimitError = new RedditRateLimitError(getRateLimitDelayMs(error.response.headers));
|
||||
this.logger.warn(rateLimitError.message);
|
||||
throw rateLimitError;
|
||||
}
|
||||
if (axios.isAxiosError(error) && error.response?.status === 404) {
|
||||
this.logger.warn(`User u/${username} profile not found`);
|
||||
return false;
|
||||
}
|
||||
this.logger.warn(`Failed to check profile for u/${username}:`, error);
|
||||
return false; // Don't block on errors - let it through
|
||||
}
|
||||
|
||||
// Bios are often clean while recent posts advertise the paid account.
|
||||
// Scan a small overview page, including comment bodies and subreddit
|
||||
// names, before allowing the candidate through.
|
||||
try {
|
||||
const cookieHeader = await this.getCookieHeader();
|
||||
const headers: Record<string, string> = {};
|
||||
if (cookieHeader) headers.Cookie = cookieHeader;
|
||||
const response = await this.client.get(`/user/${encodeURIComponent(username)}/overview.json`, {
|
||||
params: { limit: 25, raw_json: 1 },
|
||||
headers,
|
||||
});
|
||||
const children: RedditListingChild[] = Array.isArray(response.data?.data?.children)
|
||||
? response.data.data.children
|
||||
: [];
|
||||
const activityText = children.map(({ data }) => joinText([
|
||||
data?.title,
|
||||
data?.selftext,
|
||||
data?.body,
|
||||
data?.url,
|
||||
data?.subreddit,
|
||||
])).join(' ');
|
||||
const activitySignal = findSellerSignal(activityText, sellerKeywords, sellerDomains);
|
||||
if (activitySignal) {
|
||||
this.logger.info(
|
||||
`u/${username} flagged as seller from recent activity (matched ${activitySignal.kind}: "${activitySignal.value}")`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
if (axios.isAxiosError(error) && error.response?.status === 429) {
|
||||
const rateLimitError = new RedditRateLimitError(getRateLimitDelayMs(error.response.headers));
|
||||
this.logger.warn(rateLimitError.message);
|
||||
throw rateLimitError;
|
||||
}
|
||||
this.logger.warn(`Failed to check recent activity for u/${username}:`, error);
|
||||
}
|
||||
|
||||
return false; // Fail open if Reddit profile endpoints are unavailable.
|
||||
}
|
||||
|
||||
getPostUrl(post: RedditPost): string {
|
||||
|
||||
@@ -6,11 +6,18 @@ export type AccountAuthStatus =
|
||||
| 'authenticated';
|
||||
|
||||
export interface AuthenticatedAccount {
|
||||
/** Reddit account fullname, e.g. `t2_abc123`. */
|
||||
accountId: string;
|
||||
redditUsername: string;
|
||||
matrixUserId: string;
|
||||
}
|
||||
|
||||
export interface RedditAccountIdentity {
|
||||
username: string;
|
||||
/** Reddit's account ID, with or without the `t2_` prefix. */
|
||||
accountId: string;
|
||||
}
|
||||
|
||||
export interface AccountAuthSnapshot {
|
||||
status: AccountAuthStatus;
|
||||
accountId: string | null;
|
||||
@@ -34,21 +41,43 @@ export interface AccountAuthContextOptions {
|
||||
}
|
||||
|
||||
export interface CurrentAccountAuthentication {
|
||||
getRedditUsername(): Promise<string | null>;
|
||||
getRedditIdentity(): Promise<RedditAccountIdentity | null>;
|
||||
refreshMatrixCredentials(): Promise<{ userId: string } | null>;
|
||||
}
|
||||
|
||||
export function normalizeRedditAccountId(accountId: string): string {
|
||||
const trimmed = accountId.trim();
|
||||
const normalized = trimmed.startsWith('t2_') ? trimmed : `t2_${trimmed}`;
|
||||
if (!/^t2_[a-z0-9]+$/i.test(normalized)) {
|
||||
throw new Error('Reddit returned an invalid account ID');
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function expectedMatrixUserId(accountId: string): string {
|
||||
return `@${normalizeRedditAccountId(accountId)}:reddit.com`;
|
||||
}
|
||||
|
||||
export function assertMatchingAccountIdentity(account: AuthenticatedAccount): void {
|
||||
const expected = expectedMatrixUserId(account.accountId);
|
||||
if (account.matrixUserId !== expected) {
|
||||
throw new Error(
|
||||
`Reddit account u/${account.redditUsername} (${normalizeRedditAccountId(account.accountId)}) does not match Matrix account ${account.matrixUserId}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function authenticateCurrentAccount(
|
||||
authentication: CurrentAccountAuthentication,
|
||||
): Promise<AuthenticatedAccount | null> {
|
||||
const redditUsername = await authentication.getRedditUsername();
|
||||
if (!redditUsername) return null;
|
||||
const redditIdentity = await authentication.getRedditIdentity();
|
||||
if (!redditIdentity) return null;
|
||||
|
||||
const credentials = await authentication.refreshMatrixCredentials();
|
||||
if (!credentials?.userId) return null;
|
||||
return {
|
||||
accountId: credentials.userId,
|
||||
redditUsername,
|
||||
accountId: normalizeRedditAccountId(redditIdentity.accountId),
|
||||
redditUsername: redditIdentity.username,
|
||||
matrixUserId: credentials.userId,
|
||||
};
|
||||
}
|
||||
@@ -104,15 +133,15 @@ export class AccountAuthContext implements AccountAuthGate {
|
||||
if (!account.accountId || !account.redditUsername || !account.matrixUserId) {
|
||||
throw new Error('A fully authenticated account requires Reddit and Matrix identities');
|
||||
}
|
||||
if (account.accountId !== account.matrixUserId) {
|
||||
throw new Error('Authenticated account ID must match the Matrix user ID');
|
||||
}
|
||||
assertMatchingAccountIdentity(account);
|
||||
|
||||
const changedAccount = this.snapshot.accountId !== account.accountId;
|
||||
const accountId = normalizeRedditAccountId(account.accountId);
|
||||
|
||||
const changedAccount = this.snapshot.accountId !== accountId;
|
||||
const wasAuthenticated = this.isFullyAuthenticated();
|
||||
this.snapshot = {
|
||||
status: 'authenticated',
|
||||
accountId: account.accountId,
|
||||
accountId,
|
||||
redditUsername: account.redditUsername,
|
||||
matrixUserId: account.matrixUserId,
|
||||
reason: null,
|
||||
|
||||
@@ -54,6 +54,11 @@ export interface RoomSummary {
|
||||
recentEvents: MatrixTimelineEvent[];
|
||||
}
|
||||
|
||||
export interface DirectRoomMapping {
|
||||
recipientMxid: string;
|
||||
roomId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cap on how long we'll sit and sleep waiting out a 429. Anything longer
|
||||
* is surfaced as a `MatrixRateLimitError` so the main loop can back off
|
||||
@@ -405,7 +410,7 @@ export class MatrixClient {
|
||||
* "Start chat" in the UI:
|
||||
*
|
||||
* POST /_matrix/client/v3/createRoom
|
||||
* { "preset": "reddit_dm", "invite": [mxid] }
|
||||
* { "preset": "reddit_dm", "invite": [mxid], "is_direct": true }
|
||||
*
|
||||
* `reddit_dm` is a Reddit-specific preset that the homeserver expands
|
||||
* server-side into the right power levels, history visibility, and
|
||||
@@ -413,12 +418,53 @@ export class MatrixClient {
|
||||
*/
|
||||
async createDirectRoom(recipientMxid: string): Promise<string> {
|
||||
const res = await this.request<{ room_id: string }>('POST', '/_matrix/client/v3/createRoom', {
|
||||
body: { preset: 'reddit_dm', invite: [recipientMxid] },
|
||||
body: { preset: 'reddit_dm', invite: [recipientMxid], is_direct: true },
|
||||
});
|
||||
this.logger.debug(`[matrix] created DM room ${res.room_id} with ${recipientMxid}`);
|
||||
return res.room_id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add missing rooms to the account-level `m.direct` map used by Matrix
|
||||
* clients to classify joined rooms as direct chats. Existing mappings are
|
||||
* preserved verbatim; the write is skipped when there is nothing to repair.
|
||||
*/
|
||||
async repairDirectRoomMappings(mappings: DirectRoomMapping[]): Promise<number> {
|
||||
if (mappings.length === 0) return 0;
|
||||
|
||||
let directRooms: Record<string, string[]>;
|
||||
try {
|
||||
directRooms = await this.request<Record<string, string[]>>(
|
||||
'GET',
|
||||
`/_matrix/client/v3/user/${encodeURIComponent(this.creds.userId)}/account_data/m.direct`,
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.includes('→ 404')) {
|
||||
directRooms = {};
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
let repaired = 0;
|
||||
for (const { recipientMxid, roomId } of mappings) {
|
||||
const existing = Array.isArray(directRooms[recipientMxid])
|
||||
? directRooms[recipientMxid]
|
||||
: [];
|
||||
if (existing.includes(roomId)) continue;
|
||||
directRooms[recipientMxid] = [...existing, roomId];
|
||||
repaired++;
|
||||
}
|
||||
|
||||
if (repaired === 0) return 0;
|
||||
await this.request<Record<string, never>>(
|
||||
'PUT',
|
||||
`/_matrix/client/v3/user/${encodeURIComponent(this.creds.userId)}/account_data/m.direct`,
|
||||
{ body: directRooms },
|
||||
);
|
||||
return repaired;
|
||||
}
|
||||
|
||||
/**
|
||||
* One-shot: create a DM room with the recipient and send the opening
|
||||
* message. Returns the new room_id and the event_id of the message.
|
||||
@@ -428,6 +474,13 @@ export class MatrixClient {
|
||||
body: string,
|
||||
): Promise<{ roomId: string; eventId: string }> {
|
||||
const roomId = await this.createDirectRoom(recipientMxid);
|
||||
try {
|
||||
await this.repairDirectRoomMappings([{ recipientMxid, roomId }]);
|
||||
} catch (error) {
|
||||
// Room creation has already succeeded. Do not turn a metadata repair
|
||||
// failure into an orphan-and-retry cycle that creates duplicate rooms.
|
||||
this.logger.warn(`[matrix] failed to register ${roomId} as a direct chat:`, error);
|
||||
}
|
||||
const eventId = await this.sendTextMessage(roomId, body);
|
||||
return { roomId, eventId };
|
||||
}
|
||||
|
||||
@@ -402,6 +402,7 @@ export class ChatService {
|
||||
if (!chat.roomName || chat.roomName.startsWith('t2_')) chat.roomName = resolved;
|
||||
}
|
||||
await this.cache.save(chats);
|
||||
await this.repairDirectRoomMappings(chats);
|
||||
|
||||
let done = 0;
|
||||
for (const chat of chats) {
|
||||
@@ -736,17 +737,21 @@ export class ChatService {
|
||||
existing.set(roomId, next);
|
||||
}
|
||||
|
||||
if (this.mediaService) {
|
||||
for (const [roomId, room] of Object.entries(joined)) {
|
||||
const scraped = this.scrapeMessageEvents(room?.timeline?.events ?? []);
|
||||
const mediaMessages = scraped.filter((m) => m.mediaUrls.length > 0);
|
||||
if (mediaMessages.length === 0) continue;
|
||||
const chat = existing.get(roomId);
|
||||
if (!chat) continue;
|
||||
for (const [roomId, room] of Object.entries(joined)) {
|
||||
const scraped = this.scrapeMessageEvents(room?.timeline?.events ?? []);
|
||||
if (scraped.length === 0) continue;
|
||||
const chat = existing.get(roomId);
|
||||
if (!chat) continue;
|
||||
|
||||
let newMessages: ChatMessage[] = scraped.map(
|
||||
({ mediaUrls: _ignored, ...message }) => message,
|
||||
);
|
||||
if (this.mediaService && scraped.some((message) => message.mediaUrls.length > 0)) {
|
||||
try {
|
||||
const withMedia = await this.mediaService.attachMedia(chat, mediaMessages);
|
||||
this.mergeNewMessages(chat, withMedia);
|
||||
if (withMedia.some((message) => !message.isSelf && (message.media?.length ?? 0) > 0)) {
|
||||
newMessages = await this.mediaService.attachMedia(chat, scraped);
|
||||
if (newMessages.some(
|
||||
(message) => !message.isSelf && (message.media?.length ?? 0) > 0,
|
||||
)) {
|
||||
this.recordReceivedImage(chat);
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -755,6 +760,7 @@ export class ChatService {
|
||||
);
|
||||
}
|
||||
}
|
||||
this.mergeNewMessages(chat, newMessages);
|
||||
}
|
||||
|
||||
const merged = Array.from(existing.values()).sort(
|
||||
@@ -766,9 +772,49 @@ export class ChatService {
|
||||
// Resolve any t2_-prefixed usernames the delta just introduced, so the
|
||||
// dashboard doesn't show raw Matrix-style IDs for new rooms.
|
||||
await this.enrichUsernames();
|
||||
await this.repairDirectRoomMappings(merged);
|
||||
return merged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile cached one-to-one chats with Matrix account data. The durable
|
||||
* username repository lets delta sync repair older rooms even when Reddit
|
||||
* omits their member state from the latest /sync response.
|
||||
*/
|
||||
private async repairDirectRoomMappings(chats: ActiveChat[]): Promise<void> {
|
||||
const mappings: Array<{ recipientMxid: string; roomId: string }> = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const chat of chats) {
|
||||
if (!chat.roomId) continue;
|
||||
const rawT2 = this.extractT2(chat);
|
||||
const persistedT2 = rawT2
|
||||
? `t2_${rawT2}`
|
||||
: this.redditUsersRepo?.getByUsername(chat.username)?.t2Id;
|
||||
if (!persistedT2) continue;
|
||||
|
||||
const recipientMxid = MatrixClient.t2ToMxid(persistedT2);
|
||||
const key = `${recipientMxid}|${chat.roomId}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
mappings.push({ recipientMxid, roomId: chat.roomId });
|
||||
}
|
||||
|
||||
if (mappings.length === 0) return;
|
||||
try {
|
||||
const repaired = await this.withAuth(() =>
|
||||
this.matrix.repairDirectRoomMappings(mappings),
|
||||
);
|
||||
if (repaired > 0) {
|
||||
this.logger.info(`[chat] repaired ${repaired} missing direct-room mapping(s)`);
|
||||
}
|
||||
} catch (error) {
|
||||
// Sync remains useful even if Reddit temporarily rejects account-data
|
||||
// maintenance. A later delta/full sync will try the additive repair again.
|
||||
this.logger.warn('[chat] direct-room mapping repair failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
private scrapeMessageEvents(events: MatrixTimelineEvent[]): ScrapedChatMessage[] {
|
||||
return events
|
||||
.filter((e) => e.type === 'm.room.message')
|
||||
|
||||
+57
-27
@@ -36,9 +36,12 @@ export class MonitorService {
|
||||
this.logger.debug(`Checking r/${subreddit.name} (${subreddit.type}) for new posts...`);
|
||||
const posts = await this.redditApi.getNewPosts(subreddit.name, 25, subreddit.type);
|
||||
|
||||
// Filter for valid, unprocessed posts
|
||||
const validPosts: RedditPost[] = [];
|
||||
for (const post of posts) {
|
||||
// Reddit normally returns /new in descending order, but sort explicitly
|
||||
// so we can stop as soon as the first fully eligible candidate is found.
|
||||
// Previously we screened the entire batch before choosing one, causing up
|
||||
// to two profile requests for every otherwise-valid post in the listing.
|
||||
const newestFirst = [...posts].sort((a, b) => b.created_utc - a.created_utc);
|
||||
for (const post of newestFirst) {
|
||||
if (this.postsRepo.isProcessed(post.id)) {
|
||||
continue;
|
||||
}
|
||||
@@ -48,6 +51,23 @@ export class MonitorService {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
this.config.reddit.filterSellers &&
|
||||
(this.config.reddit.sellerKeywords.length > 0 || this.config.reddit.sellerDomains.length > 0)
|
||||
) {
|
||||
const isSeller = await this.redditApi.isSellerProfile(
|
||||
post.author,
|
||||
this.config.reddit.sellerKeywords,
|
||||
this.config.reddit.sellerDomains,
|
||||
post,
|
||||
);
|
||||
if (isSeller) {
|
||||
this.logger.info(`Skipping post ${post.id} from u/${post.author} - seller signals detected`);
|
||||
this.recordRejection(post, subreddit.type, 'Seller signals detected');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Chat subs almost always have a self-declared gender tag in the
|
||||
// title/body (e.g. [25F], M4F). Drop posts that don't match the
|
||||
// configured targetGenders here so they never enter the processor.
|
||||
@@ -55,41 +75,33 @@ export class MonitorService {
|
||||
const tag = parseGenderTag(`${post.title}\n${post.selftext ?? ''}`);
|
||||
if (!tag) {
|
||||
this.logger.debug(`Skipping chat post ${post.id} - no gender tag in title/body`);
|
||||
this.recordRejection(post, subreddit.type, 'No gender tag in title/body');
|
||||
continue;
|
||||
}
|
||||
if (!targetGenders.includes(tag)) {
|
||||
this.logger.debug(
|
||||
`Skipping chat post ${post.id} - gender tag "${tag}" not in target list [${targetGenders.join(', ')}]`,
|
||||
);
|
||||
this.recordRejection(post, subreddit.type, `Gender tag "${tag}" not targeted`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
validPosts.push(post);
|
||||
}
|
||||
|
||||
if (validPosts.length > 0) {
|
||||
// Sort by creation time (newest first)
|
||||
validPosts.sort((a, b) => b.created_utc - a.created_utc);
|
||||
|
||||
if (prioritizeRecent) {
|
||||
const recent = validPosts.find((p) => p.created_utc >= recentCutoffSec);
|
||||
if (recent) {
|
||||
this.logger.info(`Found ${validPosts.length} new posts in r/${subreddit.name} (recent match)`);
|
||||
return [recent];
|
||||
}
|
||||
// Remember the freshest non-recent candidate as a fallback and
|
||||
// keep scanning other subs for an actually-recent post.
|
||||
if (!fallback || validPosts[0].created_utc > fallback.created_utc) {
|
||||
fallback = validPosts[0];
|
||||
}
|
||||
} else {
|
||||
this.logger.info(`Found ${validPosts.length} new posts in r/${subreddit.name}`);
|
||||
// Return just the first (newest) valid post
|
||||
// This ensures we switch to a different random subreddit on the next poll cycle
|
||||
// instead of processing a batch from the same subreddit
|
||||
return [validPosts[0]];
|
||||
// At this point the candidate has passed every check. Select it now
|
||||
// rather than spending more Reddit requests screening users that
|
||||
// cannot be acted on during this poll cycle.
|
||||
if (!prioritizeRecent || post.created_utc >= recentCutoffSec) {
|
||||
this.logger.info(
|
||||
`Found eligible post in r/${subreddit.name}${prioritizeRecent ? ' (recent match)' : ''}`,
|
||||
);
|
||||
return [post];
|
||||
}
|
||||
|
||||
// This is the newest eligible post in this subreddit. Keep it as a
|
||||
// possible fallback, then move to the next subreddit to look for a
|
||||
// recent candidate. Older posts in this batch cannot improve it.
|
||||
if (!fallback || post.created_utc > fallback.created_utc) fallback = post;
|
||||
break;
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof RedditRateLimitError) throw error;
|
||||
@@ -160,4 +172,22 @@ export class MonitorService {
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Persist a completed screening decision so later polls do not repeat it. */
|
||||
private recordRejection(
|
||||
post: RedditPost,
|
||||
subredditType: 'image' | 'chat',
|
||||
reason: string,
|
||||
): void {
|
||||
this.postsRepo.insert({
|
||||
id: post.id,
|
||||
subreddit: post.subreddit,
|
||||
author: post.author,
|
||||
title: post.title,
|
||||
postUrl: this.redditApi.getPostUrl(post),
|
||||
imageUrl: subredditType === 'image' ? this.redditApi.getImageUrl(post) : null,
|
||||
processedAt: Date.now(),
|
||||
});
|
||||
this.postsRepo.setError(post.id, reason);
|
||||
}
|
||||
}
|
||||
@@ -2,12 +2,16 @@ import type { Cookie } from 'playwright';
|
||||
import type { MatrixClient } from './chat/matrix/client.js';
|
||||
import type { MatrixCredentials } from './chat/matrix/types.js';
|
||||
import type { AccountAuthContext } from './account-auth-context.js';
|
||||
import {
|
||||
assertMatchingAccountIdentity,
|
||||
normalizeRedditAccountId,
|
||||
type RedditAccountIdentity,
|
||||
} from './account-auth-context.js';
|
||||
|
||||
interface BrowserSessionTarget {
|
||||
importRedditCookies(cookies: Cookie[]): Promise<void>;
|
||||
isSessionValid(): Promise<boolean>;
|
||||
getAuthenticatedUsername(): Promise<string | null>;
|
||||
getAuthenticatedIdentity?(): Promise<{ username: string; accountId: string } | null>;
|
||||
getAuthenticatedIdentity(): Promise<RedditAccountIdentity | null>;
|
||||
}
|
||||
|
||||
interface ChatSessionTarget {
|
||||
@@ -28,11 +32,6 @@ export interface VerifiedRedditSessionStore {
|
||||
}): void;
|
||||
}
|
||||
|
||||
function matrixRedditAccountId(matrixUserId: string): string | null {
|
||||
const match = matrixUserId.match(/^@t2_([^:]+):reddit\.com$/i);
|
||||
return match?.[1]?.toLowerCase() ?? null;
|
||||
}
|
||||
|
||||
export class RedditSessionImporter {
|
||||
constructor(
|
||||
private readonly browser: BrowserSessionTarget,
|
||||
@@ -63,31 +62,37 @@ export class RedditSessionImporter {
|
||||
}
|
||||
|
||||
await this.browser.importRedditCookies(cookies);
|
||||
if (!(await this.browser.isSessionValid())) {
|
||||
throw new Error('Imported cookies did not authenticate with Reddit');
|
||||
}
|
||||
const identity = this.browser.getAuthenticatedIdentity
|
||||
? await this.browser.getAuthenticatedIdentity()
|
||||
: null;
|
||||
const redditUsername = identity?.username ?? await this.browser.getAuthenticatedUsername();
|
||||
if (!redditUsername) throw new Error('Reddit did not return an authenticated account identity');
|
||||
if (identity) {
|
||||
const matrixAccountId = matrixRedditAccountId(userId);
|
||||
if (!matrixAccountId || matrixAccountId !== identity.accountId.toLowerCase().replace(/^t2_/, '')) {
|
||||
throw new Error('Reddit cookies and Matrix credentials belong to different accounts');
|
||||
let redditIdentity: RedditAccountIdentity;
|
||||
try {
|
||||
if (!(await this.browser.isSessionValid())) {
|
||||
throw new Error('Imported cookies did not authenticate with Reddit');
|
||||
}
|
||||
const identity = await this.browser.getAuthenticatedIdentity();
|
||||
if (!identity) throw new Error('Reddit did not return an authenticated account identity');
|
||||
redditIdentity = identity;
|
||||
assertMatchingAccountIdentity({
|
||||
accountId: identity.accountId,
|
||||
redditUsername: identity.username,
|
||||
matrixUserId: userId,
|
||||
});
|
||||
} catch (error) {
|
||||
this.accountAuth.invalidate(error instanceof Error ? error.message : String(error));
|
||||
throw error;
|
||||
}
|
||||
|
||||
this.saveCredentials(credentials);
|
||||
this.chat.setMatrixClient(matrixClient);
|
||||
const account = {
|
||||
accountId: userId,
|
||||
redditUsername,
|
||||
redditUsername: redditIdentity.username,
|
||||
matrixUserId: userId,
|
||||
};
|
||||
this.accountAuth.markAuthenticated({
|
||||
accountId: normalizeRedditAccountId(redditIdentity.accountId),
|
||||
redditUsername: redditIdentity.username,
|
||||
matrixUserId: userId,
|
||||
});
|
||||
this.sessionStore?.save({ ...account, cookies, credentials });
|
||||
await this.chat.switchAccount?.(account.accountId);
|
||||
this.accountAuth.markAuthenticated(account);
|
||||
return account;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
export type JsonPrimitive = string | number | boolean | null;
|
||||
export type JsonSafeValue =
|
||||
| JsonPrimitive
|
||||
| JsonSafeValue[]
|
||||
| { [key: string]: JsonSafeValue };
|
||||
|
||||
const REDACTED = '[Redacted]';
|
||||
const CIRCULAR = '[Circular]';
|
||||
const MAX_DEPTH = 6;
|
||||
const MAX_COLLECTION_ENTRIES = 100;
|
||||
|
||||
function isSensitiveKey(key: string): boolean {
|
||||
return /^(?:authorization|cookie|set-cookie|password|passwd|secret|token|access[_-]?token|refresh[_-]?token|api[_-]?key)$/i.test(key);
|
||||
}
|
||||
|
||||
function constructorLabel(value: object): string {
|
||||
const constructorName = value.constructor?.name;
|
||||
return constructorName ? `[${constructorName}]` : '[Object]';
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert arbitrary diagnostic metadata into a bounded, JSON-safe value.
|
||||
* Errors intentionally expose only useful diagnostics; request/response and
|
||||
* socket internals can be circular and may contain credentials.
|
||||
*/
|
||||
export function toJsonSafe(
|
||||
value: unknown,
|
||||
depth = 0,
|
||||
ancestors = new WeakSet<object>(),
|
||||
): JsonSafeValue | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
if (value === null || typeof value === 'string' || typeof value === 'boolean') return value;
|
||||
if (typeof value === 'number') return Number.isFinite(value) ? value : String(value);
|
||||
if (typeof value === 'bigint') return value.toString();
|
||||
if (typeof value === 'symbol') return value.description ? `[Symbol: ${value.description}]` : '[Symbol]';
|
||||
if (typeof value === 'function') return `[Function: ${value.name || 'anonymous'}]`;
|
||||
|
||||
if (value instanceof Date) {
|
||||
return Number.isNaN(value.getTime()) ? '[Invalid Date]' : value.toISOString();
|
||||
}
|
||||
if (value instanceof Error) {
|
||||
const result: { [key: string]: JsonSafeValue } = {
|
||||
name: value.name,
|
||||
message: value.message,
|
||||
};
|
||||
if (value.stack) result.stack = value.stack;
|
||||
const code = Reflect.get(value, 'code');
|
||||
if (typeof code === 'string' || typeof code === 'number') result.code = code;
|
||||
if (value.cause !== undefined && depth < MAX_DEPTH) {
|
||||
const cause = toJsonSafe(value.cause, depth + 1, ancestors);
|
||||
if (cause !== undefined) result.cause = cause;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
if (depth >= MAX_DEPTH) return constructorLabel(value);
|
||||
if (ancestors.has(value)) return CIRCULAR;
|
||||
|
||||
ancestors.add(value);
|
||||
try {
|
||||
if (Array.isArray(value)) {
|
||||
const items = value
|
||||
.slice(0, MAX_COLLECTION_ENTRIES)
|
||||
.map((item) => toJsonSafe(item, depth + 1, ancestors) ?? null);
|
||||
if (value.length > MAX_COLLECTION_ENTRIES) items.push(`[Truncated ${value.length - MAX_COLLECTION_ENTRIES} items]`);
|
||||
return items;
|
||||
}
|
||||
|
||||
const prototype = Object.getPrototypeOf(value);
|
||||
if (prototype !== Object.prototype && prototype !== null) {
|
||||
return constructorLabel(value);
|
||||
}
|
||||
|
||||
const result: { [key: string]: JsonSafeValue } = {};
|
||||
const keys = Object.keys(value).slice(0, MAX_COLLECTION_ENTRIES);
|
||||
for (const key of keys) {
|
||||
if (isSensitiveKey(key)) {
|
||||
result[key] = REDACTED;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const child = toJsonSafe(Reflect.get(value, key), depth + 1, ancestors);
|
||||
if (child !== undefined) result[key] = child;
|
||||
} catch {
|
||||
result[key] = '[Unreadable]';
|
||||
}
|
||||
}
|
||||
const omitted = Object.keys(value).length - keys.length;
|
||||
if (omitted > 0) result.__truncated__ = `${omitted} properties omitted`;
|
||||
return result;
|
||||
} finally {
|
||||
ancestors.delete(value);
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
RealtimeEventHub,
|
||||
} from './events/index.js';
|
||||
import { createExpressSseHandler } from './events/express-sse-adapter.js';
|
||||
import { toJsonSafe } from '../utils/json-safe.js';
|
||||
|
||||
const MAX_LOGS = 500;
|
||||
const INITIAL_LOGS_ON_CONNECT = 50;
|
||||
@@ -28,14 +29,15 @@ export class LogBroadcaster {
|
||||
}
|
||||
|
||||
push(level: LogEntry['level'], message: string, data?: unknown): void {
|
||||
const safeData = toJsonSafe(data);
|
||||
const realtimeEvent = this.eventHub.closed
|
||||
? undefined
|
||||
: this.eventHub.publish('log.created', { level, message, data });
|
||||
: this.eventHub.publish('log.created', { level, message, data: safeData });
|
||||
const entry: LogEntry = {
|
||||
timestamp: realtimeEvent ? Date.parse(realtimeEvent.occurredAt) : Date.now(),
|
||||
level,
|
||||
message,
|
||||
data,
|
||||
data: safeData,
|
||||
};
|
||||
this.entries.push(entry);
|
||||
if (this.entries.length > MAX_LOGS) {
|
||||
|
||||
@@ -343,11 +343,18 @@ export function createConversationsRouter(ctx: WebContext): Router {
|
||||
ctx.activity.push('sent', `Sent chat reply to u/${conversation.username}`, conversation.username);
|
||||
}
|
||||
ctx.broadcaster.push('info', `Chat reply sent to u/${conversation.username}`);
|
||||
try {
|
||||
// Wait for Matrix /sync to merge the sent event into the cached
|
||||
// message history. The dashboard invalidates its queries when this
|
||||
// request completes, so returning earlier can immediately re-read
|
||||
// the stale pre-send conversation.
|
||||
await chatService.refreshTopChats(5);
|
||||
} catch (err) {
|
||||
// Delivery is already confirmed by Matrix's send response. A sync
|
||||
// failure must not turn a successful send into a reported failure.
|
||||
ctx.logger.error('Failed to refresh chats after send-reply:', err);
|
||||
}
|
||||
res.json({ success: true });
|
||||
|
||||
chatService.refreshTopChats(5).catch((err) =>
|
||||
ctx.logger.error('Failed to refresh chats after send-reply:', err),
|
||||
);
|
||||
} else {
|
||||
res.status(500).json({ error: 'Failed to send chat message' });
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user