390 lines
13 KiB
TypeScript
390 lines
13 KiB
TypeScript
import axios, { AxiosInstance } from 'axios';
|
|
import type { BrowserContext } from 'playwright';
|
|
import type { Config } from '../config/index.js';
|
|
import type { Logger } from '../utils/logger.js';
|
|
|
|
export class RedditRateLimitError extends Error {
|
|
constructor(public readonly retryAfterMs: number) {
|
|
const retryAfterSeconds = Math.ceil(retryAfterMs / 1000);
|
|
super(`Reddit API rate limit reached; retry after ${retryAfterSeconds}s`);
|
|
this.name = 'RedditRateLimitError';
|
|
}
|
|
}
|
|
|
|
function getRateLimitDelayMs(headers: unknown): number {
|
|
if (!headers || typeof headers !== 'object') return 60_000;
|
|
|
|
const values = headers as Record<string, unknown>;
|
|
const rawDelay = values['retry-after'] ?? values['x-ratelimit-reset'];
|
|
const seconds = typeof rawDelay === 'number' ? rawDelay : Number.parseFloat(String(rawDelay));
|
|
|
|
return Number.isFinite(seconds) && seconds >= 0
|
|
? Math.ceil(seconds * 1000)
|
|
: 60_000;
|
|
}
|
|
|
|
export interface RedditPost {
|
|
id: string;
|
|
name: string; // fullname like "t3_xxxxx"
|
|
subreddit: string;
|
|
author: string;
|
|
author_fullname?: string; // account fullname like "t2_xxxxx"
|
|
title: string;
|
|
url: string;
|
|
permalink: string;
|
|
thumbnail: string;
|
|
is_self: boolean;
|
|
selftext?: string;
|
|
post_hint?: string;
|
|
preview?: {
|
|
images: Array<{
|
|
source: { url: string; width: number; height: number };
|
|
resolutions: Array<{ url: string; width: number; height: number }>;
|
|
}>;
|
|
};
|
|
created_utc: number;
|
|
over_18: boolean;
|
|
spoiler: boolean;
|
|
link_flair_text?: string | null;
|
|
}
|
|
|
|
interface RedditListingChild {
|
|
data?: {
|
|
title?: unknown;
|
|
selftext?: unknown;
|
|
body?: unknown;
|
|
url?: unknown;
|
|
subreddit?: unknown;
|
|
};
|
|
}
|
|
|
|
function joinText(values: unknown[]): string {
|
|
return values.filter((value): value is string => typeof value === 'string').join(' ');
|
|
}
|
|
|
|
/** Match configured seller signals in user-controlled text. */
|
|
export function findSellerSignal(
|
|
text: string,
|
|
keywords: string[],
|
|
domains: string[],
|
|
): { kind: 'keyword' | 'domain'; value: string } | null {
|
|
const normalized = text.toLocaleLowerCase();
|
|
for (const keyword of keywords) {
|
|
const candidate = keyword.trim().toLocaleLowerCase();
|
|
if (candidate && normalized.includes(candidate)) {
|
|
return { kind: 'keyword', value: keyword };
|
|
}
|
|
}
|
|
|
|
const domain = findSellerDomain(text, domains);
|
|
return domain ? { kind: 'domain', value: domain } : null;
|
|
}
|
|
|
|
// Extracts URL-ish tokens (with or without scheme) and returns the first
|
|
// domain match. Matches `host` or any subdomain (`*.host`), case-insensitive.
|
|
export function findSellerDomain(text: string, domains: string[]): string | null {
|
|
if (!text || domains.length === 0) return null;
|
|
const normalized = text.toLowerCase();
|
|
const urlRegex = /(?:https?:\/\/|(?<![\w.@])(?=[a-z0-9-]+\.[a-z]))([a-z0-9.-]+\.[a-z]{2,})(?:[/?#][^\s)]*)?/gi;
|
|
const hosts: string[] = [];
|
|
for (const m of normalized.matchAll(urlRegex)) {
|
|
hosts.push(m[1].replace(/\.+$/, ''));
|
|
}
|
|
for (const host of hosts) {
|
|
for (const domain of domains) {
|
|
const d = domain.toLowerCase();
|
|
if (host === d || host.endsWith('.' + d)) return domain;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export class RedditApiClient {
|
|
private client: AxiosInstance;
|
|
private browserContext: BrowserContext | null = null;
|
|
|
|
constructor(
|
|
config: Config,
|
|
private logger: Logger
|
|
) {
|
|
this.client = axios.create({
|
|
baseURL: 'https://www.reddit.com',
|
|
headers: {
|
|
'User-Agent': config.reddit.userAgent,
|
|
Accept: 'application/json',
|
|
},
|
|
timeout: 30000,
|
|
});
|
|
}
|
|
|
|
setBrowserContext(context: BrowserContext): void {
|
|
this.browserContext = context;
|
|
this.logger.debug('Browser context set for authenticated API requests');
|
|
}
|
|
|
|
private async getCookieHeader(): Promise<string> {
|
|
if (!this.browserContext) {
|
|
return '';
|
|
}
|
|
|
|
try {
|
|
const cookies = await this.browserContext.cookies('https://www.reddit.com');
|
|
return cookies.map((c) => `${c.name}=${c.value}`).join('; ');
|
|
} catch (error) {
|
|
this.logger.warn('Failed to get cookies from browser context:', error);
|
|
return '';
|
|
}
|
|
}
|
|
|
|
/** Resolve a Reddit username through the authenticated browser session. */
|
|
async resolveUsernameToT2(username: string): Promise<string | null> {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) headers.Cookie = cookieHeader;
|
|
|
|
try {
|
|
const response = await this.client.get(
|
|
`/user/${encodeURIComponent(username)}/about.json`,
|
|
{ params: { raw_json: 1 }, headers },
|
|
);
|
|
const id = response.data?.data?.id;
|
|
if (typeof id !== 'string' || id.length === 0) return null;
|
|
return id.startsWith('t2_') ? id : `t2_${id}`;
|
|
} catch (error) {
|
|
if (axios.isAxiosError(error) && error.response?.status === 404) return null;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async getNewPosts(subreddit: string, limit = 25, type: 'image' | 'chat' = 'image'): Promise<RedditPost[]> {
|
|
try {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) {
|
|
headers['Cookie'] = cookieHeader;
|
|
}
|
|
|
|
const response = await this.client.get(`/r/${subreddit}/new.json`, {
|
|
params: { limit, raw_json: 1 },
|
|
headers,
|
|
});
|
|
|
|
const posts: RedditPost[] = response.data.data.children
|
|
.map((child: { data: RedditPost }) => child.data)
|
|
.filter((post: RedditPost) => this.isValidPost(post, type));
|
|
|
|
this.logger.debug(`Fetched ${posts.length} valid ${type} posts from r/${subreddit}`);
|
|
return posts;
|
|
} catch (error) {
|
|
if (axios.isAxiosError(error)) {
|
|
if (error.response?.status === 404) {
|
|
this.logger.warn(`Subreddit r/${subreddit} not found`);
|
|
return [];
|
|
}
|
|
if (error.response?.status === 403) {
|
|
this.logger.warn(`Subreddit r/${subreddit} is private or quarantined`);
|
|
return [];
|
|
}
|
|
if (error.response?.status === 429) {
|
|
const retryAfterMs = getRateLimitDelayMs(error.response.headers);
|
|
const rateLimitError = new RedditRateLimitError(retryAfterMs);
|
|
this.logger.warn(rateLimitError.message);
|
|
throw rateLimitError;
|
|
}
|
|
}
|
|
this.logger.error(`Error fetching posts from r/${subreddit}:`, error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
private isValidPost(post: RedditPost, type: 'image' | 'chat' = 'image'): boolean {
|
|
// Skip deleted/removed authors
|
|
if (post.author === '[deleted]' || post.author === 'AutoModerator') return false;
|
|
|
|
if (type === 'chat') {
|
|
// Chat subs are text-driven. Require a title (always present) and
|
|
// ideally some self text — but allow title-only since some tags live
|
|
// in the title alone (e.g. "[25F] up late, dm me").
|
|
return true;
|
|
}
|
|
|
|
// image type:
|
|
if (post.is_self) return false;
|
|
return this.hasImage(post);
|
|
}
|
|
|
|
private hasImage(post: RedditPost): boolean {
|
|
// Check post_hint
|
|
if (post.post_hint === 'image') return true;
|
|
|
|
// Check for preview images
|
|
if (post.preview?.images?.length) return true;
|
|
|
|
// Check URL patterns for direct image links
|
|
const url = post.url.toLowerCase();
|
|
const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp'];
|
|
|
|
if (imageExtensions.some((ext) => url.includes(ext))) return true;
|
|
|
|
// Check for known image hosts
|
|
const imageHosts = ['i.redd.it', 'i.imgur.com', 'imgur.com'];
|
|
if (imageHosts.some((host) => url.includes(host))) return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
getImageUrl(post: RedditPost): string | null {
|
|
// Prefer high-quality preview images (already processed by Reddit)
|
|
if (post.preview?.images?.[0]?.source?.url) {
|
|
// Reddit encodes URLs in preview, need to decode HTML entities
|
|
return this.decodeRedditUrl(post.preview.images[0].source.url);
|
|
}
|
|
|
|
// Direct i.redd.it links
|
|
if (post.url.includes('i.redd.it')) {
|
|
return post.url;
|
|
}
|
|
|
|
// Direct imgur links
|
|
if (post.url.includes('i.imgur.com')) {
|
|
return post.url;
|
|
}
|
|
|
|
// Imgur album/page links - try to get direct image
|
|
if (post.url.includes('imgur.com') && !post.url.includes('i.imgur.com')) {
|
|
// Try adding .jpg extension
|
|
const imgurId = post.url.split('/').pop()?.split('.')[0];
|
|
if (imgurId) {
|
|
return `https://i.imgur.com/${imgurId}.jpg`;
|
|
}
|
|
}
|
|
|
|
// Check if URL has image extension
|
|
const url = post.url.toLowerCase();
|
|
const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp'];
|
|
if (imageExtensions.some((ext) => url.includes(ext))) {
|
|
return post.url;
|
|
}
|
|
|
|
// Use thumbnail as last resort if it's a valid URL
|
|
if (post.thumbnail && post.thumbnail.startsWith('http')) {
|
|
return post.thumbnail;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private decodeRedditUrl(url: string): string {
|
|
return url.replace(/&/g, '&');
|
|
}
|
|
|
|
async isSellerProfile(
|
|
username: string,
|
|
sellerKeywords: string[],
|
|
sellerDomains: string[] = [],
|
|
currentPost?: RedditPost,
|
|
): Promise<boolean> {
|
|
if (sellerKeywords.length === 0 && sellerDomains.length === 0) return false;
|
|
|
|
// Check the post first. This catches direct promotional posts without an
|
|
// extra request and also covers accounts whose profile bio is deliberately
|
|
// vague.
|
|
if (currentPost) {
|
|
const postSignal = findSellerSignal(
|
|
joinText([currentPost.title, currentPost.selftext, currentPost.url]),
|
|
sellerKeywords,
|
|
sellerDomains,
|
|
);
|
|
if (postSignal) {
|
|
this.logger.info(
|
|
`u/${username} flagged as seller from current post (matched ${postSignal.kind}: "${postSignal.value}")`,
|
|
);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
try {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) {
|
|
headers['Cookie'] = cookieHeader;
|
|
}
|
|
|
|
const response = await this.client.get(`/user/${encodeURIComponent(username)}/about.json`, {
|
|
params: { raw_json: 1 },
|
|
headers,
|
|
});
|
|
|
|
const userData = response.data?.data;
|
|
if (userData) {
|
|
const profileText = joinText([
|
|
userData.subreddit?.public_description || '',
|
|
userData.subreddit?.title || '',
|
|
userData.subreddit?.display_name_prefixed || '',
|
|
]);
|
|
const profileSignal = findSellerSignal(profileText, sellerKeywords, sellerDomains);
|
|
if (profileSignal) {
|
|
this.logger.info(
|
|
`u/${username} flagged as seller profile (matched ${profileSignal.kind}: "${profileSignal.value}")`,
|
|
);
|
|
return true;
|
|
}
|
|
}
|
|
} catch (error) {
|
|
if (axios.isAxiosError(error) && error.response?.status === 429) {
|
|
const rateLimitError = new RedditRateLimitError(getRateLimitDelayMs(error.response.headers));
|
|
this.logger.warn(rateLimitError.message);
|
|
throw rateLimitError;
|
|
}
|
|
if (axios.isAxiosError(error) && error.response?.status === 404) {
|
|
this.logger.warn(`User u/${username} profile not found`);
|
|
return false;
|
|
}
|
|
this.logger.warn(`Failed to check profile for u/${username}:`, error);
|
|
}
|
|
|
|
// Bios are often clean while recent posts advertise the paid account.
|
|
// Scan a small overview page, including comment bodies and subreddit
|
|
// names, before allowing the candidate through.
|
|
try {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) headers.Cookie = cookieHeader;
|
|
const response = await this.client.get(`/user/${encodeURIComponent(username)}/overview.json`, {
|
|
params: { limit: 25, raw_json: 1 },
|
|
headers,
|
|
});
|
|
const children: RedditListingChild[] = Array.isArray(response.data?.data?.children)
|
|
? response.data.data.children
|
|
: [];
|
|
const activityText = children.map(({ data }) => joinText([
|
|
data?.title,
|
|
data?.selftext,
|
|
data?.body,
|
|
data?.url,
|
|
data?.subreddit,
|
|
])).join(' ');
|
|
const activitySignal = findSellerSignal(activityText, sellerKeywords, sellerDomains);
|
|
if (activitySignal) {
|
|
this.logger.info(
|
|
`u/${username} flagged as seller from recent activity (matched ${activitySignal.kind}: "${activitySignal.value}")`,
|
|
);
|
|
return true;
|
|
}
|
|
} catch (error) {
|
|
if (axios.isAxiosError(error) && error.response?.status === 429) {
|
|
const rateLimitError = new RedditRateLimitError(getRateLimitDelayMs(error.response.headers));
|
|
this.logger.warn(rateLimitError.message);
|
|
throw rateLimitError;
|
|
}
|
|
this.logger.warn(`Failed to check recent activity for u/${username}:`, error);
|
|
}
|
|
|
|
return false; // Fail open if Reddit profile endpoints are unavailable.
|
|
}
|
|
|
|
getPostUrl(post: RedditPost): string {
|
|
return `https://www.reddit.com${post.permalink}`;
|
|
}
|
|
}
|