795 lines
29 KiB
TypeScript
795 lines
29 KiB
TypeScript
import axios, { AxiosInstance } from 'axios';
|
|
import type { BrowserContext } from 'playwright';
|
|
import { z } from 'zod';
|
|
import type { Config } from '../config/index.js';
|
|
import type { Logger } from '../utils/logger.js';
|
|
import type { RedditRequestScheduler } from './request-coordinator.js';
|
|
|
|
const uncoordinatedRequestScheduler: RedditRequestScheduler = {
|
|
schedule: async <T>(request: () => Promise<T>) => await request(),
|
|
extendCooldown: () => undefined,
|
|
};
|
|
|
|
const RedditUserAboutResponseSchema = z.object({
|
|
data: z.object({
|
|
is_suspended: z.boolean().optional(),
|
|
}),
|
|
});
|
|
|
|
/**
|
|
* The subset of `/user/<name>/about.json` that says how established an account
|
|
* is. `has_verified_email` is only returned for the requesting user's own
|
|
* profile, so it stays nullable for anyone else.
|
|
*/
|
|
const RedditAccountProfileResponseSchema = z.object({
|
|
data: z.object({
|
|
created_utc: z.number().optional(),
|
|
link_karma: z.number().optional(),
|
|
comment_karma: z.number().optional(),
|
|
total_karma: z.number().optional(),
|
|
has_verified_email: z.boolean().nullish(),
|
|
is_suspended: z.boolean().optional(),
|
|
}),
|
|
});
|
|
|
|
export type RedditUserAccountStatus = 'active' | 'suspended' | 'deleted';
|
|
|
|
export interface RedditAccountProfile {
|
|
/** Account creation time in epoch milliseconds. */
|
|
createdAt: number | null;
|
|
linkKarma: number | null;
|
|
commentKarma: number | null;
|
|
totalKarma: number | null;
|
|
hasVerifiedEmail: boolean | null;
|
|
isSuspended: boolean;
|
|
}
|
|
|
|
export class RedditRateLimitError extends Error {
|
|
constructor(public readonly retryAfterMs: number) {
|
|
const retryAfterSeconds = Math.ceil(retryAfterMs / 1000);
|
|
super(`Reddit API rate limit reached; retry after ${retryAfterSeconds}s`);
|
|
this.name = 'RedditRateLimitError';
|
|
}
|
|
}
|
|
|
|
export class RedditAccessDeniedError extends Error {
|
|
constructor(public readonly subreddit: string) {
|
|
super(`Reddit denied listing access for r/${subreddit} (HTTP 403)`);
|
|
this.name = 'RedditAccessDeniedError';
|
|
}
|
|
}
|
|
|
|
/** Reddit confirms the community listing is gone and must no longer be polled. */
|
|
export class RedditSubredditUnavailableError extends Error {
|
|
constructor(public readonly subreddit: string) {
|
|
super(`Reddit reports that r/${subreddit} is unavailable (HTTP 404)`);
|
|
this.name = 'RedditSubredditUnavailableError';
|
|
}
|
|
}
|
|
|
|
function getRateLimitDelayMs(headers: unknown): number {
|
|
if (!headers || typeof headers !== 'object') return 60_000;
|
|
|
|
const values = headers as Record<string, unknown>;
|
|
const rawDelay = values['retry-after'] ?? values['x-ratelimit-reset'];
|
|
const seconds = typeof rawDelay === 'number' ? rawDelay : Number.parseFloat(String(rawDelay));
|
|
|
|
return Number.isFinite(seconds) && seconds >= 0
|
|
? Math.ceil(seconds * 1000)
|
|
: 60_000;
|
|
}
|
|
|
|
export interface RedditPost {
|
|
id: string;
|
|
name: string; // fullname like "t3_xxxxx"
|
|
subreddit: string;
|
|
author: string;
|
|
author_fullname?: string; // account fullname like "t2_xxxxx"
|
|
title: string;
|
|
url: string;
|
|
permalink: string;
|
|
thumbnail: string;
|
|
is_self: boolean;
|
|
selftext?: string;
|
|
post_hint?: string;
|
|
preview?: {
|
|
images: Array<{
|
|
source: { url: string; width: number; height: number };
|
|
resolutions: Array<{ url: string; width: number; height: number }>;
|
|
}>;
|
|
};
|
|
is_gallery?: boolean;
|
|
gallery_data?: {
|
|
items: Array<{ media_id: string; id?: number }>;
|
|
};
|
|
media_metadata?: Record<string, {
|
|
status?: string;
|
|
m?: string;
|
|
s?: {
|
|
u?: string;
|
|
gif?: string;
|
|
x?: number;
|
|
y?: number;
|
|
};
|
|
}>;
|
|
created_utc: number;
|
|
over_18: boolean;
|
|
spoiler: boolean;
|
|
link_flair_text?: string | null;
|
|
// Listing fields the account grower needs to judge whether a thread can and
|
|
// should be commented on. Optional because the outreach paths never read them.
|
|
num_comments?: number;
|
|
locked?: boolean;
|
|
stickied?: boolean;
|
|
archived?: boolean;
|
|
pinned?: boolean;
|
|
distinguished?: string | null;
|
|
removed_by_category?: string | null;
|
|
score?: number;
|
|
/** Present when outreach targets a commenter while retaining the source post media. */
|
|
targetComment?: {
|
|
body: string;
|
|
permalink: string;
|
|
sourcePostId: string;
|
|
sourcePostAuthor: string;
|
|
expressedGender: 'male' | 'female' | 'other' | 'unknown';
|
|
confidence: number;
|
|
};
|
|
}
|
|
|
|
/** Typed Reddit content used only by the simulation-only browsing reader. */
|
|
export interface BrowsingThing {
|
|
fullname: string;
|
|
kind: 'post' | 'comment';
|
|
subreddit: string;
|
|
author: string;
|
|
permalink: string;
|
|
createdAt: number;
|
|
score: number | null;
|
|
text: string;
|
|
nsfw: boolean;
|
|
parentPostId: string;
|
|
distinguished: string | null;
|
|
removed: boolean;
|
|
}
|
|
|
|
interface RedditListingChild {
|
|
data?: {
|
|
title?: unknown;
|
|
selftext?: unknown;
|
|
body?: unknown;
|
|
url?: unknown;
|
|
subreddit?: unknown;
|
|
};
|
|
}
|
|
|
|
function joinText(values: unknown[]): string {
|
|
return values.filter((value): value is string => typeof value === 'string').join(' ');
|
|
}
|
|
|
|
function recordValue(value: unknown, key: string): unknown {
|
|
return value && typeof value === 'object' ? (value as Record<string, unknown>)[key] : undefined;
|
|
}
|
|
|
|
/** Flatten Reddit's structured profile-link payloads across legacy/current field naming. */
|
|
function socialLinkText(userData: unknown): string {
|
|
const subreddit = recordValue(userData, 'subreddit');
|
|
const candidates = [
|
|
recordValue(userData, 'social_links'),
|
|
recordValue(userData, 'socialLinks'),
|
|
recordValue(subreddit, 'social_links'),
|
|
recordValue(subreddit, 'socialLinks'),
|
|
];
|
|
const values: unknown[] = [];
|
|
for (const candidate of candidates) {
|
|
if (!Array.isArray(candidate)) continue;
|
|
for (const link of candidate) {
|
|
values.push(
|
|
recordValue(link, 'title'),
|
|
recordValue(link, 'handle'),
|
|
recordValue(link, 'type'),
|
|
recordValue(link, 'url'),
|
|
recordValue(link, 'outbound_url'),
|
|
recordValue(link, 'outboundUrl'),
|
|
);
|
|
}
|
|
}
|
|
return joinText(values);
|
|
}
|
|
|
|
/**
|
|
* Catch a narrow paid-content funnel that deliberately keeps the platform name
|
|
* out of the bio. Requiring an NSFW profile, an explicit link CTA, and a
|
|
* promise of more/exclusive personal content avoids treating ordinary links as
|
|
* seller evidence.
|
|
*/
|
|
function findProfileFunnel(text: string, isNsfw: boolean): string | null {
|
|
if (!isNsfw) return null;
|
|
const normalized = text.toLocaleLowerCase().replace(/[\u2018\u2019]/g, "'");
|
|
const linkCallToAction = /\b(?:hit|check|tap|click|visit|follow)\s+(?:the|my)\s+link\b/;
|
|
const contentPromise = /\b(?:see|get|show(?:ing)?|watch)\s+(?:a\s+)?(?:lot\s+)?more(?:\s+of\s+me)?\b|\bexclusive\s+(?:content|pics?|photos?|videos?)\b/;
|
|
return linkCallToAction.test(normalized) && contentPromise.test(normalized)
|
|
? 'NSFW profile links to more or exclusive personal content'
|
|
: null;
|
|
}
|
|
|
|
/** Match configured seller signals in user-controlled text. */
|
|
export function findSellerSignal(
|
|
text: string,
|
|
keywords: string[],
|
|
domains: string[],
|
|
): { kind: 'keyword' | 'domain'; value: string } | null {
|
|
const normalized = text.toLocaleLowerCase();
|
|
for (const keyword of keywords) {
|
|
const candidate = keyword.trim().toLocaleLowerCase();
|
|
if (candidate && normalized.includes(candidate)) {
|
|
return { kind: 'keyword', value: keyword };
|
|
}
|
|
}
|
|
|
|
const domain = findSellerDomain(text, domains);
|
|
return domain ? { kind: 'domain', value: domain } : null;
|
|
}
|
|
|
|
// Extracts URL-ish tokens (with or without scheme) and returns the first
|
|
// domain match. Matches `host` or any subdomain (`*.host`), case-insensitive.
|
|
export function findSellerDomain(text: string, domains: string[]): string | null {
|
|
if (!text || domains.length === 0) return null;
|
|
const normalized = text.toLowerCase();
|
|
const urlRegex = /(?:https?:\/\/|(?<![\w.@])(?=[a-z0-9-]+\.[a-z]))([a-z0-9.-]+\.[a-z]{2,})(?:[/?#][^\s)]*)?/gi;
|
|
const hosts: string[] = [];
|
|
for (const m of normalized.matchAll(urlRegex)) {
|
|
hosts.push(m[1].replace(/\.+$/, ''));
|
|
}
|
|
for (const host of hosts) {
|
|
for (const domain of domains) {
|
|
const d = domain.toLowerCase();
|
|
if (host === d || host.endsWith('.' + d)) return domain;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export class RedditApiClient {
|
|
private client: AxiosInstance;
|
|
private browserContext: BrowserContext | null = null;
|
|
private consecutiveListing403s = 0;
|
|
|
|
constructor(
|
|
config: Config,
|
|
private logger: Logger,
|
|
// Production bootstraps supply one process-wide paced coordinator. Keep
|
|
// standalone clients independent for embedders and focused unit tests.
|
|
private readonly requestScheduler: RedditRequestScheduler = uncoordinatedRequestScheduler,
|
|
) {
|
|
this.client = axios.create({
|
|
baseURL: 'https://www.reddit.com',
|
|
headers: {
|
|
'User-Agent': config.reddit.userAgent,
|
|
Accept: 'application/json',
|
|
},
|
|
timeout: 30000,
|
|
});
|
|
}
|
|
|
|
setBrowserContext(context: BrowserContext): void {
|
|
this.browserContext = context;
|
|
this.logger.debug('Browser context set for authenticated API requests');
|
|
}
|
|
|
|
private async getCookieHeader(): Promise<string> {
|
|
if (!this.browserContext) {
|
|
return '';
|
|
}
|
|
|
|
try {
|
|
const cookies = await this.browserContext.cookies('https://www.reddit.com');
|
|
return cookies.map((c) => `${c.name}=${c.value}`).join('; ');
|
|
} catch (error) {
|
|
this.logger.warn('Failed to get cookies from browser context:', error);
|
|
return '';
|
|
}
|
|
}
|
|
|
|
/** Resolve a Reddit username through the authenticated browser session. */
|
|
async resolveUsernameToT2(username: string): Promise<string | null> {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) headers.Cookie = cookieHeader;
|
|
|
|
try {
|
|
const response = await this.scheduleRequest(() => this.client.get(
|
|
`/user/${encodeURIComponent(username)}/about.json`,
|
|
{ params: { raw_json: 1 }, headers },
|
|
));
|
|
const id = response.data?.data?.id;
|
|
if (typeof id !== 'string' || id.length === 0) return null;
|
|
return id.startsWith('t2_') ? id : `t2_${id}`;
|
|
} catch (error) {
|
|
if (axios.isAxiosError(error) && error.response?.status === 404) return null;
|
|
const rateLimitError = this.toRateLimitError(error);
|
|
if (rateLimitError) throw rateLimitError;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/** A 404 is Reddit's definitive signal that the username no longer has an account. */
|
|
async getUserAccountStatus(username: string): Promise<RedditUserAccountStatus> {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) headers.Cookie = cookieHeader;
|
|
|
|
try {
|
|
const response = await this.scheduleRequest(() => this.client.get(
|
|
`/user/${encodeURIComponent(username)}/about.json`,
|
|
{ params: { raw_json: 1 }, headers },
|
|
));
|
|
const parsed = RedditUserAboutResponseSchema.parse(response.data);
|
|
return parsed.data.is_suspended === true ? 'suspended' : 'active';
|
|
} catch (error) {
|
|
if (axios.isAxiosError(error) && error.response?.status === 404) return 'deleted';
|
|
const rateLimitError = this.toRateLimitError(error);
|
|
if (rateLimitError) throw rateLimitError;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Age, karma and email-verification for one account. Used to judge whether
|
|
* one of our own accounts is established enough for Reddit to let it start
|
|
* chats. Returns null when the profile no longer exists.
|
|
*/
|
|
async getAccountProfile(username: string): Promise<RedditAccountProfile | null> {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) headers.Cookie = cookieHeader;
|
|
|
|
try {
|
|
const response = await this.scheduleRequest(() => this.client.get(
|
|
`/user/${encodeURIComponent(username)}/about.json`,
|
|
{ params: { raw_json: 1 }, headers },
|
|
));
|
|
const { data } = RedditAccountProfileResponseSchema.parse(response.data);
|
|
const totalKarma = data.total_karma
|
|
?? (data.link_karma === undefined && data.comment_karma === undefined
|
|
? undefined
|
|
: (data.link_karma ?? 0) + (data.comment_karma ?? 0));
|
|
return {
|
|
createdAt: data.created_utc === undefined ? null : Math.round(data.created_utc * 1000),
|
|
linkKarma: data.link_karma ?? null,
|
|
commentKarma: data.comment_karma ?? null,
|
|
totalKarma: totalKarma ?? null,
|
|
hasVerifiedEmail: data.has_verified_email ?? null,
|
|
isSuspended: data.is_suspended === true,
|
|
};
|
|
} catch (error) {
|
|
if (axios.isAxiosError(error) && error.response?.status === 404) return null;
|
|
const rateLimitError = this.toRateLimitError(error);
|
|
if (rateLimitError) throw rateLimitError;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async getNewPosts(subreddit: string, limit = 25, type: 'image' | 'chat' = 'image'): Promise<RedditPost[]> {
|
|
try {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) {
|
|
headers['Cookie'] = cookieHeader;
|
|
}
|
|
|
|
const response = await this.scheduleRequest(() => this.client.get(`/r/${subreddit}/new.json`, {
|
|
params: { limit, raw_json: 1 },
|
|
headers,
|
|
}));
|
|
|
|
const posts: RedditPost[] = response.data.data.children
|
|
.map((child: { data: RedditPost }) => child.data)
|
|
.filter((post: RedditPost) => this.isValidPost(post, type));
|
|
|
|
this.logger.debug(`Fetched ${posts.length} valid ${type} posts from r/${subreddit}`);
|
|
this.consecutiveListing403s = 0;
|
|
return posts;
|
|
} catch (error) {
|
|
if (axios.isAxiosError(error)) {
|
|
if (error.response?.status === 404) {
|
|
throw new RedditSubredditUnavailableError(subreddit);
|
|
}
|
|
if (error.response?.status === 403) {
|
|
this.consecutiveListing403s += 1;
|
|
const denied = new RedditAccessDeniedError(subreddit);
|
|
|
|
// One 403 can genuinely be a private or quarantined subreddit. A
|
|
// second consecutive denial from unrelated listings is a strong
|
|
// signal that Reddit is temporarily rejecting this session/IP.
|
|
if (this.consecutiveListing403s >= 2) {
|
|
const cooldownMs = 120_000;
|
|
this.requestScheduler.extendCooldown(cooldownMs);
|
|
this.logger.warn(
|
|
`Reddit is temporarily denying listing requests (consecutive HTTP 403s); pausing for ${cooldownMs / 1000}s`,
|
|
);
|
|
throw new RedditRateLimitError(cooldownMs);
|
|
}
|
|
|
|
this.logger.warn(`${denied.message}; it may be private, quarantined, or a temporary Reddit block`);
|
|
return [];
|
|
}
|
|
const rateLimitError = this.toRateLimitError(error);
|
|
if (rateLimitError) throw rateLimitError;
|
|
}
|
|
this.logger.error(`Error fetching posts from r/${subreddit}:`, error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fetches a `hot` listing without the outreach validity filters.
|
|
*
|
|
* The account grower wants threads with traction rather than image or chat
|
|
* posts, and does its own screening, so this deliberately returns the listing
|
|
* close to raw — only authorless and bot-authored posts are dropped.
|
|
*/
|
|
async getHotPosts(subreddit: string, limit = 25): Promise<RedditPost[]> {
|
|
try {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) headers.Cookie = cookieHeader;
|
|
|
|
const response = await this.scheduleRequest(() => this.client.get(`/r/${subreddit}/hot.json`, {
|
|
params: { limit, raw_json: 1 },
|
|
headers,
|
|
}));
|
|
|
|
const posts: RedditPost[] = response.data.data.children
|
|
.map((child: { data: RedditPost }) => child.data)
|
|
.filter((post: RedditPost) => (
|
|
post.author !== '[deleted]' && post.author !== 'AutoModerator'
|
|
));
|
|
|
|
this.logger.debug(`Fetched ${posts.length} hot posts from r/${subreddit}`);
|
|
this.consecutiveListing403s = 0;
|
|
return posts;
|
|
} catch (error) {
|
|
if (axios.isAxiosError(error)) {
|
|
if (error.response?.status === 404) throw new RedditSubredditUnavailableError(subreddit);
|
|
if (error.response?.status === 403) {
|
|
this.consecutiveListing403s += 1;
|
|
if (this.consecutiveListing403s >= 2) {
|
|
const cooldownMs = 120_000;
|
|
this.requestScheduler.extendCooldown(cooldownMs);
|
|
this.logger.warn(
|
|
`Reddit is temporarily denying listing requests (consecutive HTTP 403s); pausing for ${cooldownMs / 1000}s`,
|
|
);
|
|
throw new RedditRateLimitError(cooldownMs);
|
|
}
|
|
throw new RedditAccessDeniedError(subreddit);
|
|
}
|
|
const rateLimitError = this.toRateLimitError(error);
|
|
if (rateLimitError) throw rateLimitError;
|
|
}
|
|
this.logger.error(`Error fetching hot posts from r/${subreddit}:`, error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Top-level comment bodies already on a thread, so a generated comment does
|
|
* not restate a point somebody else has made.
|
|
*/
|
|
async getThreadTopComments(subreddit: string, postId: string, limit = 8): Promise<string[]> {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) headers.Cookie = cookieHeader;
|
|
|
|
try {
|
|
const response = await this.scheduleRequest(() => this.client.get(
|
|
`/r/${subreddit}/comments/${postId}.json`,
|
|
{ params: { limit, depth: 1, sort: 'top', raw_json: 1 }, headers },
|
|
));
|
|
const listings = Array.isArray(response.data) ? response.data : [];
|
|
const commentListing = listings[1] as { data?: { children?: unknown } } | undefined;
|
|
const children = commentListing?.data?.children;
|
|
if (!Array.isArray(children)) return [];
|
|
return children
|
|
.map((child) => (child as { data?: { body?: unknown } })?.data?.body)
|
|
.filter((body): body is string => typeof body === 'string')
|
|
.filter((body) => body !== '[removed]' && body !== '[deleted]')
|
|
.slice(0, limit);
|
|
} catch (error) {
|
|
const rateLimitError = this.toRateLimitError(error);
|
|
if (rateLimitError) throw rateLimitError;
|
|
// Context is a nice-to-have; never fail a comment because it is missing.
|
|
this.logger.warn(`Failed to read existing comments for ${postId}:`, error);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Typed top-level comments for browsing simulation. This is intentionally
|
|
* separate from `getThreadTopComments()`, whose string-only contract belongs
|
|
* to comment generation.
|
|
*/
|
|
async getBrowsingComments(
|
|
subreddit: string,
|
|
postId: string,
|
|
limit = 25,
|
|
): Promise<BrowsingThing[]> {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) headers.Cookie = cookieHeader;
|
|
|
|
try {
|
|
const response = await this.scheduleRequest(() => this.client.get(
|
|
`/r/${subreddit}/comments/${postId}.json`,
|
|
{ params: { limit, depth: 1, sort: 'hot', raw_json: 1 }, headers },
|
|
));
|
|
const listings = Array.isArray(response.data) ? response.data : [];
|
|
const children = (listings[1] as { data?: { children?: unknown } } | undefined)?.data?.children;
|
|
if (!Array.isArray(children)) return [];
|
|
return children.flatMap((child): BrowsingThing[] => {
|
|
const data = (child as { data?: Record<string, unknown> })?.data;
|
|
if (!data || typeof data.name !== 'string' || !data.name.startsWith('t1_')) return [];
|
|
const author = typeof data.author === 'string' ? data.author : '[deleted]';
|
|
const body = typeof data.body === 'string' ? data.body : '';
|
|
return [{
|
|
fullname: data.name,
|
|
kind: 'comment',
|
|
subreddit: typeof data.subreddit === 'string' ? data.subreddit : subreddit,
|
|
author,
|
|
permalink: typeof data.permalink === 'string'
|
|
? data.permalink
|
|
: `/r/${subreddit}/comments/${postId}/_/${data.name.slice(3)}/`,
|
|
createdAt: typeof data.created_utc === 'number' ? Math.round(data.created_utc * 1_000) : 0,
|
|
score: typeof data.score === 'number' ? data.score : null,
|
|
text: body,
|
|
nsfw: data.over_18 === true,
|
|
parentPostId: postId,
|
|
distinguished: typeof data.distinguished === 'string' ? data.distinguished : null,
|
|
removed: body === '[removed]' || body === '[deleted]' || data.removed === true,
|
|
}];
|
|
}).slice(0, limit);
|
|
} catch (error) {
|
|
const rateLimitError = this.toRateLimitError(error);
|
|
if (rateLimitError) throw rateLimitError;
|
|
this.logger.warn(`Failed to read browsing comments for ${postId}:`, error);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
private isValidPost(post: RedditPost, type: 'image' | 'chat' = 'image'): boolean {
|
|
// Skip deleted/removed authors
|
|
if (post.author === '[deleted]' || post.author === 'AutoModerator') return false;
|
|
|
|
if (type === 'chat') {
|
|
// Chat subs are text-driven. Require a title (always present) and
|
|
// ideally some self text — but allow title-only since some tags live
|
|
// in the title alone (e.g. "[25F] up late, dm me").
|
|
return true;
|
|
}
|
|
|
|
// image type:
|
|
if (post.is_self) return false;
|
|
return this.hasImage(post);
|
|
}
|
|
|
|
private hasImage(post: RedditPost): boolean {
|
|
if (this.getGalleryImageUrl(post)) return true;
|
|
|
|
// Check post_hint
|
|
if (post.post_hint === 'image') return true;
|
|
|
|
// Check for preview images
|
|
if (post.preview?.images?.length) return true;
|
|
|
|
// Check URL patterns for direct image links
|
|
const url = post.url.toLowerCase();
|
|
const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp'];
|
|
|
|
if (imageExtensions.some((ext) => url.includes(ext))) return true;
|
|
|
|
// Check for known image hosts
|
|
const imageHosts = ['i.redd.it', 'i.imgur.com', 'imgur.com'];
|
|
if (imageHosts.some((host) => url.includes(host))) return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
getImageUrl(post: RedditPost): string | null {
|
|
const galleryImageUrl = this.getGalleryImageUrl(post);
|
|
if (galleryImageUrl) return galleryImageUrl;
|
|
|
|
// Prefer high-quality preview images (already processed by Reddit)
|
|
if (post.preview?.images?.[0]?.source?.url) {
|
|
// Reddit encodes URLs in preview, need to decode HTML entities
|
|
return this.decodeRedditUrl(post.preview.images[0].source.url);
|
|
}
|
|
|
|
// Direct i.redd.it links
|
|
if (post.url.includes('i.redd.it')) {
|
|
return post.url;
|
|
}
|
|
|
|
// Direct imgur links
|
|
if (post.url.includes('i.imgur.com')) {
|
|
return post.url;
|
|
}
|
|
|
|
// Imgur album/page links - try to get direct image
|
|
if (post.url.includes('imgur.com') && !post.url.includes('i.imgur.com')) {
|
|
// Try adding .jpg extension
|
|
const imgurId = post.url.split('/').pop()?.split('.')[0];
|
|
if (imgurId) {
|
|
return `https://i.imgur.com/${imgurId}.jpg`;
|
|
}
|
|
}
|
|
|
|
// Check if URL has image extension
|
|
const url = post.url.toLowerCase();
|
|
const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp'];
|
|
if (imageExtensions.some((ext) => url.includes(ext))) {
|
|
return post.url;
|
|
}
|
|
|
|
// Use thumbnail as last resort if it's a valid URL
|
|
if (post.thumbnail && post.thumbnail.startsWith('http')) {
|
|
return post.thumbnail;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/** Return the first valid gallery image in Reddit's declared display order. */
|
|
private getGalleryImageUrl(post: RedditPost): string | null {
|
|
if (!post.is_gallery || !post.media_metadata) return null;
|
|
|
|
const orderedMediaIds = post.gallery_data?.items.map((item) => item.media_id)
|
|
?? Object.keys(post.media_metadata);
|
|
for (const mediaId of orderedMediaIds) {
|
|
const media = post.media_metadata[mediaId];
|
|
if (!media || (media.status && media.status !== 'valid')) continue;
|
|
if (media.m && !media.m.startsWith('image/')) continue;
|
|
|
|
const sourceUrl = media.s?.u ?? media.s?.gif;
|
|
if (sourceUrl) return this.decodeRedditUrl(sourceUrl);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private decodeRedditUrl(url: string): string {
|
|
return url.replace(/&/g, '&');
|
|
}
|
|
|
|
async isSellerProfile(
|
|
username: string,
|
|
sellerKeywords: string[],
|
|
sellerDomains: string[] = [],
|
|
currentPost?: RedditPost,
|
|
): Promise<boolean> {
|
|
if (sellerKeywords.length === 0 && sellerDomains.length === 0) return false;
|
|
|
|
// Check the post first. This catches direct promotional posts without an
|
|
// extra request and also covers accounts whose profile bio is deliberately
|
|
// vague.
|
|
if (currentPost) {
|
|
const postSignal = findSellerSignal(
|
|
joinText([currentPost.title, currentPost.selftext, currentPost.url]),
|
|
sellerKeywords,
|
|
sellerDomains,
|
|
);
|
|
if (postSignal) {
|
|
this.logger.info(
|
|
`u/${username} flagged as seller from current post (matched ${postSignal.kind}: "${postSignal.value}")`,
|
|
);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
try {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) {
|
|
headers['Cookie'] = cookieHeader;
|
|
}
|
|
|
|
const response = await this.scheduleRequest(() => this.client.get(`/user/${encodeURIComponent(username)}/about.json`, {
|
|
params: { raw_json: 1 },
|
|
headers,
|
|
}));
|
|
|
|
const userData = response.data?.data;
|
|
if (userData) {
|
|
const subreddit = userData.subreddit;
|
|
const profileText = joinText([
|
|
subreddit?.public_description || '',
|
|
subreddit?.title || '',
|
|
subreddit?.display_name_prefixed || '',
|
|
socialLinkText(userData),
|
|
]);
|
|
const profileSignal = findSellerSignal(profileText, sellerKeywords, sellerDomains);
|
|
if (profileSignal) {
|
|
this.logger.info(
|
|
`u/${username} flagged as seller profile (matched ${profileSignal.kind}: "${profileSignal.value}")`,
|
|
);
|
|
return true;
|
|
}
|
|
const funnel = findProfileFunnel(profileText, subreddit?.over_18 === true);
|
|
if (funnel) {
|
|
this.logger.info(`u/${username} flagged as seller profile (matched funnel: "${funnel}")`);
|
|
return true;
|
|
}
|
|
}
|
|
} catch (error) {
|
|
const rateLimitError = this.toRateLimitError(error);
|
|
if (rateLimitError) throw rateLimitError;
|
|
if (axios.isAxiosError(error) && error.response?.status === 404) {
|
|
this.logger.warn(`User u/${username} profile not found`);
|
|
return false;
|
|
}
|
|
this.logger.warn(`Failed to check profile for u/${username}:`, error);
|
|
}
|
|
|
|
// Bios are often clean while recent posts advertise the paid account.
|
|
// Scan a small overview page, including comment bodies and subreddit
|
|
// names, before allowing the candidate through.
|
|
try {
|
|
const cookieHeader = await this.getCookieHeader();
|
|
const headers: Record<string, string> = {};
|
|
if (cookieHeader) headers.Cookie = cookieHeader;
|
|
const response = await this.scheduleRequest(() => this.client.get(`/user/${encodeURIComponent(username)}/overview.json`, {
|
|
params: { limit: 25, raw_json: 1 },
|
|
headers,
|
|
}));
|
|
const children: RedditListingChild[] = Array.isArray(response.data?.data?.children)
|
|
? response.data.data.children
|
|
: [];
|
|
const activityText = children.map(({ data }) => joinText([
|
|
data?.title,
|
|
data?.selftext,
|
|
data?.body,
|
|
data?.url,
|
|
data?.subreddit,
|
|
])).join(' ');
|
|
const activitySignal = findSellerSignal(activityText, sellerKeywords, sellerDomains);
|
|
if (activitySignal) {
|
|
this.logger.info(
|
|
`u/${username} flagged as seller from recent activity (matched ${activitySignal.kind}: "${activitySignal.value}")`,
|
|
);
|
|
return true;
|
|
}
|
|
} catch (error) {
|
|
const rateLimitError = this.toRateLimitError(error);
|
|
if (rateLimitError) throw rateLimitError;
|
|
this.logger.warn(`Failed to check recent activity for u/${username}:`, error);
|
|
}
|
|
|
|
return false; // Fail open if Reddit profile endpoints are unavailable.
|
|
}
|
|
|
|
getPostUrl(post: RedditPost): string {
|
|
return `https://www.reddit.com${post.permalink}`;
|
|
}
|
|
|
|
private toRateLimitError(error: unknown): RedditRateLimitError | null {
|
|
if (error instanceof RedditRateLimitError) return error;
|
|
if (!axios.isAxiosError(error) || error.response?.status !== 429) return null;
|
|
const rateLimitError = new RedditRateLimitError(getRateLimitDelayMs(error.response.headers));
|
|
this.requestScheduler.extendCooldown(rateLimitError.retryAfterMs);
|
|
this.logger.warn(rateLimitError.message);
|
|
return rateLimitError;
|
|
}
|
|
|
|
/** Record a 429 before releasing the shared request queue to its next waiter. */
|
|
private async scheduleRequest<T>(request: () => Promise<T>): Promise<T> {
|
|
return await this.requestScheduler.schedule(async () => {
|
|
try {
|
|
return await request();
|
|
} catch (error) {
|
|
const rateLimitError = this.toRateLimitError(error);
|
|
if (rateLimitError) throw rateLimitError;
|
|
throw error;
|
|
}
|
|
});
|
|
}
|
|
}
|