jjbrunton/RedditBot
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add agent API and operational analytics

Browse Source
This commit is contained in:
jjbrunton committed 2026-08-08 22:08:14 +01:00
1 parent 720ea0652a
commit e8b730b592
65 files changed
+2650 -1214
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/reddit/api.ts
+48 -25
View File
@@ -2,6 +2,12 @@ import axios, { AxiosInstance } from 'axios';
import type { BrowserContext } from 'playwright';
import type { Config } from '../config/index.js';
import type { Logger } from '../utils/logger.js';
import type { RedditRequestScheduler } from './request-coordinator.js';
const uncoordinatedRequestScheduler: RedditRequestScheduler = {
schedule: async <T>(request: () => Promise<T>) => await request(),
extendCooldown: () => undefined,
};
export class RedditRateLimitError extends Error {
constructor(public readonly retryAfterMs: number) {
@@ -105,7 +111,10 @@ export class RedditApiClient {
constructor(
config: Config,
private logger: Logger
private logger: Logger,
// Production bootstraps supply one process-wide paced coordinator. Keep
// standalone clients independent for embedders and focused unit tests.
private readonly requestScheduler: RedditRequestScheduler = uncoordinatedRequestScheduler,
) {
this.client = axios.create({
baseURL: 'https://www.reddit.com',
@@ -143,15 +152,17 @@ export class RedditApiClient {
if (cookieHeader) headers.Cookie = cookieHeader;
try {
const response = await this.client.get(
const response = await this.scheduleRequest(() => this.client.get(
`/user/${encodeURIComponent(username)}/about.json`,
{ params: { raw_json: 1 }, headers },
);
));
const id = response.data?.data?.id;
if (typeof id !== 'string' || id.length === 0) return null;
return id.startsWith('t2_') ? id : `t2_${id}`;
} catch (error) {
if (axios.isAxiosError(error) && error.response?.status === 404) return null;
const rateLimitError = this.toRateLimitError(error);
if (rateLimitError) throw rateLimitError;
throw error;
}
}
@@ -164,10 +175,10 @@ export class RedditApiClient {
headers['Cookie'] = cookieHeader;
}
const response = await this.client.get(`/r/${subreddit}/new.json`, {
const response = await this.scheduleRequest(() => this.client.get(`/r/${subreddit}/new.json`, {
params: { limit, raw_json: 1 },
headers,
});
}));
const posts: RedditPost[] = response.data.data.children
.map((child: { data: RedditPost }) => child.data)
@@ -185,12 +196,8 @@ export class RedditApiClient {
this.logger.warn(`Subreddit r/${subreddit} is private or quarantined`);
return [];
}
if (error.response?.status === 429) {
const retryAfterMs = getRateLimitDelayMs(error.response.headers);
const rateLimitError = new RedditRateLimitError(retryAfterMs);
this.logger.warn(rateLimitError.message);
throw rateLimitError;
}
const rateLimitError = this.toRateLimitError(error);
if (rateLimitError) throw rateLimitError;
}
this.logger.error(`Error fetching posts from r/${subreddit}:`, error);
throw error;
@@ -310,10 +317,10 @@ export class RedditApiClient {
headers['Cookie'] = cookieHeader;
}
const response = await this.client.get(`/user/${encodeURIComponent(username)}/about.json`, {
const response = await this.scheduleRequest(() => this.client.get(`/user/${encodeURIComponent(username)}/about.json`, {
params: { raw_json: 1 },
headers,
});
}));
const userData = response.data?.data;
if (userData) {
@@ -331,11 +338,8 @@ export class RedditApiClient {
}
}
} catch (error) {
if (axios.isAxiosError(error) && error.response?.status === 429) {
const rateLimitError = new RedditRateLimitError(getRateLimitDelayMs(error.response.headers));
this.logger.warn(rateLimitError.message);
throw rateLimitError;
}
const rateLimitError = this.toRateLimitError(error);
if (rateLimitError) throw rateLimitError;
if (axios.isAxiosError(error) && error.response?.status === 404) {
this.logger.warn(`User u/${username} profile not found`);
return false;
@@ -350,10 +354,10 @@ export class RedditApiClient {
const cookieHeader = await this.getCookieHeader();
const headers: Record<string, string> = {};
if (cookieHeader) headers.Cookie = cookieHeader;
const response = await this.client.get(`/user/${encodeURIComponent(username)}/overview.json`, {
const response = await this.scheduleRequest(() => this.client.get(`/user/${encodeURIComponent(username)}/overview.json`, {
params: { limit: 25, raw_json: 1 },
headers,
});
}));
const children: RedditListingChild[] = Array.isArray(response.data?.data?.children)
? response.data.data.children
: [];
@@ -372,11 +376,8 @@ export class RedditApiClient {
return true;
}
} catch (error) {
if (axios.isAxiosError(error) && error.response?.status === 429) {
const rateLimitError = new RedditRateLimitError(getRateLimitDelayMs(error.response.headers));
this.logger.warn(rateLimitError.message);
throw rateLimitError;
}
const rateLimitError = this.toRateLimitError(error);
if (rateLimitError) throw rateLimitError;
this.logger.warn(`Failed to check recent activity for u/${username}:`, error);
}
@@ -386,4 +387,26 @@ export class RedditApiClient {
getPostUrl(post: RedditPost): string {
return `https://www.reddit.com${post.permalink}`;
}
private toRateLimitError(error: unknown): RedditRateLimitError | null {
if (error instanceof RedditRateLimitError) return error;
if (!axios.isAxiosError(error) || error.response?.status !== 429) return null;
const rateLimitError = new RedditRateLimitError(getRateLimitDelayMs(error.response.headers));
this.requestScheduler.extendCooldown(rateLimitError.retryAfterMs);
this.logger.warn(rateLimitError.message);
return rateLimitError;
}
/** Record a 429 before releasing the shared request queue to its next waiter. */
private async scheduleRequest<T>(request: () => Promise<T>): Promise<T> {
return await this.requestScheduler.schedule(async () => {
try {
return await request();
} catch (error) {
const rateLimitError = this.toRateLimitError(error);
if (rateLimitError) throw rateLimitError;
throw error;
}
});
}
}
src/reddit/request-coordinator.ts
+56
View File
@@ -0,0 +1,56 @@
/**
* Coordinates every Reddit JSON request made by the process. Reddit applies
* limits to the shared origin/IP as well as to an individual signed-in user,
* so separate account runtimes must not independently retry or burst reads.
*/
export interface RedditRequestScheduler {
schedule<T>(request: () => Promise<T>): Promise<T>;
extendCooldown(retryAfterMs: number): void;
}
export interface RedditRequestCoordinatorDependencies {
now?: () => number;
sleep?: (ms: number) => Promise<void>;
}
const DEFAULT_MIN_REQUEST_INTERVAL_MS = 6_500;
const COOLDOWN_BUFFER_MS = 1_000;
export class RedditRequestCoordinator implements RedditRequestScheduler {
private tail: Promise<void> = Promise.resolve();
private nextRequestAt = 0;
private readonly now: () => number;
private readonly sleep: (ms: number) => Promise<void>;
constructor(
private readonly minRequestIntervalMs = DEFAULT_MIN_REQUEST_INTERVAL_MS,
dependencies: RedditRequestCoordinatorDependencies = {},
) {
this.now = dependencies.now ?? Date.now;
this.sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
}
async schedule<T>(request: () => Promise<T>): Promise<T> {
let release: () => void = () => undefined;
const previous = this.tail;
this.tail = new Promise<void>((resolve) => { release = resolve; });
await previous;
try {
const waitMs = this.nextRequestAt - this.now();
if (waitMs > 0) await this.sleep(waitMs);
const result = await request();
this.nextRequestAt = Math.max(this.nextRequestAt, this.now() + this.minRequestIntervalMs);
return result;
} finally {
release();
}
}
extendCooldown(retryAfterMs: number): void {
this.nextRequestAt = Math.max(
this.nextRequestAt,
this.now() + Math.max(0, retryAfterMs) + COOLDOWN_BUFFER_MS,
);
}
}