diff --git a/src/reddit/api.ts b/src/reddit/api.ts index 0a9cedf..66abb7c 100644 --- a/src/reddit/api.ts +++ b/src/reddit/api.ts @@ -17,6 +17,13 @@ export class RedditRateLimitError extends Error { } } +export class RedditAccessDeniedError extends Error { + constructor(public readonly subreddit: string) { + super(`Reddit denied listing access for r/${subreddit} (HTTP 403)`); + this.name = 'RedditAccessDeniedError'; + } +} + function getRateLimitDelayMs(headers: unknown): number { if (!headers || typeof headers !== 'object') return 60_000; @@ -108,6 +115,7 @@ export function findSellerDomain(text: string, domains: string[]): string | null export class RedditApiClient { private client: AxiosInstance; private browserContext: BrowserContext | null = null; + private consecutiveListing403s = 0; constructor( config: Config, @@ -185,6 +193,7 @@ export class RedditApiClient { .filter((post: RedditPost) => this.isValidPost(post, type)); this.logger.debug(`Fetched ${posts.length} valid ${type} posts from r/${subreddit}`); + this.consecutiveListing403s = 0; return posts; } catch (error) { if (axios.isAxiosError(error)) { @@ -193,7 +202,22 @@ export class RedditApiClient { return []; } if (error.response?.status === 403) { - this.logger.warn(`Subreddit r/${subreddit} is private or quarantined`); + this.consecutiveListing403s += 1; + const denied = new RedditAccessDeniedError(subreddit); + + // One 403 can genuinely be a private or quarantined subreddit. A + // second consecutive denial from unrelated listings is a strong + // signal that Reddit is temporarily rejecting this session/IP. + if (this.consecutiveListing403s >= 2) { + const cooldownMs = 120_000; + this.requestScheduler.extendCooldown(cooldownMs); + this.logger.warn( + `Reddit is temporarily denying listing requests (consecutive HTTP 403s); pausing for ${cooldownMs / 1000}s`, + ); + throw new RedditRateLimitError(cooldownMs); + } + + this.logger.warn(`${denied.message}; it may be private, quarantined, or a temporary Reddit block`); return []; } const rateLimitError = this.toRateLimitError(error); diff --git a/src/reddit/request-coordinator.ts b/src/reddit/request-coordinator.ts index 41c0f47..0889480 100644 --- a/src/reddit/request-coordinator.ts +++ b/src/reddit/request-coordinator.ts @@ -39,10 +39,13 @@ export class RedditRequestCoordinator implements RedditRequestScheduler { try { const waitMs = this.nextRequestAt - this.now(); if (waitMs > 0) await this.sleep(waitMs); - const result = await request(); - this.nextRequestAt = Math.max(this.nextRequestAt, this.now() + this.minRequestIntervalMs); - return result; + return await request(); } finally { + // A rejected request still reached Reddit. In particular, 403 responses + // are often a temporary anti-abuse block, so immediately trying the next + // subreddit turns one denial into a burst. Keep the same spacing after + // every attempted request; an explicit 429 cooldown remains authoritative. + this.nextRequestAt = Math.max(this.nextRequestAt, this.now() + this.minRequestIntervalMs); release(); } } diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 14294f0..4624619 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -1,5 +1,6 @@ import winston from 'winston'; import Transport from 'winston-transport'; +import { toJsonSafe } from './json-safe.js'; export type Logger = winston.Logger; @@ -37,27 +38,35 @@ export function attachLogSink(logger: Logger, sink: LogSink): void { } function safeStringify(value: unknown): string { - const seen = new WeakSet(); try { - return JSON.stringify(value, (_key, val) => { - if (typeof val === 'object' && val !== null) { - if (seen.has(val)) return '[Circular]'; - seen.add(val); - } - if (typeof val === 'bigint') return val.toString(); - return val; - }); + return JSON.stringify(toJsonSafe(value)); } catch (err) { return `[Unserializable: ${(err as Error).message}]`; } } +/** Strip credentials and network internals before either console or file output. */ +const redactLogMetadata = winston.format((info) => { + const record = info as Record; + const { level, message, timestamp, ...metadata } = record; + const safeMetadata = toJsonSafe(metadata); + for (const key of Object.keys(metadata)) delete record[key]; + if (safeMetadata && !Array.isArray(safeMetadata) && typeof safeMetadata === 'object') { + Object.assign(record, safeMetadata); + } + record.level = level; + record.message = message; + if (timestamp !== undefined) record.timestamp = timestamp; + return info; +}); + export function createLogger(): Logger { return winston.createLogger({ level: process.env.LOG_LEVEL || 'info', format: winston.format.combine( winston.format.timestamp(), winston.format.errors({ stack: true }), + redactLogMetadata(), winston.format.json() ), transports: [ diff --git a/tests/reddit-api.test.ts b/tests/reddit-api.test.ts index f3db3ed..a9b77be 100644 --- a/tests/reddit-api.test.ts +++ b/tests/reddit-api.test.ts @@ -490,7 +490,7 @@ describe('RedditApiClient', () => { expect(result).toEqual([]); }); - it('returns empty array for 403 private subreddit', async () => { + it('returns empty array and explains an initial 403 access denial', async () => { const error = { isAxiosError: true, response: { status: 403 } }; const mockAxios = { get: vi.fn().mockRejectedValue(error) }; vi.mocked(axios.create).mockReturnValue(mockAxios as any); @@ -498,6 +498,23 @@ describe('RedditApiClient', () => { const client2 = new RedditApiClient(getTestConfig(), logger); const result = await client2.getNewPosts('private'); expect(result).toEqual([]); + expect(logger.logs).toContainEqual(expect.objectContaining({ + level: 'warn', message: expect.stringContaining('HTTP 403'), + })); + }); + + it('treats consecutive 403 listing responses as a temporary Reddit block', async () => { + const error = { isAxiosError: true, response: { status: 403 } }; + const mockAxios = { get: vi.fn().mockRejectedValue(error) }; + vi.mocked(axios.create).mockReturnValue(mockAxios as unknown as AxiosInstance); + const scheduler: RedditRequestScheduler = { schedule: async (request) => await request(), extendCooldown: vi.fn() }; + const client2 = new RedditApiClient(getTestConfig(), logger, scheduler); + + await expect(client2.getNewPosts('first')).resolves.toEqual([]); + await expect(client2.getNewPosts('second')).rejects.toMatchObject({ + name: 'RedditRateLimitError', retryAfterMs: 120_000, + }); + expect(scheduler.extendCooldown).toHaveBeenCalledWith(120_000); }); it('throws a compact error with the Reddit rate-limit reset delay on 429', async () => { diff --git a/tests/reddit-request-coordinator.test.ts b/tests/reddit-request-coordinator.test.ts index bfdf70a..a01a398 100644 --- a/tests/reddit-request-coordinator.test.ts +++ b/tests/reddit-request-coordinator.test.ts @@ -55,4 +55,18 @@ describe('RedditRequestCoordinator', () => { await second; expect(sleeps).toEqual([16_000]); }); + + it('paces a queued request after a failed attempt without an explicit cooldown', async () => { + let now = 0; + const sleeps: number[] = []; + const coordinator = new RedditRequestCoordinator(6_500, { + now: () => now, + sleep: async (ms) => { sleeps.push(ms); now += ms; }, + }); + + await expect(coordinator.schedule(async () => { throw new Error('403'); })).rejects.toThrow('403'); + await coordinator.schedule(async () => undefined); + + expect(sleeps).toEqual([6_500]); + }); });