fix: back off after Reddit access denials
This commit is contained in:
1 parent
e8b730b592
commit
53e9a44a5c
5 files changed
+81
-14
No files matched your search
+25
-1
@@ -17,6 +17,13 @@ export class RedditRateLimitError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
export class RedditAccessDeniedError extends Error {
|
||||
constructor(public readonly subreddit: string) {
|
||||
super(`Reddit denied listing access for r/${subreddit} (HTTP 403)`);
|
||||
this.name = 'RedditAccessDeniedError';
|
||||
}
|
||||
}
|
||||
|
||||
function getRateLimitDelayMs(headers: unknown): number {
|
||||
if (!headers || typeof headers !== 'object') return 60_000;
|
||||
|
||||
@@ -108,6 +115,7 @@ export function findSellerDomain(text: string, domains: string[]): string | null
|
||||
export class RedditApiClient {
|
||||
private client: AxiosInstance;
|
||||
private browserContext: BrowserContext | null = null;
|
||||
private consecutiveListing403s = 0;
|
||||
|
||||
constructor(
|
||||
config: Config,
|
||||
@@ -185,6 +193,7 @@ export class RedditApiClient {
|
||||
.filter((post: RedditPost) => this.isValidPost(post, type));
|
||||
|
||||
this.logger.debug(`Fetched ${posts.length} valid ${type} posts from r/${subreddit}`);
|
||||
this.consecutiveListing403s = 0;
|
||||
return posts;
|
||||
} catch (error) {
|
||||
if (axios.isAxiosError(error)) {
|
||||
@@ -193,7 +202,22 @@ export class RedditApiClient {
|
||||
return [];
|
||||
}
|
||||
if (error.response?.status === 403) {
|
||||
this.logger.warn(`Subreddit r/${subreddit} is private or quarantined`);
|
||||
this.consecutiveListing403s += 1;
|
||||
const denied = new RedditAccessDeniedError(subreddit);
|
||||
|
||||
// One 403 can genuinely be a private or quarantined subreddit. A
|
||||
// second consecutive denial from unrelated listings is a strong
|
||||
// signal that Reddit is temporarily rejecting this session/IP.
|
||||
if (this.consecutiveListing403s >= 2) {
|
||||
const cooldownMs = 120_000;
|
||||
this.requestScheduler.extendCooldown(cooldownMs);
|
||||
this.logger.warn(
|
||||
`Reddit is temporarily denying listing requests (consecutive HTTP 403s); pausing for ${cooldownMs / 1000}s`,
|
||||
);
|
||||
throw new RedditRateLimitError(cooldownMs);
|
||||
}
|
||||
|
||||
this.logger.warn(`${denied.message}; it may be private, quarantined, or a temporary Reddit block`);
|
||||
return [];
|
||||
}
|
||||
const rateLimitError = this.toRateLimitError(error);
|
||||
|
||||
Reference in new issue
Block a user