diff --git a/docs/configuration.md b/docs/configuration.md index 2bd998a..cf36b85 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -35,7 +35,7 @@ Never document real values. `.env` is ignored and may contain production credent - `targetGenders`: allowed parsed values. - `filterSellers`, `sellerKeywords`, `sellerDomains`: seller policy inputs. - `prioritizeRecentPosts`, `recentWindowMinutes`: recency/fallback behavior. -- `maxReadyCandidates`: maximum prepared backlog admitted before screening pauses; valid range 1–500. +- `maxReadyCandidates`: hard ceiling for the active candidate target; valid range 1–500. Runtime lowers the actual target to what authenticated running accounts can send within `recentWindowMinutes` under current rolling-hour and message-spacing limits. Subreddit `type` is `image` or `chat`. Conversation policy contains `mode`, `maxAutomatedTurns`, and `rapportTurnsBeforeMediaAsk`; hybrid rapport turns must be lower than the maximum. diff --git a/docs/data-flow.md b/docs/data-flow.md index b81b3f9..d30da10 100644 --- a/docs/data-flow.md +++ b/docs/data-flow.md @@ -25,7 +25,7 @@ stateDiagram-v2 The discovery loop rotates through enabled subreddits and authenticated runtimes. A runtime fetches Reddit `/new` data using the process-shared `RedditRequestCoordinator`. Intake deduplicates post IDs and authors, excludes processed/contacted history, stores the full post snapshot, source type, weight, and timestamps, and logs exact intake counts. -The queue has a bounded active size. Screening also pauses at configured `reddit.maxReadyCandidates`, and due delivery retries are promoted only while that ready capacity exists. Reddit request pacing, not a tight application poll, is the primary limiter. +The queue grows to a demand-based target rather than a fixed backlog. The target is the smaller of configured `reddit.maxReadyCandidates` and the work that authenticated running accounts can still send within the recent-post window, accounting for rolling hourly capacity and minimum message spacing. Intake admits only the current deficit, pauses at zero outbound capacity, and reports fetched posts left unevaluated because the funnel is full. Reddit request pacing, not a tight application poll, remains the primary request limiter. ### 2. Screening and preparation diff --git a/docs/product-and-safety.md b/docs/product-and-safety.md index 1117caf..5e7eeaa 100644 --- a/docs/product-and-safety.md +++ b/docs/product-and-safety.md @@ -16,6 +16,7 @@ The product has five connected goals: - Eligibility is completed before selecting a recipient: already-processed posts, recipient history, self-declared age/gender for chat posts, seller signals, content suitability, and AI safety all have explicit outcomes. - Existing replies take priority over new outreach. An account with actionable inbound messages is not available for a new candidate. +- Candidate inventory follows near-term outbound capacity and the recent-post window; discovery pauses instead of accumulating an ever-older backlog when accounts cannot keep pace. - Discovery is global while authentication, rate limits, conversations, and content remain account-owned. - Recipient deduplication is case-insensitive, durable across restarts, and protected by a live reservation during sends. - Remote side effects are fail-closed: a send is recorded only after the chat service confirms it; interrupted reply claims reconcile against Matrix history before retrying. diff --git a/src/database/candidate-queue-repository.ts b/src/database/candidate-queue-repository.ts index 1f2a9f9..9428482 100644 --- a/src/database/candidate-queue-repository.ts +++ b/src/database/candidate-queue-repository.ts @@ -44,6 +44,7 @@ export interface CandidateEnqueueResult { duplicateAuthor: number; previouslyProcessed: number; previouslyContacted: number; + deferredAtCapacity: number; } const ACTIVE_STATUSES: CandidateStatus[] = ['discovered', 'screening', 'ready', 'sending', 'retryable']; @@ -57,13 +58,19 @@ export class CandidateQueueRepository { sourceType: 'image' | 'chat', priority: number, now: number = Date.now(), + maxNewCandidates: number = Number.POSITIVE_INFINITY, ): CandidateEnqueueResult { + if ((!Number.isInteger(maxNewCandidates) && maxNewCandidates !== Number.POSITIVE_INFINITY) + || maxNewCandidates < 0) { + throw new Error('Candidate intake capacity must be a non-negative integer'); + } const result: CandidateEnqueueResult = { discovered: 0, duplicatePost: 0, duplicateAuthor: 0, previouslyProcessed: 0, previouslyContacted: 0, + deferredAtCapacity: 0, }; const existingPost = this.db.prepare('SELECT 1 FROM candidate_queue WHERE post_id = ?'); const processedPost = this.db.prepare('SELECT 1 FROM processed_posts WHERE id = ? LIMIT 1'); @@ -90,7 +97,11 @@ export class CandidateQueueRepository { `); this.db.transaction(() => { - for (const post of posts) { + for (const [index, post] of posts.entries()) { + if (result.discovered >= maxNewCandidates) { + result.deferredAtCapacity += posts.length - index; + break; + } if (existingPost.get(post.id)) { result.duplicatePost += 1; continue; diff --git a/src/main-loop.ts b/src/main-loop.ts index 442efa6..d83d131 100644 --- a/src/main-loop.ts +++ b/src/main-loop.ts @@ -16,6 +16,30 @@ export function currentHourlySendCount( return limiter.getStats().messagesThisHour; } +export interface CandidateCapacitySnapshot { + messagesRemaining: number; +} + +/** Keeps fresh candidate inventory proportional to near-term outbound work. */ +export function calculateCandidateFunnelTarget( + accounts: readonly CandidateCapacitySnapshot[], + options: { + maxReadyCandidates: number; + recentWindowMinutes: number; + minDelayBetweenMessagesMs: number; + }, +): number { + if (accounts.length === 0) return 0; + const freshnessWindowMs = options.recentWindowMinutes * 60_000; + const pacingCapacity = options.minDelayBetweenMessagesMs > 0 + ? Math.max(1, Math.floor(freshnessWindowMs / options.minDelayBetweenMessagesMs)) + : Number.POSITIVE_INFINITY; + const outboundCapacity = accounts.reduce((total, account) => ( + total + Math.min(Math.max(0, account.messagesRemaining), pacingCapacity) + ), 0); + return Math.min(options.maxReadyCandidates, outboundCapacity); +} + /** * Runs a task again only after its previous run has settled. This deliberately * replaces setInterval: a slow browser response check must never overlap the @@ -428,7 +452,6 @@ async function runConcurrentAccountRuntimes(app: AppContainer): Promise { const { config, logger, accountRuntimeManager, browserClient, db, webServer, candidateQueueRepo } = app; const workerInstance = `candidate-funnel-${process.pid}-${Date.now()}`; const leaseMs = 5 * 60_000; - const maxActiveCandidates = 500; const recovered = candidateQueueRepo.recoverAllLeases(); app.recipientDeduplicator.recoverStaleReservations(Date.now() - 10 * 60_000); logger.info( @@ -472,6 +495,17 @@ async function runConcurrentAccountRuntimes(app: AppContainer): Promise { )); }; + const candidateFunnelTarget = (): number => calculateCandidateFunnelTarget( + runningRuntimes() + .filter((runtime) => runtime.resources.accountAuth.isFullyAuthenticated()) + .map((runtime) => runtime.resources.rateLimiter.getTimers()), + { + maxReadyCandidates: config.reddit.maxReadyCandidates, + recentWindowMinutes: config.reddit.recentWindowMinutes, + minDelayBetweenMessagesMs: config.rateLimit.minDelayBetweenMessagesMs, + }, + ); + const discoveryLoop = (async (): Promise => { while (!controller.signal.aborted) { try { @@ -479,10 +513,6 @@ async function runConcurrentAccountRuntimes(app: AppContainer): Promise { await sleep(1_000, controller.signal).catch(() => undefined); continue; } - if (candidateQueueRepo.countActive() >= maxActiveCandidates) { - await sleep(5_000, controller.signal).catch(() => undefined); - continue; - } const enabled = config.reddit.subreddits.filter((subreddit) => subreddit.enabled !== false); if (enabled.length === 0) { await sleep(5_000, controller.signal).catch(() => undefined); @@ -493,15 +523,28 @@ async function runConcurrentAccountRuntimes(app: AppContainer): Promise { await sleep(5_000, controller.signal).catch(() => undefined); continue; } + const target = candidateFunnelTarget(); + const active = candidateQueueRepo.countActive(); + if (target === 0 || active >= target) { + await sleep(5_000, controller.signal).catch(() => undefined); + continue; + } const subreddit = enabled[nextSubredditIndex % enabled.length]; nextSubredditIndex += 1; const posts = await scanner.resources.monitorService.fetchCandidates(subreddit); - const intake = candidateQueueRepo.enqueue(posts, subreddit.type, subreddit.weight ?? 1); + const intake = candidateQueueRepo.enqueue( + posts, + subreddit.type, + subreddit.weight ?? 1, + Date.now(), + Math.max(0, target - active), + ); logger.info( `Candidate intake r/${subreddit.name}: ${posts.length} fetched; ${intake.discovered} queued; ` + `${intake.duplicatePost} duplicate posts; ${intake.duplicateAuthor} duplicate authors; ` + `${intake.previouslyProcessed} processed; ${intake.previouslyContacted} contacted; ` - + `${candidateQueueRepo.countActive()} active in funnel`, + + `${intake.deferredAtCapacity} deferred at capacity; ` + + `${candidateQueueRepo.countActive()}/${target} active in funnel`, ); } catch (error) { if (controller.signal.aborted) return; @@ -526,7 +569,8 @@ async function runConcurrentAccountRuntimes(app: AppContainer): Promise { candidateQueueRepo.expireStale( Date.now() - config.reddit.recentWindowMinutes * 60_000, ); - if (candidateQueueRepo.countReady() >= config.reddit.maxReadyCandidates) { + const target = candidateFunnelTarget(); + if (target === 0 || candidateQueueRepo.countReady() >= target) { await sleep(2_000, controller.signal).catch(() => undefined); continue; } @@ -589,7 +633,8 @@ async function runConcurrentAccountRuntimes(app: AppContainer): Promise { await sleep(1_000, controller.signal).catch(() => undefined); continue; } - candidateQueueRepo.promoteDueRetries(config.reddit.maxReadyCandidates); + const target = candidateFunnelTarget(); + if (target > 0) candidateQueueRepo.promoteDueRetries(target); candidateQueueRepo.recoverExpiredLeases(); app.recipientDeduplicator.recoverStaleReservations(Date.now() - 10 * 60_000); const recipients = await availableRecipients(); diff --git a/tests/candidate-queue.test.ts b/tests/candidate-queue.test.ts index 0fa15da..9a2d74a 100644 --- a/tests/candidate-queue.test.ts +++ b/tests/candidate-queue.test.ts @@ -77,10 +77,23 @@ describe('CandidateQueueRepository', () => { duplicateAuthor: 1, previouslyProcessed: 0, previouslyContacted: 0, + deferredAtCapacity: 0, }); expect(queue.getStats().byStatus.discovered).toBe(2); }); + it('fills only the requested intake deficit and reports untouched posts', () => { + const result = queue.enqueue([ + post('p1', 'alice'), + post('p2', 'bob'), + post('p3', 'carol'), + ], 'image', 1, 1_000, 2); + + expect(result.discovered).toBe(2); + expect(result.deferredAtCapacity).toBe(1); + expect(queue.countActive()).toBe(2); + }); + it('excludes posts and recipients already handled anywhere', () => { db.prepare(` INSERT INTO processed_posts (account_id, id, subreddit, author, title, processed_at) diff --git a/tests/main-loop.test.ts b/tests/main-loop.test.ts index 2d2128f..743dcd3 100644 --- a/tests/main-loop.test.ts +++ b/tests/main-loop.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it, vi } from 'vitest'; -import { currentHourlySendCount, startNonOverlappingSchedule } from '../src/main-loop.js'; +import { + calculateCandidateFunnelTarget, + currentHourlySendCount, + startNonOverlappingSchedule, +} from '../src/main-loop.js'; describe('startNonOverlappingSchedule', () => { it('does not begin a second response check until the prior check settles', async () => { @@ -43,3 +47,29 @@ describe('currentHourlySendCount', () => { expect(currentHourlySendCount({ getStats: () => ({ messagesThisHour: 1 }) })).toBe(1); }); }); + +describe('calculateCandidateFunnelTarget', () => { + it('tracks remaining outbound capacity instead of growing to a fixed backlog', () => { + expect(calculateCandidateFunnelTarget( + [{ messagesRemaining: 3 }, { messagesRemaining: 5 }], + { maxReadyCandidates: 25, recentWindowMinutes: 60, minDelayBetweenMessagesMs: 300_000 }, + )).toBe(8); + }); + + it('retains the configured ceiling and caps capacity by the freshness window', () => { + expect(calculateCandidateFunnelTarget( + [{ messagesRemaining: 100 }, { messagesRemaining: 100 }], + { maxReadyCandidates: 25, recentWindowMinutes: 30, minDelayBetweenMessagesMs: 300_000 }, + )).toBe(12); + }); + + it('stops growing inventory when no running account can send', () => { + const options = { + maxReadyCandidates: 25, + recentWindowMinutes: 60, + minDelayBetweenMessagesMs: 300_000, + }; + expect(calculateCandidateFunnelTarget([], options)).toBe(0); + expect(calculateCandidateFunnelTarget([{ messagesRemaining: 0 }], options)).toBe(0); + }); +});