mirror of
https://github.com/jjbrunton/Sauci.git
synced 2026-11-02 02:19:43 +00:00
perf(api): bound worker pools and add telemetry
Share one configured PostgreSQL pool across worker repositories and make API and worker connection budgets explicit. Add privacy-safe aggregate request, query, pool, sync, and outbox telemetry, a loopback-only representative load fixture, bounded worker shutdown, and focused regression and integration coverage. Document the database-wide budget formula and defer pg_stat_statements activation to a reviewed migration and planned restart.
This commit is contained in:
1 parent
a13bbc3f65
commit
5c973c4ac5
25 files changed
+471
-16
No files matched your search
@@ -15,9 +15,19 @@ export interface OperationsRepository {
|
||||
message(id: string): Promise<MessageForClassification | null>;
|
||||
classifierConfig(): Promise<ClassifierRuntimeConfig | null>;
|
||||
classify(id: string, status: 'safe' | 'flagged', reason: string | null, category: string): Promise<void>;
|
||||
outboxState(): Promise<{ due: number; oldestDueAgeSeconds: number }>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
/** Matches `claim`: only currently due, retryable, and unlocked/expired leases. */
|
||||
export const OUTBOX_STATE_SQL = `select count(*)::int due,
|
||||
coalesce(floor(extract(epoch from now()-min(available_at))),0)::int oldest_due_age_seconds
|
||||
from operations_outbox
|
||||
where sent_at is null
|
||||
and attempts<5
|
||||
and available_at<=now()
|
||||
and (locked_at is null or locked_at<now()-interval '5 minutes')`;
|
||||
|
||||
function item(row: OutboxRow): OperationItem {
|
||||
return { id: row.id, kind: row.kind, dedupeKey: row.dedupe_key, recipientId: row.recipient_id,
|
||||
pushToken: row.push_token, payload: row.payload, attempts: row.attempts };
|
||||
@@ -280,6 +290,30 @@ export class PostgresOperationsRepository implements OperationsRepository {
|
||||
return result.rows.map(item);
|
||||
});
|
||||
}
|
||||
async outboxState(): Promise<{ due: number; oldestDueAgeSeconds: number }> {
|
||||
const client = await this.pool.connect();
|
||||
let released = false;
|
||||
try {
|
||||
await client.query('begin');
|
||||
// A server-side timeout cancels the query and transaction; unlike a
|
||||
// Promise race it cannot leave an unobserved query or borrowed client.
|
||||
await client.query("set local statement_timeout = '250ms'");
|
||||
const result = await client.query<{ due: number; oldest_due_age_seconds: number }>(OUTBOX_STATE_SQL);
|
||||
await client.query('commit');
|
||||
const row = result.rows[0];
|
||||
return { due: row?.due ?? 0, oldestDueAgeSeconds: row?.oldest_due_age_seconds ?? 0 };
|
||||
} catch (cause) {
|
||||
try { await client.query('rollback'); }
|
||||
catch (rollbackCause) {
|
||||
// A failed rollback leaves transaction state uncertain. Evict this
|
||||
// client instead of returning it to the shared pool, but retain the
|
||||
// original observation failure for callers and logs.
|
||||
client.release(rollbackCause instanceof Error ? rollbackCause : new Error('outbox observation rollback failed'));
|
||||
released = true;
|
||||
}
|
||||
throw cause;
|
||||
} finally { if (!released) client.release(); }
|
||||
}
|
||||
async complete(id:string) { await this.pool.query('update operations_outbox set sent_at=now(),locked_at=null,last_error=null where id=$1',[id]); }
|
||||
async fail(id:string,message:string) { await this.pool.query(
|
||||
`update operations_outbox set locked_at=null,last_error=$2,available_at=now()+make_interval(secs=>least(3600,power(2,attempts)::int*30)) where id=$1`,[id,message.slice(0,500)]); }
|
||||
|
||||
Reference in new issue
Block a user