perf(api): bound worker pools and add telemetry

Share one configured PostgreSQL pool across worker repositories and make API and worker connection budgets explicit.

Add privacy-safe aggregate request, query, pool, sync, and outbox telemetry, a loopback-only representative load fixture, bounded worker shutdown, and focused regression and integration coverage.

Document the database-wide budget formula and defer pg_stat_statements activation to a reviewed migration and planned restart.

apps/api/src/workers/config.ts

@@ -4,6 +4,7 @@ const optionalString=z.preprocess(value=>value===''?undefined:value,z.string().m
 const optionalUrl=z.preprocess(value=>value===''?undefined:value,z.string().url().optional());
 const schema=z.object({
   NODE_ENV:z.enum(['development','test','production']).default('development'), DATABASE_URL:z.string().url(),
+  DATABASE_POOL_MAX:z.coerce.number().int().min(1).max(200).default(4), DATABASE_POOL_IDLE_TIMEOUT_MS:z.coerce.number().int().min(0).max(3_600_000).default(30_000), DATABASE_POOL_CONNECTION_TIMEOUT_MS:z.coerce.number().int().min(0).max(60_000).default(10_000),
   WORKER_POLL_INTERVAL_MS:z.coerce.number().int().min(1000).max(300000).default(30000), WORKER_BATCH_SIZE:z.coerce.number().int().min(1).max(100).default(25),
   MEDIA_ROOT:z.string().min(1).default('/data/media'), MEDIA_SIGNING_SECRET:z.string().min(32).default('local-media-signing-secret-change-me-123456'),
   MEDIA_PUBLIC_BASE_URL:z.string().url().default('http://127.0.0.1:3003'), CLASSIFIER_ENABLED:z.enum(['true','false']).default('true').transform(v=>v==='true'),
@@ -15,7 +16,7 @@ export function loadWorkerConfig(env:NodeJS.ProcessEnv=process.env) {
   const value=schema.parse(env);
   if(value.NODE_ENV==='production'&&value.CLASSIFIER_ENABLED&&!value.OPENROUTER_API_KEY)throw new Error('OPENROUTER_API_KEY is required when classification is enabled');
   if(value.ADMIN_PRIVATE_KEY_JWK){try{JSON.parse(value.ADMIN_PRIVATE_KEY_JWK);}catch{throw new Error('ADMIN_PRIVATE_KEY_JWK must be valid JSON');}}
-  return {nodeEnv:value.NODE_ENV,databaseUrl:value.DATABASE_URL,pollIntervalMs:value.WORKER_POLL_INTERVAL_MS,batchSize:value.WORKER_BATCH_SIZE,
+  return {nodeEnv:value.NODE_ENV,databaseUrl:value.DATABASE_URL,databasePool:{max:value.DATABASE_POOL_MAX,idleTimeoutMillis:value.DATABASE_POOL_IDLE_TIMEOUT_MS,connectionTimeoutMillis:value.DATABASE_POOL_CONNECTION_TIMEOUT_MS},pollIntervalMs:value.WORKER_POLL_INTERVAL_MS,batchSize:value.WORKER_BATCH_SIZE,
     mediaRoot:value.MEDIA_ROOT,mediaSigningSecret:value.MEDIA_SIGNING_SECRET,mediaPublicBaseUrl:value.MEDIA_PUBLIC_BASE_URL,classifierEnabled:value.CLASSIFIER_ENABLED,
     openRouterApiKey:value.OPENROUTER_API_KEY,classifierModel:value.CLASSIFIER_MODEL,classifierPrompt:env.CLASSIFIER_PROMPT||defaultPrompt,
     adminPrivateKeyJwk:value.ADMIN_PRIVATE_KEY_JWK,discordWebhookUrl:value.DISCORD_WEBHOOK_URL};

apps/api/src/workers/index.ts

@@ -3,14 +3,16 @@ import { PostgresMediaRepository } from '../domains/media/repository.js';
 import { FilesystemMediaStorage } from '../domains/media/storage.js';
 import { DiscordProvider, ExpoPushProvider, MessageClassifier, OperationsRunner, PostgresOperationsRepository } from '../domains/operations/index.js';
 import { loadWorkerConfig } from './config.js';
+import { createDatabasePool } from '../db/pool.js';
+import { flushTelemetry, recordPool, recordSync } from '../telemetry.js';
+import { WorkerLifecycle } from './lifecycle.js';
 
-const config=loadWorkerConfig(); const operations=new PostgresOperationsRepository(config.databaseUrl); const mediaRepository=new PostgresMediaRepository(config.databaseUrl);
+const config=loadWorkerConfig(); const pool=createDatabasePool(config.databaseUrl,config.databasePool,'worker'); const operations=new PostgresOperationsRepository(pool); const mediaRepository=new PostgresMediaRepository(pool);
 const mediaStorage=new FilesystemMediaStorage(config.mediaRoot,config.mediaSigningSecret,config.mediaPublicBaseUrl);
 const runner=new OperationsRunner(operations,new ExpoPushProvider(),new DiscordProvider(config.discordWebhookUrl),new MessageClassifier(operations,
   {enabled:config.classifierEnabled,apiKey:config.openRouterApiKey,model:config.classifierModel,prompt:config.classifierPrompt,adminPrivateKeyJwk:config.adminPrivateKeyJwk,mediaRoot:config.mediaRoot}),
   new MediaJanitor(mediaRepository,mediaStorage));
-let stopped=false;let timer:NodeJS.Timeout|undefined;
-async function tick(){if(stopped)return;try{const result=await runner.runOnce(new Date(),config.batchSize);if(result.claimed||result.failed)console.log('Operations worker tick',result);}catch(cause){console.error('Operations worker tick failed',cause);}finally{if(!stopped){timer=setTimeout(()=>void tick(),config.pollIntervalMs);}}}
-async function shutdown(signal:string){if(stopped)return;stopped=true;if(timer)clearTimeout(timer);console.log(`Received ${signal}; stopping operations worker`);await Promise.all([operations.close(),mediaRepository.close()]);}
-process.on('SIGINT',()=>void shutdown('SIGINT').then(()=>process.exit(0))); process.on('SIGTERM',()=>void shutdown('SIGTERM').then(()=>process.exit(0)));
-console.log('Sauci operations worker started');void tick();
+const lifecycle=new WorkerLifecycle({pollIntervalMs:config.pollIntervalMs,closePool:()=>pool.end(),onTickFailure:cause=>console.error('Operations worker tick failed',cause),runTick:async()=>{const startedAt=performance.now();try{const result=await runner.runOnce(new Date(),config.batchSize);recordSync('worker','ok',startedAt);recordPool('worker',pool);flushTelemetry();if(result.claimed||result.failed)console.log('Operations worker tick',result);}catch(cause){recordSync('worker','error',startedAt);recordPool('worker',pool);flushTelemetry();throw cause;}}});
+function shutdown(signal:string){console.log(`Received ${signal}; stopping operations worker`);return lifecycle.shutdown();}
+process.on('SIGINT',()=>void shutdown('SIGINT').then(code=>process.exit(code))); process.on('SIGTERM',()=>void shutdown('SIGTERM').then(code=>process.exit(code)));
+console.log('Sauci operations worker started');lifecycle.start();

apps/api/src/workers/lifecycle.ts

@@ -0,0 +1,47 @@
+export interface WorkerLifecycleDependencies {
+  pollIntervalMs: number;
+  runTick: () => Promise<void>;
+  closePool: () => Promise<void>;
+  onTickFailure: (cause: unknown) => void;
+  graceMs?: number;
+}
+
+/** Owns worker tick scheduling and makes shutdown single-flight and bounded. */
+export class WorkerLifecycle {
+  private stopped = false;
+  private timer: NodeJS.Timeout | undefined;
+  private activeTick: Promise<void> | undefined;
+  private shutdownPromise: Promise<number> | undefined;
+  private readonly graceMs: number;
+
+  constructor(private readonly deps: WorkerLifecycleDependencies) { this.graceMs = deps.graceMs ?? 25_000; }
+
+  start(): void { if (!this.activeTick && !this.stopped) this.activeTick = this.tick(); }
+
+  shutdown(): Promise<number> {
+    if (!this.shutdownPromise) this.shutdownPromise = this.stop();
+    return this.shutdownPromise;
+  }
+
+  private async tick(): Promise<void> {
+    try { await this.deps.runTick(); }
+    catch (cause) { this.deps.onTickFailure(cause); }
+    finally {
+      if (!this.stopped) this.timer = setTimeout(() => { this.activeTick = this.tick(); }, this.deps.pollIntervalMs);
+    }
+  }
+
+  private async stop(): Promise<number> {
+    this.stopped = true;
+    if (this.timer) clearTimeout(this.timer);
+    const work = (async () => {
+      try { if (this.activeTick) await this.activeTick; await this.deps.closePool(); return 0; }
+      catch { return 1; }
+    })();
+    let timeout: NodeJS.Timeout | undefined;
+    const timedOut = new Promise<number>((resolve) => { timeout = setTimeout(() => resolve(1), this.graceMs); });
+    const result = await Promise.race([work, timedOut]);
+    if (timeout) clearTimeout(timeout);
+    return result;
+  }
+}