perf(api): bound worker pools and add telemetry

Share one configured PostgreSQL pool across worker repositories and make API and worker connection budgets explicit.

Add privacy-safe aggregate request, query, pool, sync, and outbox telemetry, a loopback-only representative load fixture, bounded worker shutdown, and focused regression and integration coverage.

Document the database-wide budget formula and defer pg_stat_statements activation to a reviewed migration and planned restart.

apps/api/src/domains/app-data/routes.ts

@@ -3,6 +3,7 @@ import type { Hono } from 'hono';
 import { z } from 'zod';
 import type { AuthIdentity } from '../../auth.js';
 import { AppDataError, type AppDataRepository } from './repository.js';
+import { recordSync } from '../../telemetry.js';
 
 type App = Hono<{ Variables: { identity: AuthIdentity } }>;
 const uuid = z.string().uuid();
@@ -46,7 +47,9 @@ export function registerAppDataRoutes(app: App, repository: AppDataRepository):
   });
   // The client polls this instead of re-fetching every domain; see SyncSummary.
   app.get('/v1/me/sync', async c => {
+    const startedAt = performance.now();
     const result = await run(() => repository.syncSummary(c.get('identity').id));
+    recordSync('api',result.ok?'ok':'error',startedAt);
     return result.ok ? c.json(result.value) : c.json(error(result.cause.code, result.cause.message), result.cause.status);
   });
   app.get('/v1/me/nudge-status', async c => c.json(await repository.nudgeStatus(c.get('identity').id)));

apps/api/src/domains/operations/repository.ts

@@ -15,9 +15,19 @@ export interface OperationsRepository {
   message(id: string): Promise<MessageForClassification | null>;
   classifierConfig(): Promise<ClassifierRuntimeConfig | null>;
   classify(id: string, status: 'safe' | 'flagged', reason: string | null, category: string): Promise<void>;
+  outboxState(): Promise<{ due: number; oldestDueAgeSeconds: number }>;
   close(): Promise<void>;
 }
 
+/** Matches `claim`: only currently due, retryable, and unlocked/expired leases. */
+export const OUTBOX_STATE_SQL = `select count(*)::int due,
+  coalesce(floor(extract(epoch from now()-min(available_at))),0)::int oldest_due_age_seconds
+ from operations_outbox
+where sent_at is null
+  and attempts<5
+  and available_at<=now()
+  and (locked_at is null or locked_at<now()-interval '5 minutes')`;
+
 function item(row: OutboxRow): OperationItem {
   return { id: row.id, kind: row.kind, dedupeKey: row.dedupe_key, recipientId: row.recipient_id,
     pushToken: row.push_token, payload: row.payload, attempts: row.attempts };
@@ -280,6 +290,30 @@ export class PostgresOperationsRepository implements OperationsRepository {
       return result.rows.map(item);
     });
   }
+  async outboxState(): Promise<{ due: number; oldestDueAgeSeconds: number }> {
+    const client = await this.pool.connect();
+    let released = false;
+    try {
+      await client.query('begin');
+      // A server-side timeout cancels the query and transaction; unlike a
+      // Promise race it cannot leave an unobserved query or borrowed client.
+      await client.query("set local statement_timeout = '250ms'");
+      const result = await client.query<{ due: number; oldest_due_age_seconds: number }>(OUTBOX_STATE_SQL);
+      await client.query('commit');
+      const row = result.rows[0];
+      return { due: row?.due ?? 0, oldestDueAgeSeconds: row?.oldest_due_age_seconds ?? 0 };
+    } catch (cause) {
+      try { await client.query('rollback'); }
+      catch (rollbackCause) {
+        // A failed rollback leaves transaction state uncertain. Evict this
+        // client instead of returning it to the shared pool, but retain the
+        // original observation failure for callers and logs.
+        client.release(rollbackCause instanceof Error ? rollbackCause : new Error('outbox observation rollback failed'));
+        released = true;
+      }
+      throw cause;
+    } finally { if (!released) client.release(); }
+  }
   async complete(id:string) { await this.pool.query('update operations_outbox set sent_at=now(),locked_at=null,last_error=null where id=$1',[id]); }
   async fail(id:string,message:string) { await this.pool.query(
     `update operations_outbox set locked_at=null,last_error=$2,available_at=now()+make_interval(secs=>least(3600,power(2,attempts)::int*30)) where id=$1`,[id,message.slice(0,500)]); }

apps/api/src/domains/operations/runner.ts

@@ -1,6 +1,7 @@
 import type { MediaJanitor } from '../media/janitor.js';
 import type { OperationsRepository } from './repository.js';
 import type { DiscordProvider, ExpoPushProvider, MessageClassifier } from './providers.js';
+import { emitTelemetry } from '../../telemetry.js';
 
 export class OperationsRunner {
   constructor(private readonly repository:OperationsRepository,private readonly expo:ExpoPushProvider,private readonly discord:DiscordProvider,private readonly classifier:MessageClassifier,private readonly mediaJanitor?:MediaJanitor) {}
@@ -13,7 +14,11 @@ export class OperationsRunner {
         await this.repository.complete(item.id);completed++;
       } catch(cause) { await this.repository.fail(item.id,cause instanceof Error?cause.message:'unknown error');failed++; }
     }
+    // Observation never blocks useful work. It runs after claiming/completing and
+    // has a short caller-side deadline; the claim lock predicate remains the
+    // source of truth when several worker instances race.
+    const after=await this.repository.outboxState().catch(()=>null);
+    if(after)emitTelemetry({event:'outbox',process:'worker',due_after:after.due,oldest_due_age_seconds:after.oldestDueAgeSeconds,claimed:items.length,completed,failed,count:1});
     return {produced,claimed:items.length,completed,failed};
   }
 }
-