From 85238738313c51aa61cd5fc0fdcb06d2ec03f043 Mon Sep 17 00:00:00 2001 From: jjbrunton Date: Fri, 14 Aug 2026 12:56:00 +0100 Subject: [PATCH] Add conversation image replacement --- .../conversations/conversation-api.ts | 24 ++ .../conversations/conversation-detail.tsx | 2 + .../conversation-image-replacement.tsx | 136 ++++++++ .../conversations/conversation-queries.ts | 16 + .../features/conversations/conversations.css | 20 ++ docs/api.md | 2 + docs/data-flow.md | 4 +- docs/frontend.md | 2 + docs/services.md | 3 +- packages/api-contracts/src/chats.ts | 25 ++ src/services/image-replacement.ts | 301 ++++++++++++++++++ src/web/api-catalog.ts | 13 + src/web/context.ts | 2 + src/web/routes/conversations.ts | 102 ++++++ src/web/server.ts | 6 +- tests/dashboard/conversations.test.tsx | 47 +++ tests/image-replacement.test.ts | 35 ++ tests/web/conversations-route.test.ts | 75 +++++ 18 files changed, 812 insertions(+), 3 deletions(-) create mode 100644 apps/dashboard/src/features/conversations/conversation-image-replacement.tsx create mode 100644 src/services/image-replacement.ts create mode 100644 tests/image-replacement.test.ts diff --git a/apps/dashboard/src/features/conversations/conversation-api.ts b/apps/dashboard/src/features/conversations/conversation-api.ts index 8eaaaa9..6daabdc 100644 --- a/apps/dashboard/src/features/conversations/conversation-api.ts +++ b/apps/dashboard/src/features/conversations/conversation-api.ts @@ -3,6 +3,9 @@ import { ConversationsResponseSchema, ConversationDetailResponseSchema, ConversationMediaShareOptionsResponseSchema, + ConversationImageReplacementOptionsResponseSchema, + GenerateConversationImageReplacementRequestSchema, + GenerateConversationImageReplacementResponseSchema, ErrorResponseSchema, ProcessedPostsResponseSchema, SendReplyRequestSchema, @@ -21,6 +24,8 @@ import { type BulkMarkConversationsReadResponse, type ConversationDetailResponse, type ConversationMediaShareOptions, + type ConversationImageReplacementOptions, + type ConversationImageReplacementResult, type ConversationTriageStatus, type ProcessedPost, type SellerScanResponse, @@ -47,6 +52,8 @@ export interface ConversationApi { suggestReply(id: string): Promise; suggestMediaAsk(id: string): Promise; mediaShareOptions(id: string): Promise; + imageReplacementOptions(id: string): Promise; + generateImageReplacement(id: string, sourceAssetId: string, targetId: string): Promise; processedPosts(username: string): Promise; sendReply(id: string, message: string): Promise; shareMedia(id: string, assetId: string): Promise; @@ -99,6 +106,23 @@ export function createConversationApi(fetchImplementation?: typeof fetch): Conve ConversationMediaShareOptionsResponseSchema, ); }, + async imageReplacementOptions(id) { + return parseResponse( + await request(`${conversationPath(id)}/image-replacement-options`), + ConversationImageReplacementOptionsResponseSchema, + ); + }, + async generateImageReplacement(id, sourceAssetId, targetId) { + const body = GenerateConversationImageReplacementRequestSchema.parse({ sourceAssetId, targetId }); + return parseResponse( + await request(`${conversationPath(id)}/generate-image-replacement`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }), + GenerateConversationImageReplacementResponseSchema, + ); + }, async processedPosts(username) { return parseResponse( await request(`/api/posts/by-author/${encodeURIComponent(username.trim().replace(/^u\//i, ''))}`), diff --git a/apps/dashboard/src/features/conversations/conversation-detail.tsx b/apps/dashboard/src/features/conversations/conversation-detail.tsx index 5dee8e5..02abb4e 100644 --- a/apps/dashboard/src/features/conversations/conversation-detail.tsx +++ b/apps/dashboard/src/features/conversations/conversation-detail.tsx @@ -23,6 +23,7 @@ import { AccountOwnerBadge } from '../../components/account-owner-badge.js'; import { formatDayLabel, formatExactTime, formatTimeOfDay, isSameDay } from './relative-time.js'; import { getStoredComposerDraft, storeComposerDraft } from './composer-drafts.js'; import { SNOOZE_OPTIONS, isUnread, triageStatus } from './conversation-triage.js'; +import { ConversationImageReplacement } from './conversation-image-replacement.js'; /** Reddit chat rejects messages longer than roughly this many characters. */ export const REDDIT_CHAT_MAX_MESSAGE_LENGTH = 10_000; @@ -378,6 +379,7 @@ function ConversationComposer({ conversation, conversationId }: { conversation: {wasSent ? (
Reply sent to
) : null} +
{ diff --git a/apps/dashboard/src/features/conversations/conversation-image-replacement.tsx b/apps/dashboard/src/features/conversations/conversation-image-replacement.tsx new file mode 100644 index 0000000..57a9cde --- /dev/null +++ b/apps/dashboard/src/features/conversations/conversation-image-replacement.tsx @@ -0,0 +1,136 @@ +import { useState } from 'react'; +import { + useConversationImageReplacementOptionsQuery, + useGenerateConversationImageReplacementMutation, +} from './conversation-queries.js'; + +export function ConversationImageReplacement({ conversationId }: { conversationId: string }) { + const [isOpen, setIsOpen] = useState(false); + const [areSourcesRevealed, setAreSourcesRevealed] = useState(false); + const [areTargetsRevealed, setAreTargetsRevealed] = useState(false); + const [isResultRevealed, setIsResultRevealed] = useState(false); + const [sourceAssetId, setSourceAssetId] = useState(null); + const [targetId, setTargetId] = useState(null); + const optionsQuery = useConversationImageReplacementOptionsQuery(conversationId, isOpen); + const generation = useGenerateConversationImageReplacementMutation(conversationId); + const options = optionsQuery.data; + + function clearResult() { + generation.reset(); + setIsResultRevealed(false); + } + + return ( +
+ { + event.preventDefault(); + setIsOpen((current) => !current); + }} + > + Create a green-screen image + +

+ Pick a green-screen source from your exchange library, then place the original post or any conversation image into it. Generation only creates a private preview; it does not send anything. +

+ {optionsQuery.isLoading ? Loading image choices… : null} + {optionsQuery.isError ?
{optionsQuery.error.message}
: null} + {options?.reason ?
{options.reason}
: null} + {options?.sources.length ? ( +
+ 1. Green-screen source + +
+ {options.sources.map((source) => ( + + ))} +
+
+ ) : null} + {options?.targets.length ? ( +
+ 2. Image to place on the green screen + +
+ {options.targets.map((target) => ( + + ))} +
+
+ ) : null} + + {generation.isError ?
{generation.error.message}
: null} + {generation.data ? ( +
+ Generated preview + {generation.data.width} × {generation.data.height} JPEG +
+ Generated green-screen replacement + +
+ Download generated image +
+ ) : null} +
+ ); +} diff --git a/apps/dashboard/src/features/conversations/conversation-queries.ts b/apps/dashboard/src/features/conversations/conversation-queries.ts index 531576a..b5b6ae5 100644 --- a/apps/dashboard/src/features/conversations/conversation-queries.ts +++ b/apps/dashboard/src/features/conversations/conversation-queries.ts @@ -8,6 +8,7 @@ export const conversationQueryKeys = { detail: (id: string) => ['conversations', 'detail', id] as const, processedPosts: (username: string) => ['conversations', 'processed-posts', username] as const, mediaShareOptions: (id: string) => ['conversations', 'media-share-options', id] as const, + imageReplacementOptions: (id: string) => ['conversations', 'image-replacement-options', id] as const, }; export const CONVERSATION_REFRESH_INTERVAL_MS = 30_000; @@ -45,6 +46,21 @@ export function useConversationMediaShareOptionsQuery(id: string, enabled: boole }); } +export function useConversationImageReplacementOptionsQuery(id: string, enabled: boolean) { + return useQuery({ + queryKey: conversationQueryKeys.imageReplacementOptions(id), + queryFn: () => conversationApi.imageReplacementOptions(id), + enabled, + }); +} + +export function useGenerateConversationImageReplacementMutation(id: string) { + return useMutation({ + mutationFn: ({ sourceAssetId, targetId }: { sourceAssetId: string; targetId: string }) => + conversationApi.generateImageReplacement(id, sourceAssetId, targetId), + }); +} + export function useProcessedPostsQuery(username: string) { return useQuery({ queryKey: conversationQueryKeys.processedPosts(username), diff --git a/apps/dashboard/src/features/conversations/conversations.css b/apps/dashboard/src/features/conversations/conversations.css index 95d6823..5f74880 100644 --- a/apps/dashboard/src/features/conversations/conversations.css +++ b/apps/dashboard/src/features/conversations/conversations.css @@ -158,6 +158,26 @@ .conversation-inline-error { color: #ffd1ce; background: #461b18; border: 1px solid #80352f; } .conversation-inline-success { color: #b9f3ce; background: #163a26; border: 1px solid #28593d; } .conversation-media-share { display: grid; gap: .7rem; padding: .75rem; background: #0b1410; border: 1px solid var(--border); border-radius: .65rem; } +.conversation-image-replacement { display: grid; gap: .7rem; padding: .75rem; background: #0b1410; border: 1px solid var(--border); border-radius: .65rem; } +.conversation-image-replacement[open] > * + * { margin-top: .7rem; } +.conversation-image-replacement summary { color: #e9f1ec; font-weight: 800; cursor: pointer; } +.conversation-image-replacement > p { margin: 0; color: var(--muted); font-size: .8rem; } +.conversation-image-replacement fieldset { display: grid; gap: .55rem; min-width: 0; padding: .65rem; border: 1px solid var(--border); border-radius: .55rem; } +.conversation-image-replacement legend { padding: 0 .3rem; color: #e9f1ec; font-size: .8rem; font-weight: 800; } +.conversation-image-replacement fieldset > button { justify-self: start; } +.conversation-replacement-options { display: grid; grid-template-columns: repeat(auto-fit, minmax(8rem, 1fr)); gap: .65rem; } +.conversation-replacement-options label { display: grid; gap: .35rem; padding: .35rem; overflow: hidden; border: 1px solid var(--border); border-radius: .55rem; cursor: pointer; } +.conversation-replacement-options label.is-selected { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); } +.conversation-replacement-options label:focus-within { outline: 2px solid var(--accent); outline-offset: 2px; } +.conversation-replacement-options input { position: absolute; opacity: 0; pointer-events: none; } +.conversation-replacement-options img { display: block; width: 100%; height: 8rem; object-fit: contain; background: #050907; border-radius: .35rem; transition: filter 160ms ease, transform 160ms ease; } +.conversation-replacement-options img.is-blurred { filter: blur(22px) brightness(.55); transform: scale(1.1); } +.conversation-replacement-options span { color: var(--muted); font-size: .7rem; } +.conversation-replacement-result { display: grid; gap: .45rem; padding: .65rem; color: var(--muted); background: var(--surface-raised); border-radius: .55rem; } +.conversation-replacement-result > strong { color: var(--text); } +.conversation-replacement-result img { display: block; width: 100%; max-height: 55vh; object-fit: contain; background: #050907; border-radius: .55rem; transition: filter 160ms ease; } +.conversation-replacement-result img.is-blurred { filter: blur(24px) brightness(.55); } +.conversation-replacement-result > a { justify-self: start; color: var(--accent); font-weight: 800; } .conversation-media-share[open] > * + * { margin-top: .7rem; } .conversation-media-share summary { color: #e9f1ec; font-weight: 800; cursor: pointer; } .conversation-media-share p, .conversation-media-unavailable { margin: 0; color: var(--muted); font-size: .8rem; } diff --git a/docs/api.md b/docs/api.md index b925ff3..b9df579 100644 --- a/docs/api.md +++ b/docs/api.md @@ -82,6 +82,8 @@ an administrator credential. | `POST` | `/api/conversations/{id}/send-reply` | Send a conversation reply | admin | yes | | `GET` | `/api/conversations/{id}/shareable-media` | List every enabled exchange-library image not yet shared in this conversation | read | no | | `POST` | `/api/conversations/{id}/share-media` | Send a selected exchange-library image | admin | yes | +| `GET` | `/api/conversations/{id}/image-replacement-options` | List green-screen sources and conversation-owned replacement targets | read | no | +| `POST` | `/api/conversations/{id}/generate-image-replacement` | Composite a selected conversation image into a green-screen source | admin | yes | | `POST` | `/api/conversations/start` | Start a new conversation | admin | yes | | `POST` | `/api/conversations/{id}/read` | Advance the operator read watermark | admin | yes | | `POST` | `/api/conversations/{id}/unread` | Return a conversation to the unread queue | admin | yes | diff --git a/docs/data-flow.md b/docs/data-flow.md index aca1027..b06b248 100644 --- a/docs/data-flow.md +++ b/docs/data-flow.md @@ -23,7 +23,7 @@ stateDiagram-v2 ### 1. Discovery -The discovery loop rotates through enabled subreddits and authenticated runtimes. A runtime fetches Reddit `/new` data using the process-shared `RedditRequestCoordinator`. In the default `post-author` targeting mode those posts enter intake directly. An image subreddit configured for `comment-author` instead reads top-level comments from at most five newly listed posts, batches previously unseen eligible comment text through the reaction classifier, and creates candidates only for role-alignment verdicts at or above 0.8 confidence. A bounded in-process cache prevents unchanged comments from repeatedly consuming model calls. The candidate ID, author, timestamp, and permalink come from the comment while the original post image and title remain attached as preparation context; the usual image screen later verifies the depicted gender against `reddit.targetGenders`. Intake then deduplicates candidate IDs and authors, excludes processed/contacted history, stores the full snapshot, source type, weight, and timestamps, and logs exact intake counts. +The discovery loop rotates through enabled subreddits and authenticated runtimes. A runtime fetches Reddit `/new` data using the process-shared `RedditRequestCoordinator`. In the default `post-author` targeting mode those posts enter intake directly. An image subreddit configured for `comment-author` instead reads top-level comments from at most five newly listed posts and batches previously unseen eligible comment text through the reaction classifier. Candidates require a role-alignment verdict at or above 0.8 confidence; when `reddit.targetGenders` is non-empty, the expressed gender perspective must also be included. An `unknown` perspective fails closed unless explicitly targeted. A bounded in-process cache prevents unchanged comments from repeatedly consuming model calls. The candidate ID, author, timestamp, and permalink come from the comment while the original post image and title remain attached as preparation context; the usual image screen later verifies the depicted gender against the same target list. Intake then deduplicates candidate IDs and authors, excludes processed/contacted history, stores the full snapshot, source type, weight, and timestamps, and logs exact intake counts. The queue grows to a demand-based target rather than a fixed backlog. The target is the smaller of configured `reddit.maxReadyCandidates` and the work that authenticated running accounts can still send within the recent-post window, accounting for rolling hourly capacity and minimum message spacing. Intake admits only the current deficit, newest Reddit posts first, pauses at zero outbound capacity, and reports fetched posts left unevaluated because the funnel is full. Reddit request pacing, not a tight application poll, remains the primary request limiter. @@ -130,6 +130,8 @@ After the normal reply limiter waits, automation refreshes Matrix and rechecks t The conversation UI also supports an explicitly confirmed manual choice from this same eligible set. Its read endpoint reports request-matched options without exposing filesystem paths. The send endpoint refreshes the conversation inside the owning account's rate limiter, rechecks the latest inbound request and safety boundaries, reserves the selected asset in the same durable ledger, and uses the same Matrix image-plus-reciprocal delivery path. A successful manual share marks automation as user-overridden. +Green-screen replacement is a generation-only conversation flow. The operator selects a source already held in the consent-controlled exchange library and a target identified by the server as either the triggering post image or a downloaded attachment in that conversation. `ImageReplacementService` detects the largest green area at bounded resolution, derives its perspective quadrilateral, cover-fits and warps the target, composites through the actual green pixels so foreground occlusions remain, and carries a bounded amount of the source lighting into the insert. The JPEG is returned as a private data-URL preview; it is not persisted, classified, added to the share ledger, or sent automatically. + Conversation performance is derived from the complete attributed inbound transcript. Engagement cannot become a prompt-learning example while automation is active, and a later decline, boundary, or upsell removes an earlier example before future prompts load it. ## Dashboard data and events diff --git a/docs/frontend.md b/docs/frontend.md index 374dcfe..0a720cd 100644 --- a/docs/frontend.md +++ b/docs/frontend.md @@ -78,6 +78,8 @@ Exchange-library uploads require an explicit consent checkbox before the raw ima Conversation reply composers include a collapsed saved-image picker. It lists only server-approved assets for the latest inbound request, keeps thumbnails blurred until revealed, and requires selection plus a dedicated confirmation dialog before sending. The server remains authoritative and may reject a choice if the conversation changes before the rate-limited send executes. +The composer also has a separate green-screen creation panel. Its source picker uses consent-controlled exchange-library assets; its target picker is derived server-side from the triggering post image and the attachments already loaded into that conversation. Source, target, and generated preview are blurred independently. Generation returns an authenticated in-memory JPEG preview/download and never adds it to the library or sends it to Matrix, so delivery remains a separate explicit operator action. + The conversation inbox assigns every row to exactly one primary workflow queue: **Needs attention**, **Waiting**, **Sellers**, **Snoozed**, **Done**, **Deleted**, or **Archived**. Removed-account history belongs only to Archived, deleted recipients belong only to Deleted, and seller/off-platform outcomes belong only to Sellers, so none can leak into actionable work. Unread state, manual ownership, and received images are independent badges and secondary toggles rather than competing queues. The default queue is Needs attention. Suspended profiles are not treated as deleted. ## Feature structure diff --git a/docs/services.md b/docs/services.md index 50f9eb8..b84ac31 100644 --- a/docs/services.md +++ b/docs/services.md @@ -20,7 +20,7 @@ This is an ownership map, not an API dump. Read the owning implementation and te | Module | Owns | Important constraints | | --- | --- | --- | -| `services/monitor.ts` | Subreddit ordering, post/comment candidate fetching, reaction/age/gender/seller screening, rejection summaries | Comment reactions fail closed at the confidence and target-gender gates; cheap checks run first; preserve exact rejection/deferred counts; listing 404 disables the subreddit. | +| `services/monitor.ts` | Subreddit ordering, post/comment candidate fetching, reaction/age/gender/seller screening, rejection summaries | Comment reactions fail closed at the confidence gate and, when configured, the expressed-gender gate; cheap checks run first; preserve exact rejection/deferred counts; listing 404 disables the subreddit. | | `reddit/api.ts` | Reddit JSON calls, post/comment parsing, seller/profile evidence, username resolution, error types | All reads must use the coordinator; redact error metadata. | | `reddit/request-coordinator.ts` | Process-wide Reddit request spacing and cooldown | Space failed requests too; 429 cooldown is global. | | `services/processor.ts` | Source-specific AI preparation and outbound safety | Returns prepared or explicit skip; performs no remote chat send. | @@ -70,6 +70,7 @@ Verification is the safety mechanism that matters most. Posted comments are swep | `services/media-sync.ts` | Periodic authenticated media sync and operator-visible completion status | Stop while auth is unavailable and on runtime shutdown; do not report partial work as success. | | `services/media-library.ts` | Shared operator uploads, image validation/re-encoding, classification state, and private files | Require consent confirmation; received media never enters this store. | | `services/media-library-exchange.ts` | Inbound request detection, progressive selection, reservation, and image-first exchange | Never offer proactively; exact tags do not substitute; no asset reuse per conversation. | +| `services/image-replacement.ts` | Green-screen detection, perspective fitting, compositing, and private preview generation | Sources come from the consent-controlled library; targets resolve from authoritative conversation data; generation never sends. | | `database/media-library-repository.ts` | Shared asset metadata and durable account/conversation send audit | Reserve before remote send; uncertain sends stay consumed. | ## Persistence diff --git a/packages/api-contracts/src/chats.ts b/packages/api-contracts/src/chats.ts index 01278a1..25554ca 100644 --- a/packages/api-contracts/src/chats.ts +++ b/packages/api-contracts/src/chats.ts @@ -147,6 +147,28 @@ export const ConversationMediaShareOptionsResponseSchema = z.strictObject({ }); export const ShareConversationMediaRequestSchema = z.strictObject({ assetId: z.string().uuid() }); +export const ConversationImageReplacementTargetSchema = z.strictObject({ + id: z.string().min(1), + label: z.string().min(1), + urlPath: z.string().min(1), +}); +export const ConversationImageReplacementOptionsResponseSchema = z.strictObject({ + sources: z.array(MediaLibraryAssetSchema), + targets: z.array(ConversationImageReplacementTargetSchema), + reason: z.string().nullable(), +}); +export const GenerateConversationImageReplacementRequestSchema = z.strictObject({ + sourceAssetId: z.string().uuid(), + targetId: z.string().min(1), +}); +export const GenerateConversationImageReplacementResponseSchema = z.strictObject({ + imageDataUrl: z.string().startsWith('data:image/jpeg;base64,'), + mimeType: z.literal('image/jpeg'), + fileName: z.string().min(1), + width: z.number().int().positive(), + height: z.number().int().positive(), +}); + export const MarkConversationReadRequestSchema = z.strictObject({ /** Defaults to the conversation's newest message when omitted. */ throughTimestamp: z.number().int().nonnegative().optional(), @@ -224,3 +246,6 @@ export type SellerScanResponse = z.infer; export type ChatSyncStatus = z.infer; export type ConversationDetailResponse = z.infer; export type ConversationMediaShareOptions = z.infer; +export type ConversationImageReplacementTarget = z.infer; +export type ConversationImageReplacementOptions = z.infer; +export type ConversationImageReplacementResult = z.infer; diff --git a/src/services/image-replacement.ts b/src/services/image-replacement.ts new file mode 100644 index 0000000..f5a0222 --- /dev/null +++ b/src/services/image-replacement.ts @@ -0,0 +1,301 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import sharp from 'sharp'; +import { downloadImage } from '../utils/image.js'; + +const MAX_INPUT_PIXELS = 25_000_000; +const DETECTION_MAX_DIMENSION = 900; + +export interface ImageReplacementResult { + bytes: Buffer; + mimeType: 'image/jpeg'; + width: number; + height: number; +} + +export type ImageReplacementTarget = + | { kind: 'conversation-media'; localPath: string } + | { kind: 'remote'; url: string; userAgent: string }; + +export interface ImageReplacementGenerator { + generate(sourcePath: string, target: ImageReplacementTarget): Promise; +} + +export class ImageReplacementInputError extends Error {} + +type RawImage = { data: Buffer; width: number; height: number; channels: 3 | 4 }; +type Point = { x: number; y: number }; + +/** + * Ports the green-screen replacement prototype into the Node runtime. The + * source is an operator-owned exchange-library image; the replacement is a + * conversation image selected from authoritative server data. + */ +export class GreenScreenImageReplacementService implements ImageReplacementGenerator { + constructor(private readonly mediaRoot = path.join(process.cwd(), 'data', 'media')) {} + + async generate(sourcePath: string, target: ImageReplacementTarget): Promise { + const sourceBytes = await fs.readFile(sourcePath); + const targetBytes = target.kind === 'remote' + ? (await downloadImage(target.url, target.userAgent)).buffer + : await fs.readFile(resolveInside(this.mediaRoot, target.localPath)); + return replaceGreenScreen(sourceBytes, targetBytes); + } +} + +export async function replaceGreenScreen(sourceBytes: Buffer, targetBytes: Buffer): Promise { + const [source, detection] = await Promise.all([ + decode(sourceBytes), + decode(sourceBytes, DETECTION_MAX_DIMENSION), + ]).catch((error: unknown) => { + throw new ImageReplacementInputError(`Unable to read the selected image: ${safeError(error)}`); + }); + + const smallMask = greenMask(detection); + const component = largestComponent(smallMask, detection.width, detection.height); + if (component.length < Math.max(64, Math.round(detection.width * detection.height * 0.002))) { + throw new ImageReplacementInputError('No usable green-screen region was found in the source image'); + } + const smallQuad = componentExtremes(component, detection.width); + const scaleX = source.width / detection.width; + const scaleY = source.height / detection.height; + const quad = smallQuad.map(({ x, y }) => ({ x: x * scaleX, y: y * scaleY })); + const screenWidth = Math.max(1, Math.round(Math.max(distance(quad[0], quad[1]), distance(quad[3], quad[2])))); + const screenHeight = Math.max(1, Math.round(Math.max(distance(quad[0], quad[3]), distance(quad[1], quad[2])))); + const fitted = await fitCover(targetBytes, screenWidth, screenHeight).catch((error: unknown) => { + throw new ImageReplacementInputError(`Unable to read the selected replacement image: ${safeError(error)}`); + }); + const homography = solveHomography(quad, screenWidth, screenHeight); + + const rawMask = greenMask(source); + const clippedMask = Buffer.alloc(source.width * source.height); + let luminanceTotal = 0; + let maskedPixels = 0; + for (let y = 0; y < source.height; y++) { + for (let x = 0; x < source.width; x++) { + const index = y * source.width + x; + if (rawMask[index] && pointInQuad(x, y, quad)) { + clippedMask[index] = 255; + const offset = index * source.channels; + luminanceTotal += luminance(source.data[offset], source.data[offset + 1], source.data[offset + 2]); + maskedPixels++; + } + } + } + if (maskedPixels === 0) throw new ImageReplacementInputError('The green-screen region could not be composited'); + // Expand by one pixel before feathering so antialiased green edge pixels do + // not survive as a bright fringe around the insert. + const expandedMask = dilateMask(clippedMask, source.width, source.height); + const featheredMask = await sharp(expandedMask, { raw: { width: source.width, height: source.height, channels: 1 } }) + .blur(1.5) + .greyscale() + .raw() + .toBuffer(); + const referenceLuminance = luminanceTotal / maskedPixels; + const output = Buffer.from(source.data); + const bounds = quadBounds(quad, source.width, source.height); + + for (let y = bounds.top; y <= bounds.bottom; y++) { + for (let x = bounds.left; x <= bounds.right; x++) { + const pixel = y * source.width + x; + const alpha = featheredMask[pixel] / 255; + if (alpha <= 0.002) continue; + const mapped = mapPoint(homography, x, y); + if (mapped.x < 0 || mapped.y < 0 || mapped.x >= screenWidth || mapped.y >= screenHeight) continue; + const sampled = bilinearSample(fitted, mapped.x, mapped.y); + const offset = pixel * source.channels; + const sourceLuminance = luminance(source.data[offset], source.data[offset + 1], source.data[offset + 2]); + const gain = clamp(1 + 0.35 * (sourceLuminance / Math.max(referenceLuminance, 1) - 1), 0.55, 1.65); + for (let channel = 0; channel < 3; channel++) { + const replacement = clamp(sampled[channel] * gain, 0, 255); + output[offset + channel] = Math.round(source.data[offset + channel] * (1 - alpha) + replacement * alpha); + } + } + } + + const bytes = await sharp(output, { + raw: { width: source.width, height: source.height, channels: source.channels }, + }).jpeg({ quality: 92, mozjpeg: true }).toBuffer(); + return { bytes, mimeType: 'image/jpeg', width: source.width, height: source.height }; +} + +async function decode(bytes: Buffer, maxDimension?: number): Promise { + let pipeline = sharp(bytes, { limitInputPixels: MAX_INPUT_PIXELS, animated: false }).rotate().removeAlpha().toColourspace('srgb'); + if (maxDimension) pipeline = pipeline.resize({ width: maxDimension, height: maxDimension, fit: 'inside', withoutEnlargement: true }); + const decoded = await pipeline.raw().toBuffer({ resolveWithObject: true }); + if (decoded.info.channels < 3) throw new ImageReplacementInputError('Selected images must contain RGB colour data'); + return { data: decoded.data, width: decoded.info.width, height: decoded.info.height, channels: decoded.info.channels as 3 | 4 }; +} + +async function fitCover(bytes: Buffer, width: number, height: number): Promise { + const decoded = await sharp(bytes, { limitInputPixels: MAX_INPUT_PIXELS, animated: false }) + .rotate().removeAlpha().toColourspace('srgb') + .resize(width, height, { fit: 'cover', position: 'centre' }) + .raw().toBuffer({ resolveWithObject: true }); + return { data: decoded.data, width, height, channels: decoded.info.channels as 3 | 4 }; +} + +function greenMask(image: RawImage): Uint8Array { + const mask = new Uint8Array(image.width * image.height); + for (let pixel = 0; pixel < mask.length; pixel++) { + const offset = pixel * image.channels; + const r = image.data[offset] / 255; + const g = image.data[offset + 1] / 255; + const b = image.data[offset + 2] / 255; + const max = Math.max(r, g, b); + const min = Math.min(r, g, b); + const saturation = max === 0 ? 0 : (max - min) / max; + let hue = 0; + if (max !== min) { + if (max === r) hue = 60 * (((g - b) / (max - min)) % 6); + else if (max === g) hue = 60 * ((b - r) / (max - min) + 2); + else hue = 60 * ((r - g) / (max - min) + 4); + if (hue < 0) hue += 360; + } + if (hue >= 70 && hue <= 180 && saturation >= 70 / 255 && max >= 50 / 255) mask[pixel] = 1; + } + return mask; +} + +function largestComponent(mask: Uint8Array, width: number, height: number): number[] { + const seen = new Uint8Array(mask.length); + const queue = new Int32Array(mask.length); + let largest: number[] = []; + for (let start = 0; start < mask.length; start++) { + if (!mask[start] || seen[start]) continue; + let head = 0; + let tail = 0; + queue[tail++] = start; + seen[start] = 1; + const component: number[] = []; + while (head < tail) { + const pixel = queue[head++]; + component.push(pixel); + const x = pixel % width; + const y = Math.floor(pixel / width); + const neighbours = [pixel - width, pixel + width, pixel - 1, pixel + 1]; + for (let index = 0; index < neighbours.length; index++) { + const neighbour = neighbours[index]; + if (neighbour < 0 || neighbour >= mask.length) continue; + if (index === 2 && x === 0) continue; + if (index === 3 && x === width - 1) continue; + if ((index === 0 && y === 0) || (index === 1 && y === height - 1)) continue; + if (mask[neighbour] && !seen[neighbour]) { + seen[neighbour] = 1; + queue[tail++] = neighbour; + } + } + } + if (component.length > largest.length) largest = component; + } + return largest; +} + +function dilateMask(mask: Buffer, width: number, height: number): Buffer { + const expanded = Buffer.from(mask); + for (let y = 1; y < height - 1; y++) { + for (let x = 1; x < width - 1; x++) { + const pixel = y * width + x; + if (!mask[pixel]) continue; + for (let offsetY = -1; offsetY <= 1; offsetY++) { + for (let offsetX = -1; offsetX <= 1; offsetX++) expanded[pixel + offsetY * width + offsetX] = 255; + } + } + } + return expanded; +} + +function componentExtremes(component: number[], width: number): [Point, Point, Point, Point] { + const points = component.map((pixel) => ({ x: pixel % width, y: Math.floor(pixel / width) })); + return [ + points.reduce((best, point) => point.x + point.y < best.x + best.y ? point : best), + points.reduce((best, point) => point.x - point.y > best.x - best.y ? point : best), + points.reduce((best, point) => point.x + point.y > best.x + best.y ? point : best), + points.reduce((best, point) => point.y - point.x > best.y - best.x ? point : best), + ]; +} + +function solveHomography(quad: Point[], width: number, height: number): number[] { + const destinations = [{ x: 0, y: 0 }, { x: width - 1, y: 0 }, { x: width - 1, y: height - 1 }, { x: 0, y: height - 1 }]; + const matrix: number[][] = []; + for (let index = 0; index < 4; index++) { + const { x, y } = quad[index]; + const { x: u, y: v } = destinations[index]; + matrix.push([x, y, 1, 0, 0, 0, -u * x, -u * y, u]); + matrix.push([0, 0, 0, x, y, 1, -v * x, -v * y, v]); + } + for (let column = 0; column < 8; column++) { + let pivot = column; + for (let row = column + 1; row < 8; row++) if (Math.abs(matrix[row][column]) > Math.abs(matrix[pivot][column])) pivot = row; + [matrix[column], matrix[pivot]] = [matrix[pivot], matrix[column]]; + const divisor = matrix[column][column]; + if (Math.abs(divisor) < 1e-9) throw new ImageReplacementInputError('The detected green-screen shape is not usable'); + for (let cell = column; cell < 9; cell++) matrix[column][cell] /= divisor; + for (let row = 0; row < 8; row++) { + if (row === column) continue; + const factor = matrix[row][column]; + for (let cell = column; cell < 9; cell++) matrix[row][cell] -= factor * matrix[column][cell]; + } + } + return matrix.map((row) => row[8]); +} + +function mapPoint(h: number[], x: number, y: number): Point { + const denominator = h[6] * x + h[7] * y + 1; + return { x: (h[0] * x + h[1] * y + h[2]) / denominator, y: (h[3] * x + h[4] * y + h[5]) / denominator }; +} + +function bilinearSample(image: RawImage, x: number, y: number): [number, number, number] { + const x0 = Math.floor(x); + const y0 = Math.floor(y); + const x1 = Math.min(x0 + 1, image.width - 1); + const y1 = Math.min(y0 + 1, image.height - 1); + const dx = x - x0; + const dy = y - y0; + const sample = (sx: number, sy: number, channel: number) => image.data[(sy * image.width + sx) * image.channels + channel]; + return [0, 1, 2].map((channel) => ( + sample(x0, y0, channel) * (1 - dx) * (1 - dy) + + sample(x1, y0, channel) * dx * (1 - dy) + + sample(x0, y1, channel) * (1 - dx) * dy + + sample(x1, y1, channel) * dx * dy + )) as [number, number, number]; +} + +function pointInQuad(x: number, y: number, quad: Point[]): boolean { + let sign = 0; + for (let index = 0; index < 4; index++) { + const a = quad[index]; + const b = quad[(index + 1) % 4]; + const cross = (b.x - a.x) * (y - a.y) - (b.y - a.y) * (x - a.x); + if (Math.abs(cross) < 1e-6) continue; + const nextSign = Math.sign(cross); + if (sign !== 0 && nextSign !== sign) return false; + sign = nextSign; + } + return true; +} + +function quadBounds(quad: Point[], width: number, height: number) { + const xs = quad.map((point) => point.x); + const ys = quad.map((point) => point.y); + return { + left: clamp(Math.floor(Math.min(...xs)) - 3, 0, width - 1), + right: clamp(Math.ceil(Math.max(...xs)) + 3, 0, width - 1), + top: clamp(Math.floor(Math.min(...ys)) - 3, 0, height - 1), + bottom: clamp(Math.ceil(Math.max(...ys)) + 3, 0, height - 1), + }; +} + +function resolveInside(root: string, relativePath: string): string { + const resolvedRoot = path.resolve(root); + const resolved = path.resolve(resolvedRoot, relativePath); + if (resolved === resolvedRoot || !resolved.startsWith(`${resolvedRoot}${path.sep}`)) { + throw new ImageReplacementInputError('Unsafe conversation media path'); + } + return resolved; +} + +function distance(a: Point, b: Point): number { return Math.hypot(a.x - b.x, a.y - b.y); } +function luminance(r: number, g: number, b: number): number { return 0.2126 * r + 0.7152 * g + 0.0722 * b; } +function clamp(value: number, minimum: number, maximum: number): number { return Math.min(maximum, Math.max(minimum, value)); } +function safeError(error: unknown): string { return error instanceof Error ? error.message : String(error); } diff --git a/src/web/api-catalog.ts b/src/web/api-catalog.ts index 8725a0d..3e9fc8b 100644 --- a/src/web/api-catalog.ts +++ b/src/web/api-catalog.ts @@ -107,6 +107,19 @@ export const API_OPERATIONS: readonly ApiOperation[] = [ operation('sendReply', 'POST', '/api/conversations/{id}/send-reply', 'Send a conversation reply', ['Conversations']), operation('getConversationMediaOptions', 'GET', '/api/conversations/{id}/shareable-media', 'List every enabled exchange-library image not yet shared in this conversation', ['Conversations']), operation('shareConversationMedia', 'POST', '/api/conversations/{id}/share-media', 'Send a selected exchange-library image', ['Conversations']), + operation('getConversationImageReplacementOptions', 'GET', '/api/conversations/{id}/image-replacement-options', 'List green-screen sources and conversation-owned replacement targets', ['Conversations'], { + responseSchema: z.object({ sources: z.array(z.unknown()), targets: z.array(z.unknown()), reason: z.string().nullable() }), + }), + operation('generateConversationImageReplacement', 'POST', '/api/conversations/{id}/generate-image-replacement', 'Composite a selected conversation image into a green-screen source', ['Conversations'], { + requestSchema: z.strictObject({ sourceAssetId: z.string().uuid(), targetId: z.string().min(1) }), + responseSchema: z.strictObject({ + imageDataUrl: z.string().startsWith('data:image/jpeg;base64,'), + mimeType: z.literal('image/jpeg'), + fileName: z.string(), + width: z.number().int().positive(), + height: z.number().int().positive(), + }), + }), operation('startConversation', 'POST', '/api/conversations/start', 'Start a new conversation', ['Conversations']), operation('markConversationRead', 'POST', '/api/conversations/{id}/read', 'Advance the operator read watermark', ['Conversations'], { requestSchema: z.strictObject({ throughTimestamp: z.number().int().nonnegative().optional() }), diff --git a/src/web/context.ts b/src/web/context.ts index df0456d..7ba5ca2 100644 --- a/src/web/context.ts +++ b/src/web/context.ts @@ -31,6 +31,7 @@ import type { AccountGrowthRepository } from '../database/growth-repository.js'; import type { AccountBrowsingSimulationRepository } from '../database/browsing-simulation-repository.js'; import type { PushNotifier } from '../services/push-notifier.js'; import type { BotMode } from '../services/bot-mode.js'; +import type { ImageReplacementGenerator } from '../services/image-replacement.js'; /** Bag of everything the route modules might need. */ export interface WebContext { @@ -69,6 +70,7 @@ export interface WebContext { browsing?: AccountBrowsingSimulationRepository; mediaLibrary?: MediaLibraryService; pushNotifier?: PushNotifier; + imageReplacement?: ImageReplacementGenerator; }; /** Operator's run/scan/stop intent, read by the main loop. */ diff --git a/src/web/routes/conversations.ts b/src/web/routes/conversations.ts index cb90876..b147b70 100644 --- a/src/web/routes/conversations.ts +++ b/src/web/routes/conversations.ts @@ -23,8 +23,13 @@ import type { ConversationReadStateRepository, } from '../../database/repositories.js'; import type { RedditRecipientAccountStatusRecord } from '../../database/account-repositories.js'; +import { ImageReplacementInputError, type ImageReplacementTarget } from '../../services/image-replacement.js'; const ShareConversationMediaRequestSchema = z.strictObject({ assetId: z.string().uuid() }); +const GenerateConversationImageReplacementRequestSchema = z.strictObject({ + sourceAssetId: z.string().uuid(), + targetId: z.string().min(1), +}); const MarkReadRequestSchema = z.strictObject({ throughTimestamp: z.number().int().nonnegative().optional(), }); @@ -259,6 +264,42 @@ export function createConversationsRouter(ctx: WebContext): Router { }; } + function triggeringImageUrl( + conversation: ActiveChat, + runtime: NonNullable['get']>> | null, + ): string | null { + const messagesRepo = runtime?.resources.messagesRepo ?? ctx.repos.messages; + const sent = conversation.roomId + ? messagesRepo.getByChatRoomId(conversation.roomId) ?? messagesRepo.getLatestByRecipient(conversation.username) + : messagesRepo.getLatestByRecipient(conversation.username); + return sent?.postImageUrl ?? null; + } + + function replacementTargets(conversation: ActiveChat, originalPostImageUrl: string | null) { + const targets: Array<{ id: string; label: string; urlPath: string; target: ImageReplacementTarget }> = []; + let conversationImageNumber = 0; + if (originalPostImageUrl) { + targets.push({ + id: 'original-post', + label: 'Original post image', + urlPath: originalPostImageUrl, + target: { kind: 'remote', url: originalPostImageUrl, userAgent: ctx.config.reddit.userAgent }, + }); + } + (conversation.messages ?? []).forEach((message, messageIndex) => { + (message.media ?? []).forEach((media, mediaIndex) => { + conversationImageNumber++; + targets.push({ + id: `message-${messageIndex}-${mediaIndex}`, + label: `${message.isSelf ? 'Your' : `u/${conversation.username}'s`} conversation image ${conversationImageNumber}`, + urlPath: `/media/${media.localPath.split(/[\\/]/).map(encodeURIComponent).join('/')}`, + target: { kind: 'conversation-media', localPath: media.localPath }, + }); + }); + }); + return targets; + } + router.get('/conversations', async (req, res) => { const chatService = ctx.services.chat; if (!chatService) { @@ -1007,6 +1048,67 @@ export function createConversationsRouter(ctx: WebContext): Router { } }); + router.get('/conversations/:id/image-replacement-options', async (req, res) => { + try { + const resolved = await findConversation(decodeURIComponent(req.params.id)); + if (!resolved) return res.status(404).json({ error: 'Conversation not found' }); + const library = ctx.services.mediaLibrary; + if (!library || !ctx.services.imageReplacement) { + return res.status(503).json({ error: 'Image replacement is not initialized' }); + } + const targets = replacementTargets( + resolved.conversation, + triggeringImageUrl(resolved.conversation, resolved.runtime), + ); + const sources = library.list().map(toMediaLibraryResponse); + const reason = sources.length === 0 + ? 'Upload a green-screen source image to the exchange library first.' + : targets.length === 0 + ? 'This conversation has no original-post or conversation images to use as the replacement.' + : null; + return res.json({ + sources, + targets: targets.map(({ target: _target, ...publicTarget }) => publicTarget), + reason, + }); + } catch (error) { + ctx.logger.error('Failed to list image replacement options:', error); + return res.status(500).json({ error: 'Failed to list image replacement options' }); + } + }); + + router.post('/conversations/:id/generate-image-replacement', async (req, res) => { + const parsed = GenerateConversationImageReplacementRequestSchema.safeParse(req.body); + if (!parsed.success) return res.status(400).json({ error: 'Select a valid source and target image' }); + try { + const resolved = await findConversation(decodeURIComponent(req.params.id)); + if (!resolved) return res.status(404).json({ error: 'Conversation not found' }); + const library = ctx.services.mediaLibrary; + const generator = ctx.services.imageReplacement; + if (!library || !generator) return res.status(503).json({ error: 'Image replacement is not initialized' }); + const source = library.get(parsed.data.sourceAssetId); + if (!source) return res.status(404).json({ error: 'Source image not found in the exchange library' }); + const targets = replacementTargets( + resolved.conversation, + triggeringImageUrl(resolved.conversation, resolved.runtime), + ); + const selectedTarget = targets.find((target) => target.id === parsed.data.targetId); + if (!selectedTarget) return res.status(409).json({ error: 'The selected target image is no longer available' }); + const result = await generator.generate(library.resolvePath(source), selectedTarget.target); + return res.json({ + imageDataUrl: `data:${result.mimeType};base64,${result.bytes.toString('base64')}`, + mimeType: result.mimeType, + fileName: `replacement-${resolved.conversation.username.replace(/[^a-zA-Z0-9_-]/g, '_')}.jpg`, + width: result.width, + height: result.height, + }); + } catch (error) { + if (error instanceof ImageReplacementInputError) return res.status(400).json({ error: error.message }); + ctx.logger.error('Failed to generate conversation image replacement:', error); + return res.status(500).json({ error: 'Failed to generate image replacement' }); + } + }); + /** * Start a brand-new DM with a Reddit user and send the opening message. * Body: { username: string, message: string } diff --git a/src/web/server.ts b/src/web/server.ts index 4dc165f..2880e1b 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -56,6 +56,7 @@ import type { AccountBrowsingSimulationRepository } from '../database/browsing-s import { createGrowthRouter } from './routes/growth.js'; import { createLearningRouter } from './routes/learning.js'; import { allowsSending, type BotMode } from '../services/bot-mode.js'; +import { GreenScreenImageReplacementService } from '../services/image-replacement.js'; export type { BotStats, ActivityItem, LogEntry, RuntimeConfig } from './types.js'; @@ -77,7 +78,7 @@ export class WebServer { private runtimeConfig: RuntimeConfig; private onConfigChange: ((config: RuntimeConfig) => void) | null = null; - private services: WebContext['services'] = {}; + private services: WebContext['services']; constructor( private port: number, @@ -94,6 +95,9 @@ export class WebServer { private readonly persistBotMode: (mode: BotMode) => void = () => undefined, private readonly accountContentRepo?: AccountContentRepository, ) { + this.services = { + imageReplacement: new GreenScreenImageReplacementService(path.join(this.workspaceDirectory, 'data', 'media')), + }; this.botMode = initialBotMode; this.broadcaster = broadcaster ?? new LogBroadcaster(); this.activity = new ActivityLog(activityLogRepo); diff --git a/tests/dashboard/conversations.test.tsx b/tests/dashboard/conversations.test.tsx index dda1d04..7adc75b 100644 --- a/tests/dashboard/conversations.test.tsx +++ b/tests/dashboard/conversations.test.tsx @@ -468,6 +468,53 @@ describe('ConversationsPage', () => { expect(screen.getByText('Share a saved image')).toHaveFocus(); }); + it('creates a blurred green-screen preview from a saved source and conversation target', async () => { + installConversationList(); + const asset = { + id: '11111111-1111-4111-8111-111111111111', tags: ['selfie'], safety: 'sfw', + classificationStatus: 'ready', description: 'green phone', confidence: 1, + fileName: 'green-phone.jpg', mimeType: 'image/jpeg', sizeBytes: 1024, width: 800, height: 1000, + enabled: true, consentConfirmedAt: 1, createdAt: 1, updatedAt: 1, + sendCount: 0, lastSentAt: null, urlPath: '/library-media/green-phone.jpg', + }; + server.use( + http.get('/api/conversations/room-alice/image-replacement-options', () => HttpResponse.json({ + sources: [asset], + targets: [ + { id: 'original-post', label: 'Original post image', urlPath: 'https://i.redd.it/source-post.jpg' }, + { id: 'message-1-0', label: "u/alice's conversation image 2", urlPath: '/media/alice/image.jpg' }, + ], + reason: null, + })), + http.post('/api/conversations/room-alice/generate-image-replacement', async ({ request }) => { + expect(await request.json()).toEqual({ sourceAssetId: asset.id, targetId: 'original-post' }); + return HttpResponse.json({ + imageDataUrl: 'data:image/jpeg;base64,Z2VuZXJhdGVk', mimeType: 'image/jpeg', + fileName: 'replacement-alice.jpg', width: 800, height: 1000, + }); + }), + ); + const user = userEvent.setup(); + renderConversations(); + + await user.click(await screen.findByRole('button', { name: /u\/alice/ })); + await user.click(await screen.findByText('Create a green-screen image')); + expect(await screen.findByRole('img', { name: 'Green-screen source green-phone.jpg' })).toHaveClass('is-blurred'); + await user.click(screen.getByRole('button', { name: 'Reveal source images' })); + await user.click(screen.getByRole('radio', { name: /green-phone\.jpg/ })); + await user.click(screen.getByRole('button', { name: 'Reveal conversation choices' })); + await user.click(screen.getByRole('radio', { name: 'Original post image' })); + await user.click(screen.getByRole('button', { name: 'Create image' })); + + const generated = await screen.findByRole('img', { name: 'Generated green-screen replacement' }); + expect(generated).toHaveClass('is-blurred'); + await user.click(screen.getByRole('button', { name: 'Reveal generated image' })); + expect(generated).not.toHaveClass('is-blurred'); + expect(screen.getByRole('link', { name: 'Download generated image' })).toHaveAttribute( + 'download', 'replacement-alice.jpg', + ); + }); + it('shows unused turns separately from a safety handoff', async () => { server.use( http.get('/api/conversations', () => HttpResponse.json([alice])), diff --git a/tests/image-replacement.test.ts b/tests/image-replacement.test.ts new file mode 100644 index 0000000..6a29726 --- /dev/null +++ b/tests/image-replacement.test.ts @@ -0,0 +1,35 @@ +import sharp from 'sharp'; +import { describe, expect, it } from 'vitest'; +import { ImageReplacementInputError, replaceGreenScreen } from '../src/services/image-replacement.js'; + +describe('replaceGreenScreen', () => { + it('warps a replacement into the detected green region and preserves the surrounding source', async () => { + const source = await sharp({ + create: { width: 200, height: 160, channels: 3, background: '#20252a' }, + }).composite([{ input: Buffer.from( + '', + ) }]).png().toBuffer(); + const target = await sharp({ + create: { width: 80, height: 120, channels: 3, background: '#ee2233' }, + }).png().toBuffer(); + + const result = await replaceGreenScreen(source, target); + const decoded = await sharp(result.bytes).raw().toBuffer({ resolveWithObject: true }); + const pixel = (x: number, y: number) => { + const offset = (y * decoded.info.width + x) * decoded.info.channels; + return [...decoded.data.subarray(offset, offset + 3)]; + }; + + expect(result).toMatchObject({ mimeType: 'image/jpeg', width: 200, height: 160 }); + expect(pixel(100, 80)[0]).toBeGreaterThan(150); + expect(pixel(100, 80)[1]).toBeLessThan(100); + expect(pixel(10, 10)[0]).toBeLessThan(70); + }); + + it('rejects a source without a usable green-screen area', async () => { + const source = await sharp({ create: { width: 100, height: 100, channels: 3, background: '#223344' } }).png().toBuffer(); + const target = await sharp({ create: { width: 20, height: 20, channels: 3, background: '#ffffff' } }).png().toBuffer(); + + await expect(replaceGreenScreen(source, target)).rejects.toThrow(ImageReplacementInputError); + }); +}); diff --git a/tests/web/conversations-route.test.ts b/tests/web/conversations-route.test.ts index 1cc3102..98c317b 100644 --- a/tests/web/conversations-route.test.ts +++ b/tests/web/conversations-route.test.ts @@ -37,6 +37,81 @@ afterEach(async () => { }); describe('conversations route', () => { + it('lists authoritative replacement targets and generates a private preview', async () => { + const accountId = '@t2_one:reddit.com'; + const assetId = '11111111-1111-4111-8111-111111111111'; + const conversation = { + username: 'alice', roomName: 'alice', roomId: 'room-one', url: '/room/room-one', + lastMessageTime: 2, lastMessageSnippet: '[Image]', isFromUs: false, unread: true, + messages: [{ + sender: 'alice', text: '[Image]', timestamp: 2, isSelf: false, + media: [{ originalUrl: 'mxc://reddit.com/one', localPath: 'account/one.jpg', fileName: 'one.jpg' }], + }], + }; + const asset = { + id: assetId, tags: ['selfie'], safety: 'sfw', classificationStatus: 'ready', description: 'phone template', confidence: 1, + fileName: 'green-phone.jpg', relativePath: 'green-phone.jpg', mimeType: 'image/jpeg', sizeBytes: 4, + width: 10, height: 10, enabled: true, consentConfirmedAt: 1, createdAt: 1, updatedAt: 1, + sendCount: 0, lastSentAt: null, + }; + const imageReplacement = { + generate: vi.fn().mockResolvedValue({ bytes: Buffer.from('generated'), mimeType: 'image/jpeg', width: 100, height: 200 }), + }; + const runtime = { + snapshot: () => ({ state: 'running' }), + resources: { + chatService: { getCachedConversations: vi.fn().mockResolvedValue([conversation]) }, + messagesRepo: { + getByChatRoomId: vi.fn().mockReturnValue({ postImageUrl: 'https://i.redd.it/original.jpg' }), + getLatestByRecipient: vi.fn(), + }, + }, + }; + const context = { + config: { reddit: { userAgent: 'test-agent' } }, + logger: { error: vi.fn() }, + repos: { messages: {} }, + services: { + chat: {}, imageReplacement, + mediaLibrary: { + list: vi.fn().mockReturnValue([asset]), + get: vi.fn().mockReturnValue(asset), + resolvePath: vi.fn().mockReturnValue('/private/green-phone.jpg'), + }, + accountRuntimes: { + list: () => [{ accountId, redditUsername: 'owner-one', state: 'running' }], + get: (requested: string) => requested === accountId ? runtime : null, + }, + }, + } as unknown as WebContext; + const app = express().use(express.json()).use('/api', createConversationsRouter(context)); + const ownedId = encodeURIComponent(`${encodeURIComponent(accountId)}~room-one`); + + const optionsResponse = await request(app, `/api/conversations/${ownedId}/image-replacement-options`); + expect(optionsResponse.status).toBe(200); + expect(await optionsResponse.json()).toMatchObject({ + reason: null, + sources: [{ id: assetId, urlPath: '/library-media/green-phone.jpg' }], + targets: [ + { id: 'original-post', label: 'Original post image', urlPath: 'https://i.redd.it/original.jpg' }, + { id: 'message-0-0', urlPath: '/media/account/one.jpg' }, + ], + }); + + const generateResponse = await request(app, `/api/conversations/${ownedId}/generate-image-replacement`, { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ sourceAssetId: assetId, targetId: 'message-0-0' }), + }); + expect(generateResponse.status).toBe(200); + expect(await generateResponse.json()).toEqual({ + imageDataUrl: `data:image/jpeg;base64,${Buffer.from('generated').toString('base64')}`, + mimeType: 'image/jpeg', fileName: 'replacement-alice.jpg', width: 100, height: 200, + }); + expect(imageReplacement.generate).toHaveBeenCalledWith('/private/green-phone.jpg', { + kind: 'conversation-media', localPath: 'account/one.jpg', + }); + }); + it('aggregates simultaneous account runtimes and labels identical room IDs with their owner', async () => { const chat = (username: string) => ({ username, roomName: username, roomId: 'same-room', lastMessageTime: 1,