Add conversation image replacement
This commit is contained in:
1 parent
2500f0a23c
commit
8523873831
18 files changed
+812
-3
No files matched your search
@@ -3,6 +3,9 @@ import {
|
||||
ConversationsResponseSchema,
|
||||
ConversationDetailResponseSchema,
|
||||
ConversationMediaShareOptionsResponseSchema,
|
||||
ConversationImageReplacementOptionsResponseSchema,
|
||||
GenerateConversationImageReplacementRequestSchema,
|
||||
GenerateConversationImageReplacementResponseSchema,
|
||||
ErrorResponseSchema,
|
||||
ProcessedPostsResponseSchema,
|
||||
SendReplyRequestSchema,
|
||||
@@ -21,6 +24,8 @@ import {
|
||||
type BulkMarkConversationsReadResponse,
|
||||
type ConversationDetailResponse,
|
||||
type ConversationMediaShareOptions,
|
||||
type ConversationImageReplacementOptions,
|
||||
type ConversationImageReplacementResult,
|
||||
type ConversationTriageStatus,
|
||||
type ProcessedPost,
|
||||
type SellerScanResponse,
|
||||
@@ -47,6 +52,8 @@ export interface ConversationApi {
|
||||
suggestReply(id: string): Promise<string>;
|
||||
suggestMediaAsk(id: string): Promise<string>;
|
||||
mediaShareOptions(id: string): Promise<ConversationMediaShareOptions>;
|
||||
imageReplacementOptions(id: string): Promise<ConversationImageReplacementOptions>;
|
||||
generateImageReplacement(id: string, sourceAssetId: string, targetId: string): Promise<ConversationImageReplacementResult>;
|
||||
processedPosts(username: string): Promise<ProcessedPost[]>;
|
||||
sendReply(id: string, message: string): Promise<SuccessResponse>;
|
||||
shareMedia(id: string, assetId: string): Promise<SuccessResponse>;
|
||||
@@ -99,6 +106,23 @@ export function createConversationApi(fetchImplementation?: typeof fetch): Conve
|
||||
ConversationMediaShareOptionsResponseSchema,
|
||||
);
|
||||
},
|
||||
async imageReplacementOptions(id) {
|
||||
return parseResponse(
|
||||
await request(`${conversationPath(id)}/image-replacement-options`),
|
||||
ConversationImageReplacementOptionsResponseSchema,
|
||||
);
|
||||
},
|
||||
async generateImageReplacement(id, sourceAssetId, targetId) {
|
||||
const body = GenerateConversationImageReplacementRequestSchema.parse({ sourceAssetId, targetId });
|
||||
return parseResponse(
|
||||
await request(`${conversationPath(id)}/generate-image-replacement`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
}),
|
||||
GenerateConversationImageReplacementResponseSchema,
|
||||
);
|
||||
},
|
||||
async processedPosts(username) {
|
||||
return parseResponse(
|
||||
await request(`/api/posts/by-author/${encodeURIComponent(username.trim().replace(/^u\//i, ''))}`),
|
||||
|
||||
@@ -23,6 +23,7 @@ import { AccountOwnerBadge } from '../../components/account-owner-badge.js';
|
||||
import { formatDayLabel, formatExactTime, formatTimeOfDay, isSameDay } from './relative-time.js';
|
||||
import { getStoredComposerDraft, storeComposerDraft } from './composer-drafts.js';
|
||||
import { SNOOZE_OPTIONS, isUnread, triageStatus } from './conversation-triage.js';
|
||||
import { ConversationImageReplacement } from './conversation-image-replacement.js';
|
||||
|
||||
/** Reddit chat rejects messages longer than roughly this many characters. */
|
||||
export const REDDIT_CHAT_MAX_MESSAGE_LENGTH = 10_000;
|
||||
@@ -378,6 +379,7 @@ function ConversationComposer({ conversation, conversationId }: { conversation:
|
||||
{wasSent ? (
|
||||
<div aria-label="Reply sent" className="conversation-inline-success" role="status">Reply sent to <RedditUsernameLink username={conversation.username} /></div>
|
||||
) : null}
|
||||
<ConversationImageReplacement conversationId={conversationId} />
|
||||
<details className="conversation-media-share" open={isMediaShareOpen}>
|
||||
<summary
|
||||
onClick={(event) => {
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { useState } from 'react';
|
||||
import {
|
||||
useConversationImageReplacementOptionsQuery,
|
||||
useGenerateConversationImageReplacementMutation,
|
||||
} from './conversation-queries.js';
|
||||
|
||||
export function ConversationImageReplacement({ conversationId }: { conversationId: string }) {
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const [areSourcesRevealed, setAreSourcesRevealed] = useState(false);
|
||||
const [areTargetsRevealed, setAreTargetsRevealed] = useState(false);
|
||||
const [isResultRevealed, setIsResultRevealed] = useState(false);
|
||||
const [sourceAssetId, setSourceAssetId] = useState<string | null>(null);
|
||||
const [targetId, setTargetId] = useState<string | null>(null);
|
||||
const optionsQuery = useConversationImageReplacementOptionsQuery(conversationId, isOpen);
|
||||
const generation = useGenerateConversationImageReplacementMutation(conversationId);
|
||||
const options = optionsQuery.data;
|
||||
|
||||
function clearResult() {
|
||||
generation.reset();
|
||||
setIsResultRevealed(false);
|
||||
}
|
||||
|
||||
return (
|
||||
<details className="conversation-image-replacement" open={isOpen}>
|
||||
<summary
|
||||
onClick={(event) => {
|
||||
event.preventDefault();
|
||||
setIsOpen((current) => !current);
|
||||
}}
|
||||
>
|
||||
Create a green-screen image
|
||||
</summary>
|
||||
<p>
|
||||
Pick a green-screen source from your exchange library, then place the original post or any conversation image into it. Generation only creates a private preview; it does not send anything.
|
||||
</p>
|
||||
{optionsQuery.isLoading ? <span role="status">Loading image choices…</span> : null}
|
||||
{optionsQuery.isError ? <div className="conversation-inline-error" role="alert">{optionsQuery.error.message}</div> : null}
|
||||
{options?.reason ? <div className="conversation-media-unavailable">{options.reason}</div> : null}
|
||||
{options?.sources.length ? (
|
||||
<fieldset>
|
||||
<legend>1. Green-screen source</legend>
|
||||
<button onClick={() => setAreSourcesRevealed((current) => !current)} type="button">
|
||||
{areSourcesRevealed ? 'Blur source images' : 'Reveal source images'}
|
||||
</button>
|
||||
<div className="conversation-replacement-options">
|
||||
{options.sources.map((source) => (
|
||||
<label className={sourceAssetId === source.id ? 'is-selected' : ''} key={source.id}>
|
||||
<input
|
||||
aria-label={`Green-screen source ${source.fileName}`}
|
||||
checked={sourceAssetId === source.id}
|
||||
disabled={!areSourcesRevealed}
|
||||
name="replacement-source"
|
||||
onChange={() => {
|
||||
setSourceAssetId(source.id);
|
||||
clearResult();
|
||||
}}
|
||||
type="radio"
|
||||
/>
|
||||
<img
|
||||
alt={`Green-screen source ${source.fileName}`}
|
||||
className={areSourcesRevealed ? undefined : 'is-blurred'}
|
||||
src={source.urlPath}
|
||||
/>
|
||||
<span>{source.fileName}</span>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</fieldset>
|
||||
) : null}
|
||||
{options?.targets.length ? (
|
||||
<fieldset>
|
||||
<legend>2. Image to place on the green screen</legend>
|
||||
<button onClick={() => setAreTargetsRevealed((current) => !current)} type="button">
|
||||
{areTargetsRevealed ? 'Blur conversation choices' : 'Reveal conversation choices'}
|
||||
</button>
|
||||
<div className="conversation-replacement-options">
|
||||
{options.targets.map((target) => (
|
||||
<label className={targetId === target.id ? 'is-selected' : ''} key={target.id}>
|
||||
<input
|
||||
aria-label={target.label}
|
||||
checked={targetId === target.id}
|
||||
disabled={!areTargetsRevealed}
|
||||
name="replacement-target"
|
||||
onChange={() => {
|
||||
setTargetId(target.id);
|
||||
clearResult();
|
||||
}}
|
||||
type="radio"
|
||||
/>
|
||||
<img
|
||||
alt={target.label}
|
||||
className={areTargetsRevealed ? undefined : 'is-blurred'}
|
||||
src={target.urlPath}
|
||||
/>
|
||||
<span>{target.label}</span>
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
</fieldset>
|
||||
) : null}
|
||||
<button
|
||||
className="conversation-primary"
|
||||
disabled={!sourceAssetId || !targetId || generation.isPending}
|
||||
onClick={() => {
|
||||
if (sourceAssetId && targetId) generation.mutate({ sourceAssetId, targetId });
|
||||
}}
|
||||
type="button"
|
||||
>
|
||||
{generation.isPending ? 'Creating image…' : 'Create image'}
|
||||
</button>
|
||||
{generation.isError ? <div className="conversation-inline-error" role="alert">{generation.error.message}</div> : null}
|
||||
{generation.data ? (
|
||||
<div className="conversation-replacement-result" role="status">
|
||||
<strong>Generated preview</strong>
|
||||
<span>{generation.data.width} × {generation.data.height} JPEG</span>
|
||||
<div className="conversation-message-image">
|
||||
<img
|
||||
alt="Generated green-screen replacement"
|
||||
className={isResultRevealed ? undefined : 'is-blurred'}
|
||||
src={generation.data.imageDataUrl}
|
||||
/>
|
||||
<button
|
||||
aria-pressed={isResultRevealed}
|
||||
className="conversation-image-reveal"
|
||||
onClick={() => setIsResultRevealed((current) => !current)}
|
||||
type="button"
|
||||
>
|
||||
{isResultRevealed ? 'Blur generated image' : 'Reveal generated image'}
|
||||
</button>
|
||||
</div>
|
||||
<a download={generation.data.fileName} href={generation.data.imageDataUrl}>Download generated image</a>
|
||||
</div>
|
||||
) : null}
|
||||
</details>
|
||||
);
|
||||
}
|
||||
@@ -8,6 +8,7 @@ export const conversationQueryKeys = {
|
||||
detail: (id: string) => ['conversations', 'detail', id] as const,
|
||||
processedPosts: (username: string) => ['conversations', 'processed-posts', username] as const,
|
||||
mediaShareOptions: (id: string) => ['conversations', 'media-share-options', id] as const,
|
||||
imageReplacementOptions: (id: string) => ['conversations', 'image-replacement-options', id] as const,
|
||||
};
|
||||
|
||||
export const CONVERSATION_REFRESH_INTERVAL_MS = 30_000;
|
||||
@@ -45,6 +46,21 @@ export function useConversationMediaShareOptionsQuery(id: string, enabled: boole
|
||||
});
|
||||
}
|
||||
|
||||
export function useConversationImageReplacementOptionsQuery(id: string, enabled: boolean) {
|
||||
return useQuery({
|
||||
queryKey: conversationQueryKeys.imageReplacementOptions(id),
|
||||
queryFn: () => conversationApi.imageReplacementOptions(id),
|
||||
enabled,
|
||||
});
|
||||
}
|
||||
|
||||
export function useGenerateConversationImageReplacementMutation(id: string) {
|
||||
return useMutation({
|
||||
mutationFn: ({ sourceAssetId, targetId }: { sourceAssetId: string; targetId: string }) =>
|
||||
conversationApi.generateImageReplacement(id, sourceAssetId, targetId),
|
||||
});
|
||||
}
|
||||
|
||||
export function useProcessedPostsQuery(username: string) {
|
||||
return useQuery({
|
||||
queryKey: conversationQueryKeys.processedPosts(username),
|
||||
|
||||
@@ -158,6 +158,26 @@
|
||||
.conversation-inline-error { color: #ffd1ce; background: #461b18; border: 1px solid #80352f; }
|
||||
.conversation-inline-success { color: #b9f3ce; background: #163a26; border: 1px solid #28593d; }
|
||||
.conversation-media-share { display: grid; gap: .7rem; padding: .75rem; background: #0b1410; border: 1px solid var(--border); border-radius: .65rem; }
|
||||
.conversation-image-replacement { display: grid; gap: .7rem; padding: .75rem; background: #0b1410; border: 1px solid var(--border); border-radius: .65rem; }
|
||||
.conversation-image-replacement[open] > * + * { margin-top: .7rem; }
|
||||
.conversation-image-replacement summary { color: #e9f1ec; font-weight: 800; cursor: pointer; }
|
||||
.conversation-image-replacement > p { margin: 0; color: var(--muted); font-size: .8rem; }
|
||||
.conversation-image-replacement fieldset { display: grid; gap: .55rem; min-width: 0; padding: .65rem; border: 1px solid var(--border); border-radius: .55rem; }
|
||||
.conversation-image-replacement legend { padding: 0 .3rem; color: #e9f1ec; font-size: .8rem; font-weight: 800; }
|
||||
.conversation-image-replacement fieldset > button { justify-self: start; }
|
||||
.conversation-replacement-options { display: grid; grid-template-columns: repeat(auto-fit, minmax(8rem, 1fr)); gap: .65rem; }
|
||||
.conversation-replacement-options label { display: grid; gap: .35rem; padding: .35rem; overflow: hidden; border: 1px solid var(--border); border-radius: .55rem; cursor: pointer; }
|
||||
.conversation-replacement-options label.is-selected { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); }
|
||||
.conversation-replacement-options label:focus-within { outline: 2px solid var(--accent); outline-offset: 2px; }
|
||||
.conversation-replacement-options input { position: absolute; opacity: 0; pointer-events: none; }
|
||||
.conversation-replacement-options img { display: block; width: 100%; height: 8rem; object-fit: contain; background: #050907; border-radius: .35rem; transition: filter 160ms ease, transform 160ms ease; }
|
||||
.conversation-replacement-options img.is-blurred { filter: blur(22px) brightness(.55); transform: scale(1.1); }
|
||||
.conversation-replacement-options span { color: var(--muted); font-size: .7rem; }
|
||||
.conversation-replacement-result { display: grid; gap: .45rem; padding: .65rem; color: var(--muted); background: var(--surface-raised); border-radius: .55rem; }
|
||||
.conversation-replacement-result > strong { color: var(--text); }
|
||||
.conversation-replacement-result img { display: block; width: 100%; max-height: 55vh; object-fit: contain; background: #050907; border-radius: .55rem; transition: filter 160ms ease; }
|
||||
.conversation-replacement-result img.is-blurred { filter: blur(24px) brightness(.55); }
|
||||
.conversation-replacement-result > a { justify-self: start; color: var(--accent); font-weight: 800; }
|
||||
.conversation-media-share[open] > * + * { margin-top: .7rem; }
|
||||
.conversation-media-share summary { color: #e9f1ec; font-weight: 800; cursor: pointer; }
|
||||
.conversation-media-share p, .conversation-media-unavailable { margin: 0; color: var(--muted); font-size: .8rem; }
|
||||
|
||||
@@ -82,6 +82,8 @@ an administrator credential.
|
||||
| `POST` | `/api/conversations/{id}/send-reply` | Send a conversation reply | admin | yes |
|
||||
| `GET` | `/api/conversations/{id}/shareable-media` | List every enabled exchange-library image not yet shared in this conversation | read | no |
|
||||
| `POST` | `/api/conversations/{id}/share-media` | Send a selected exchange-library image | admin | yes |
|
||||
| `GET` | `/api/conversations/{id}/image-replacement-options` | List green-screen sources and conversation-owned replacement targets | read | no |
|
||||
| `POST` | `/api/conversations/{id}/generate-image-replacement` | Composite a selected conversation image into a green-screen source | admin | yes |
|
||||
| `POST` | `/api/conversations/start` | Start a new conversation | admin | yes |
|
||||
| `POST` | `/api/conversations/{id}/read` | Advance the operator read watermark | admin | yes |
|
||||
| `POST` | `/api/conversations/{id}/unread` | Return a conversation to the unread queue | admin | yes |
|
||||
|
||||
+3
-1
@@ -23,7 +23,7 @@ stateDiagram-v2
|
||||
|
||||
### 1. Discovery
|
||||
|
||||
The discovery loop rotates through enabled subreddits and authenticated runtimes. A runtime fetches Reddit `/new` data using the process-shared `RedditRequestCoordinator`. In the default `post-author` targeting mode those posts enter intake directly. An image subreddit configured for `comment-author` instead reads top-level comments from at most five newly listed posts, batches previously unseen eligible comment text through the reaction classifier, and creates candidates only for role-alignment verdicts at or above 0.8 confidence. A bounded in-process cache prevents unchanged comments from repeatedly consuming model calls. The candidate ID, author, timestamp, and permalink come from the comment while the original post image and title remain attached as preparation context; the usual image screen later verifies the depicted gender against `reddit.targetGenders`. Intake then deduplicates candidate IDs and authors, excludes processed/contacted history, stores the full snapshot, source type, weight, and timestamps, and logs exact intake counts.
|
||||
The discovery loop rotates through enabled subreddits and authenticated runtimes. A runtime fetches Reddit `/new` data using the process-shared `RedditRequestCoordinator`. In the default `post-author` targeting mode those posts enter intake directly. An image subreddit configured for `comment-author` instead reads top-level comments from at most five newly listed posts and batches previously unseen eligible comment text through the reaction classifier. Candidates require a role-alignment verdict at or above 0.8 confidence; when `reddit.targetGenders` is non-empty, the expressed gender perspective must also be included. An `unknown` perspective fails closed unless explicitly targeted. A bounded in-process cache prevents unchanged comments from repeatedly consuming model calls. The candidate ID, author, timestamp, and permalink come from the comment while the original post image and title remain attached as preparation context; the usual image screen later verifies the depicted gender against the same target list. Intake then deduplicates candidate IDs and authors, excludes processed/contacted history, stores the full snapshot, source type, weight, and timestamps, and logs exact intake counts.
|
||||
|
||||
The queue grows to a demand-based target rather than a fixed backlog. The target is the smaller of configured `reddit.maxReadyCandidates` and the work that authenticated running accounts can still send within the recent-post window, accounting for rolling hourly capacity and minimum message spacing. Intake admits only the current deficit, newest Reddit posts first, pauses at zero outbound capacity, and reports fetched posts left unevaluated because the funnel is full. Reddit request pacing, not a tight application poll, remains the primary request limiter.
|
||||
|
||||
@@ -130,6 +130,8 @@ After the normal reply limiter waits, automation refreshes Matrix and rechecks t
|
||||
|
||||
The conversation UI also supports an explicitly confirmed manual choice from this same eligible set. Its read endpoint reports request-matched options without exposing filesystem paths. The send endpoint refreshes the conversation inside the owning account's rate limiter, rechecks the latest inbound request and safety boundaries, reserves the selected asset in the same durable ledger, and uses the same Matrix image-plus-reciprocal delivery path. A successful manual share marks automation as user-overridden.
|
||||
|
||||
Green-screen replacement is a generation-only conversation flow. The operator selects a source already held in the consent-controlled exchange library and a target identified by the server as either the triggering post image or a downloaded attachment in that conversation. `ImageReplacementService` detects the largest green area at bounded resolution, derives its perspective quadrilateral, cover-fits and warps the target, composites through the actual green pixels so foreground occlusions remain, and carries a bounded amount of the source lighting into the insert. The JPEG is returned as a private data-URL preview; it is not persisted, classified, added to the share ledger, or sent automatically.
|
||||
|
||||
Conversation performance is derived from the complete attributed inbound transcript. Engagement cannot become a prompt-learning example while automation is active, and a later decline, boundary, or upsell removes an earlier example before future prompts load it.
|
||||
|
||||
## Dashboard data and events
|
||||
|
||||
@@ -78,6 +78,8 @@ Exchange-library uploads require an explicit consent checkbox before the raw ima
|
||||
|
||||
Conversation reply composers include a collapsed saved-image picker. It lists only server-approved assets for the latest inbound request, keeps thumbnails blurred until revealed, and requires selection plus a dedicated confirmation dialog before sending. The server remains authoritative and may reject a choice if the conversation changes before the rate-limited send executes.
|
||||
|
||||
The composer also has a separate green-screen creation panel. Its source picker uses consent-controlled exchange-library assets; its target picker is derived server-side from the triggering post image and the attachments already loaded into that conversation. Source, target, and generated preview are blurred independently. Generation returns an authenticated in-memory JPEG preview/download and never adds it to the library or sends it to Matrix, so delivery remains a separate explicit operator action.
|
||||
|
||||
The conversation inbox assigns every row to exactly one primary workflow queue: **Needs attention**, **Waiting**, **Sellers**, **Snoozed**, **Done**, **Deleted**, or **Archived**. Removed-account history belongs only to Archived, deleted recipients belong only to Deleted, and seller/off-platform outcomes belong only to Sellers, so none can leak into actionable work. Unread state, manual ownership, and received images are independent badges and secondary toggles rather than competing queues. The default queue is Needs attention. Suspended profiles are not treated as deleted.
|
||||
|
||||
## Feature structure
|
||||
|
||||
+2
-1
@@ -20,7 +20,7 @@ This is an ownership map, not an API dump. Read the owning implementation and te
|
||||
|
||||
| Module | Owns | Important constraints |
|
||||
| --- | --- | --- |
|
||||
| `services/monitor.ts` | Subreddit ordering, post/comment candidate fetching, reaction/age/gender/seller screening, rejection summaries | Comment reactions fail closed at the confidence and target-gender gates; cheap checks run first; preserve exact rejection/deferred counts; listing 404 disables the subreddit. |
|
||||
| `services/monitor.ts` | Subreddit ordering, post/comment candidate fetching, reaction/age/gender/seller screening, rejection summaries | Comment reactions fail closed at the confidence gate and, when configured, the expressed-gender gate; cheap checks run first; preserve exact rejection/deferred counts; listing 404 disables the subreddit. |
|
||||
| `reddit/api.ts` | Reddit JSON calls, post/comment parsing, seller/profile evidence, username resolution, error types | All reads must use the coordinator; redact error metadata. |
|
||||
| `reddit/request-coordinator.ts` | Process-wide Reddit request spacing and cooldown | Space failed requests too; 429 cooldown is global. |
|
||||
| `services/processor.ts` | Source-specific AI preparation and outbound safety | Returns prepared or explicit skip; performs no remote chat send. |
|
||||
@@ -70,6 +70,7 @@ Verification is the safety mechanism that matters most. Posted comments are swep
|
||||
| `services/media-sync.ts` | Periodic authenticated media sync and operator-visible completion status | Stop while auth is unavailable and on runtime shutdown; do not report partial work as success. |
|
||||
| `services/media-library.ts` | Shared operator uploads, image validation/re-encoding, classification state, and private files | Require consent confirmation; received media never enters this store. |
|
||||
| `services/media-library-exchange.ts` | Inbound request detection, progressive selection, reservation, and image-first exchange | Never offer proactively; exact tags do not substitute; no asset reuse per conversation. |
|
||||
| `services/image-replacement.ts` | Green-screen detection, perspective fitting, compositing, and private preview generation | Sources come from the consent-controlled library; targets resolve from authoritative conversation data; generation never sends. |
|
||||
| `database/media-library-repository.ts` | Shared asset metadata and durable account/conversation send audit | Reserve before remote send; uncertain sends stay consumed. |
|
||||
|
||||
## Persistence
|
||||
|
||||
@@ -147,6 +147,28 @@ export const ConversationMediaShareOptionsResponseSchema = z.strictObject({
|
||||
});
|
||||
export const ShareConversationMediaRequestSchema = z.strictObject({ assetId: z.string().uuid() });
|
||||
|
||||
export const ConversationImageReplacementTargetSchema = z.strictObject({
|
||||
id: z.string().min(1),
|
||||
label: z.string().min(1),
|
||||
urlPath: z.string().min(1),
|
||||
});
|
||||
export const ConversationImageReplacementOptionsResponseSchema = z.strictObject({
|
||||
sources: z.array(MediaLibraryAssetSchema),
|
||||
targets: z.array(ConversationImageReplacementTargetSchema),
|
||||
reason: z.string().nullable(),
|
||||
});
|
||||
export const GenerateConversationImageReplacementRequestSchema = z.strictObject({
|
||||
sourceAssetId: z.string().uuid(),
|
||||
targetId: z.string().min(1),
|
||||
});
|
||||
export const GenerateConversationImageReplacementResponseSchema = z.strictObject({
|
||||
imageDataUrl: z.string().startsWith('data:image/jpeg;base64,'),
|
||||
mimeType: z.literal('image/jpeg'),
|
||||
fileName: z.string().min(1),
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
});
|
||||
|
||||
export const MarkConversationReadRequestSchema = z.strictObject({
|
||||
/** Defaults to the conversation's newest message when omitted. */
|
||||
throughTimestamp: z.number().int().nonnegative().optional(),
|
||||
@@ -224,3 +246,6 @@ export type SellerScanResponse = z.infer<typeof SellerScanResponseSchema>;
|
||||
export type ChatSyncStatus = z.infer<typeof ChatSyncStatusSchema>;
|
||||
export type ConversationDetailResponse = z.infer<typeof ConversationDetailResponseSchema>;
|
||||
export type ConversationMediaShareOptions = z.infer<typeof ConversationMediaShareOptionsResponseSchema>;
|
||||
export type ConversationImageReplacementTarget = z.infer<typeof ConversationImageReplacementTargetSchema>;
|
||||
export type ConversationImageReplacementOptions = z.infer<typeof ConversationImageReplacementOptionsResponseSchema>;
|
||||
export type ConversationImageReplacementResult = z.infer<typeof GenerateConversationImageReplacementResponseSchema>;
|
||||
@@ -0,0 +1,301 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import sharp from 'sharp';
|
||||
import { downloadImage } from '../utils/image.js';
|
||||
|
||||
const MAX_INPUT_PIXELS = 25_000_000;
|
||||
const DETECTION_MAX_DIMENSION = 900;
|
||||
|
||||
export interface ImageReplacementResult {
|
||||
bytes: Buffer;
|
||||
mimeType: 'image/jpeg';
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export type ImageReplacementTarget =
|
||||
| { kind: 'conversation-media'; localPath: string }
|
||||
| { kind: 'remote'; url: string; userAgent: string };
|
||||
|
||||
export interface ImageReplacementGenerator {
|
||||
generate(sourcePath: string, target: ImageReplacementTarget): Promise<ImageReplacementResult>;
|
||||
}
|
||||
|
||||
export class ImageReplacementInputError extends Error {}
|
||||
|
||||
type RawImage = { data: Buffer; width: number; height: number; channels: 3 | 4 };
|
||||
type Point = { x: number; y: number };
|
||||
|
||||
/**
|
||||
* Ports the green-screen replacement prototype into the Node runtime. The
|
||||
* source is an operator-owned exchange-library image; the replacement is a
|
||||
* conversation image selected from authoritative server data.
|
||||
*/
|
||||
export class GreenScreenImageReplacementService implements ImageReplacementGenerator {
|
||||
constructor(private readonly mediaRoot = path.join(process.cwd(), 'data', 'media')) {}
|
||||
|
||||
async generate(sourcePath: string, target: ImageReplacementTarget): Promise<ImageReplacementResult> {
|
||||
const sourceBytes = await fs.readFile(sourcePath);
|
||||
const targetBytes = target.kind === 'remote'
|
||||
? (await downloadImage(target.url, target.userAgent)).buffer
|
||||
: await fs.readFile(resolveInside(this.mediaRoot, target.localPath));
|
||||
return replaceGreenScreen(sourceBytes, targetBytes);
|
||||
}
|
||||
}
|
||||
|
||||
export async function replaceGreenScreen(sourceBytes: Buffer, targetBytes: Buffer): Promise<ImageReplacementResult> {
|
||||
const [source, detection] = await Promise.all([
|
||||
decode(sourceBytes),
|
||||
decode(sourceBytes, DETECTION_MAX_DIMENSION),
|
||||
]).catch((error: unknown) => {
|
||||
throw new ImageReplacementInputError(`Unable to read the selected image: ${safeError(error)}`);
|
||||
});
|
||||
|
||||
const smallMask = greenMask(detection);
|
||||
const component = largestComponent(smallMask, detection.width, detection.height);
|
||||
if (component.length < Math.max(64, Math.round(detection.width * detection.height * 0.002))) {
|
||||
throw new ImageReplacementInputError('No usable green-screen region was found in the source image');
|
||||
}
|
||||
const smallQuad = componentExtremes(component, detection.width);
|
||||
const scaleX = source.width / detection.width;
|
||||
const scaleY = source.height / detection.height;
|
||||
const quad = smallQuad.map(({ x, y }) => ({ x: x * scaleX, y: y * scaleY }));
|
||||
const screenWidth = Math.max(1, Math.round(Math.max(distance(quad[0], quad[1]), distance(quad[3], quad[2]))));
|
||||
const screenHeight = Math.max(1, Math.round(Math.max(distance(quad[0], quad[3]), distance(quad[1], quad[2]))));
|
||||
const fitted = await fitCover(targetBytes, screenWidth, screenHeight).catch((error: unknown) => {
|
||||
throw new ImageReplacementInputError(`Unable to read the selected replacement image: ${safeError(error)}`);
|
||||
});
|
||||
const homography = solveHomography(quad, screenWidth, screenHeight);
|
||||
|
||||
const rawMask = greenMask(source);
|
||||
const clippedMask = Buffer.alloc(source.width * source.height);
|
||||
let luminanceTotal = 0;
|
||||
let maskedPixels = 0;
|
||||
for (let y = 0; y < source.height; y++) {
|
||||
for (let x = 0; x < source.width; x++) {
|
||||
const index = y * source.width + x;
|
||||
if (rawMask[index] && pointInQuad(x, y, quad)) {
|
||||
clippedMask[index] = 255;
|
||||
const offset = index * source.channels;
|
||||
luminanceTotal += luminance(source.data[offset], source.data[offset + 1], source.data[offset + 2]);
|
||||
maskedPixels++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (maskedPixels === 0) throw new ImageReplacementInputError('The green-screen region could not be composited');
|
||||
// Expand by one pixel before feathering so antialiased green edge pixels do
|
||||
// not survive as a bright fringe around the insert.
|
||||
const expandedMask = dilateMask(clippedMask, source.width, source.height);
|
||||
const featheredMask = await sharp(expandedMask, { raw: { width: source.width, height: source.height, channels: 1 } })
|
||||
.blur(1.5)
|
||||
.greyscale()
|
||||
.raw()
|
||||
.toBuffer();
|
||||
const referenceLuminance = luminanceTotal / maskedPixels;
|
||||
const output = Buffer.from(source.data);
|
||||
const bounds = quadBounds(quad, source.width, source.height);
|
||||
|
||||
for (let y = bounds.top; y <= bounds.bottom; y++) {
|
||||
for (let x = bounds.left; x <= bounds.right; x++) {
|
||||
const pixel = y * source.width + x;
|
||||
const alpha = featheredMask[pixel] / 255;
|
||||
if (alpha <= 0.002) continue;
|
||||
const mapped = mapPoint(homography, x, y);
|
||||
if (mapped.x < 0 || mapped.y < 0 || mapped.x >= screenWidth || mapped.y >= screenHeight) continue;
|
||||
const sampled = bilinearSample(fitted, mapped.x, mapped.y);
|
||||
const offset = pixel * source.channels;
|
||||
const sourceLuminance = luminance(source.data[offset], source.data[offset + 1], source.data[offset + 2]);
|
||||
const gain = clamp(1 + 0.35 * (sourceLuminance / Math.max(referenceLuminance, 1) - 1), 0.55, 1.65);
|
||||
for (let channel = 0; channel < 3; channel++) {
|
||||
const replacement = clamp(sampled[channel] * gain, 0, 255);
|
||||
output[offset + channel] = Math.round(source.data[offset + channel] * (1 - alpha) + replacement * alpha);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const bytes = await sharp(output, {
|
||||
raw: { width: source.width, height: source.height, channels: source.channels },
|
||||
}).jpeg({ quality: 92, mozjpeg: true }).toBuffer();
|
||||
return { bytes, mimeType: 'image/jpeg', width: source.width, height: source.height };
|
||||
}
|
||||
|
||||
async function decode(bytes: Buffer, maxDimension?: number): Promise<RawImage> {
|
||||
let pipeline = sharp(bytes, { limitInputPixels: MAX_INPUT_PIXELS, animated: false }).rotate().removeAlpha().toColourspace('srgb');
|
||||
if (maxDimension) pipeline = pipeline.resize({ width: maxDimension, height: maxDimension, fit: 'inside', withoutEnlargement: true });
|
||||
const decoded = await pipeline.raw().toBuffer({ resolveWithObject: true });
|
||||
if (decoded.info.channels < 3) throw new ImageReplacementInputError('Selected images must contain RGB colour data');
|
||||
return { data: decoded.data, width: decoded.info.width, height: decoded.info.height, channels: decoded.info.channels as 3 | 4 };
|
||||
}
|
||||
|
||||
async function fitCover(bytes: Buffer, width: number, height: number): Promise<RawImage> {
|
||||
const decoded = await sharp(bytes, { limitInputPixels: MAX_INPUT_PIXELS, animated: false })
|
||||
.rotate().removeAlpha().toColourspace('srgb')
|
||||
.resize(width, height, { fit: 'cover', position: 'centre' })
|
||||
.raw().toBuffer({ resolveWithObject: true });
|
||||
return { data: decoded.data, width, height, channels: decoded.info.channels as 3 | 4 };
|
||||
}
|
||||
|
||||
function greenMask(image: RawImage): Uint8Array {
|
||||
const mask = new Uint8Array(image.width * image.height);
|
||||
for (let pixel = 0; pixel < mask.length; pixel++) {
|
||||
const offset = pixel * image.channels;
|
||||
const r = image.data[offset] / 255;
|
||||
const g = image.data[offset + 1] / 255;
|
||||
const b = image.data[offset + 2] / 255;
|
||||
const max = Math.max(r, g, b);
|
||||
const min = Math.min(r, g, b);
|
||||
const saturation = max === 0 ? 0 : (max - min) / max;
|
||||
let hue = 0;
|
||||
if (max !== min) {
|
||||
if (max === r) hue = 60 * (((g - b) / (max - min)) % 6);
|
||||
else if (max === g) hue = 60 * ((b - r) / (max - min) + 2);
|
||||
else hue = 60 * ((r - g) / (max - min) + 4);
|
||||
if (hue < 0) hue += 360;
|
||||
}
|
||||
if (hue >= 70 && hue <= 180 && saturation >= 70 / 255 && max >= 50 / 255) mask[pixel] = 1;
|
||||
}
|
||||
return mask;
|
||||
}
|
||||
|
||||
function largestComponent(mask: Uint8Array, width: number, height: number): number[] {
|
||||
const seen = new Uint8Array(mask.length);
|
||||
const queue = new Int32Array(mask.length);
|
||||
let largest: number[] = [];
|
||||
for (let start = 0; start < mask.length; start++) {
|
||||
if (!mask[start] || seen[start]) continue;
|
||||
let head = 0;
|
||||
let tail = 0;
|
||||
queue[tail++] = start;
|
||||
seen[start] = 1;
|
||||
const component: number[] = [];
|
||||
while (head < tail) {
|
||||
const pixel = queue[head++];
|
||||
component.push(pixel);
|
||||
const x = pixel % width;
|
||||
const y = Math.floor(pixel / width);
|
||||
const neighbours = [pixel - width, pixel + width, pixel - 1, pixel + 1];
|
||||
for (let index = 0; index < neighbours.length; index++) {
|
||||
const neighbour = neighbours[index];
|
||||
if (neighbour < 0 || neighbour >= mask.length) continue;
|
||||
if (index === 2 && x === 0) continue;
|
||||
if (index === 3 && x === width - 1) continue;
|
||||
if ((index === 0 && y === 0) || (index === 1 && y === height - 1)) continue;
|
||||
if (mask[neighbour] && !seen[neighbour]) {
|
||||
seen[neighbour] = 1;
|
||||
queue[tail++] = neighbour;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (component.length > largest.length) largest = component;
|
||||
}
|
||||
return largest;
|
||||
}
|
||||
|
||||
function dilateMask(mask: Buffer, width: number, height: number): Buffer {
|
||||
const expanded = Buffer.from(mask);
|
||||
for (let y = 1; y < height - 1; y++) {
|
||||
for (let x = 1; x < width - 1; x++) {
|
||||
const pixel = y * width + x;
|
||||
if (!mask[pixel]) continue;
|
||||
for (let offsetY = -1; offsetY <= 1; offsetY++) {
|
||||
for (let offsetX = -1; offsetX <= 1; offsetX++) expanded[pixel + offsetY * width + offsetX] = 255;
|
||||
}
|
||||
}
|
||||
}
|
||||
return expanded;
|
||||
}
|
||||
|
||||
function componentExtremes(component: number[], width: number): [Point, Point, Point, Point] {
|
||||
const points = component.map((pixel) => ({ x: pixel % width, y: Math.floor(pixel / width) }));
|
||||
return [
|
||||
points.reduce((best, point) => point.x + point.y < best.x + best.y ? point : best),
|
||||
points.reduce((best, point) => point.x - point.y > best.x - best.y ? point : best),
|
||||
points.reduce((best, point) => point.x + point.y > best.x + best.y ? point : best),
|
||||
points.reduce((best, point) => point.y - point.x > best.y - best.x ? point : best),
|
||||
];
|
||||
}
|
||||
|
||||
function solveHomography(quad: Point[], width: number, height: number): number[] {
|
||||
const destinations = [{ x: 0, y: 0 }, { x: width - 1, y: 0 }, { x: width - 1, y: height - 1 }, { x: 0, y: height - 1 }];
|
||||
const matrix: number[][] = [];
|
||||
for (let index = 0; index < 4; index++) {
|
||||
const { x, y } = quad[index];
|
||||
const { x: u, y: v } = destinations[index];
|
||||
matrix.push([x, y, 1, 0, 0, 0, -u * x, -u * y, u]);
|
||||
matrix.push([0, 0, 0, x, y, 1, -v * x, -v * y, v]);
|
||||
}
|
||||
for (let column = 0; column < 8; column++) {
|
||||
let pivot = column;
|
||||
for (let row = column + 1; row < 8; row++) if (Math.abs(matrix[row][column]) > Math.abs(matrix[pivot][column])) pivot = row;
|
||||
[matrix[column], matrix[pivot]] = [matrix[pivot], matrix[column]];
|
||||
const divisor = matrix[column][column];
|
||||
if (Math.abs(divisor) < 1e-9) throw new ImageReplacementInputError('The detected green-screen shape is not usable');
|
||||
for (let cell = column; cell < 9; cell++) matrix[column][cell] /= divisor;
|
||||
for (let row = 0; row < 8; row++) {
|
||||
if (row === column) continue;
|
||||
const factor = matrix[row][column];
|
||||
for (let cell = column; cell < 9; cell++) matrix[row][cell] -= factor * matrix[column][cell];
|
||||
}
|
||||
}
|
||||
return matrix.map((row) => row[8]);
|
||||
}
|
||||
|
||||
function mapPoint(h: number[], x: number, y: number): Point {
|
||||
const denominator = h[6] * x + h[7] * y + 1;
|
||||
return { x: (h[0] * x + h[1] * y + h[2]) / denominator, y: (h[3] * x + h[4] * y + h[5]) / denominator };
|
||||
}
|
||||
|
||||
function bilinearSample(image: RawImage, x: number, y: number): [number, number, number] {
|
||||
const x0 = Math.floor(x);
|
||||
const y0 = Math.floor(y);
|
||||
const x1 = Math.min(x0 + 1, image.width - 1);
|
||||
const y1 = Math.min(y0 + 1, image.height - 1);
|
||||
const dx = x - x0;
|
||||
const dy = y - y0;
|
||||
const sample = (sx: number, sy: number, channel: number) => image.data[(sy * image.width + sx) * image.channels + channel];
|
||||
return [0, 1, 2].map((channel) => (
|
||||
sample(x0, y0, channel) * (1 - dx) * (1 - dy)
|
||||
+ sample(x1, y0, channel) * dx * (1 - dy)
|
||||
+ sample(x0, y1, channel) * (1 - dx) * dy
|
||||
+ sample(x1, y1, channel) * dx * dy
|
||||
)) as [number, number, number];
|
||||
}
|
||||
|
||||
function pointInQuad(x: number, y: number, quad: Point[]): boolean {
|
||||
let sign = 0;
|
||||
for (let index = 0; index < 4; index++) {
|
||||
const a = quad[index];
|
||||
const b = quad[(index + 1) % 4];
|
||||
const cross = (b.x - a.x) * (y - a.y) - (b.y - a.y) * (x - a.x);
|
||||
if (Math.abs(cross) < 1e-6) continue;
|
||||
const nextSign = Math.sign(cross);
|
||||
if (sign !== 0 && nextSign !== sign) return false;
|
||||
sign = nextSign;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function quadBounds(quad: Point[], width: number, height: number) {
|
||||
const xs = quad.map((point) => point.x);
|
||||
const ys = quad.map((point) => point.y);
|
||||
return {
|
||||
left: clamp(Math.floor(Math.min(...xs)) - 3, 0, width - 1),
|
||||
right: clamp(Math.ceil(Math.max(...xs)) + 3, 0, width - 1),
|
||||
top: clamp(Math.floor(Math.min(...ys)) - 3, 0, height - 1),
|
||||
bottom: clamp(Math.ceil(Math.max(...ys)) + 3, 0, height - 1),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveInside(root: string, relativePath: string): string {
|
||||
const resolvedRoot = path.resolve(root);
|
||||
const resolved = path.resolve(resolvedRoot, relativePath);
|
||||
if (resolved === resolvedRoot || !resolved.startsWith(`${resolvedRoot}${path.sep}`)) {
|
||||
throw new ImageReplacementInputError('Unsafe conversation media path');
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function distance(a: Point, b: Point): number { return Math.hypot(a.x - b.x, a.y - b.y); }
|
||||
function luminance(r: number, g: number, b: number): number { return 0.2126 * r + 0.7152 * g + 0.0722 * b; }
|
||||
function clamp(value: number, minimum: number, maximum: number): number { return Math.min(maximum, Math.max(minimum, value)); }
|
||||
function safeError(error: unknown): string { return error instanceof Error ? error.message : String(error); }
|
||||
@@ -107,6 +107,19 @@ export const API_OPERATIONS: readonly ApiOperation[] = [
|
||||
operation('sendReply', 'POST', '/api/conversations/{id}/send-reply', 'Send a conversation reply', ['Conversations']),
|
||||
operation('getConversationMediaOptions', 'GET', '/api/conversations/{id}/shareable-media', 'List every enabled exchange-library image not yet shared in this conversation', ['Conversations']),
|
||||
operation('shareConversationMedia', 'POST', '/api/conversations/{id}/share-media', 'Send a selected exchange-library image', ['Conversations']),
|
||||
operation('getConversationImageReplacementOptions', 'GET', '/api/conversations/{id}/image-replacement-options', 'List green-screen sources and conversation-owned replacement targets', ['Conversations'], {
|
||||
responseSchema: z.object({ sources: z.array(z.unknown()), targets: z.array(z.unknown()), reason: z.string().nullable() }),
|
||||
}),
|
||||
operation('generateConversationImageReplacement', 'POST', '/api/conversations/{id}/generate-image-replacement', 'Composite a selected conversation image into a green-screen source', ['Conversations'], {
|
||||
requestSchema: z.strictObject({ sourceAssetId: z.string().uuid(), targetId: z.string().min(1) }),
|
||||
responseSchema: z.strictObject({
|
||||
imageDataUrl: z.string().startsWith('data:image/jpeg;base64,'),
|
||||
mimeType: z.literal('image/jpeg'),
|
||||
fileName: z.string(),
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
}),
|
||||
}),
|
||||
operation('startConversation', 'POST', '/api/conversations/start', 'Start a new conversation', ['Conversations']),
|
||||
operation('markConversationRead', 'POST', '/api/conversations/{id}/read', 'Advance the operator read watermark', ['Conversations'], {
|
||||
requestSchema: z.strictObject({ throughTimestamp: z.number().int().nonnegative().optional() }),
|
||||
|
||||
@@ -31,6 +31,7 @@ import type { AccountGrowthRepository } from '../database/growth-repository.js';
|
||||
import type { AccountBrowsingSimulationRepository } from '../database/browsing-simulation-repository.js';
|
||||
import type { PushNotifier } from '../services/push-notifier.js';
|
||||
import type { BotMode } from '../services/bot-mode.js';
|
||||
import type { ImageReplacementGenerator } from '../services/image-replacement.js';
|
||||
|
||||
/** Bag of everything the route modules might need. */
|
||||
export interface WebContext {
|
||||
@@ -69,6 +70,7 @@ export interface WebContext {
|
||||
browsing?: AccountBrowsingSimulationRepository;
|
||||
mediaLibrary?: MediaLibraryService;
|
||||
pushNotifier?: PushNotifier;
|
||||
imageReplacement?: ImageReplacementGenerator;
|
||||
};
|
||||
|
||||
/** Operator's run/scan/stop intent, read by the main loop. */
|
||||
|
||||
@@ -23,8 +23,13 @@ import type {
|
||||
ConversationReadStateRepository,
|
||||
} from '../../database/repositories.js';
|
||||
import type { RedditRecipientAccountStatusRecord } from '../../database/account-repositories.js';
|
||||
import { ImageReplacementInputError, type ImageReplacementTarget } from '../../services/image-replacement.js';
|
||||
|
||||
const ShareConversationMediaRequestSchema = z.strictObject({ assetId: z.string().uuid() });
|
||||
const GenerateConversationImageReplacementRequestSchema = z.strictObject({
|
||||
sourceAssetId: z.string().uuid(),
|
||||
targetId: z.string().min(1),
|
||||
});
|
||||
const MarkReadRequestSchema = z.strictObject({
|
||||
throughTimestamp: z.number().int().nonnegative().optional(),
|
||||
});
|
||||
@@ -259,6 +264,42 @@ export function createConversationsRouter(ctx: WebContext): Router {
|
||||
};
|
||||
}
|
||||
|
||||
function triggeringImageUrl(
|
||||
conversation: ActiveChat,
|
||||
runtime: NonNullable<ReturnType<NonNullable<WebContext['services']['accountRuntimes']>['get']>> | null,
|
||||
): string | null {
|
||||
const messagesRepo = runtime?.resources.messagesRepo ?? ctx.repos.messages;
|
||||
const sent = conversation.roomId
|
||||
? messagesRepo.getByChatRoomId(conversation.roomId) ?? messagesRepo.getLatestByRecipient(conversation.username)
|
||||
: messagesRepo.getLatestByRecipient(conversation.username);
|
||||
return sent?.postImageUrl ?? null;
|
||||
}
|
||||
|
||||
function replacementTargets(conversation: ActiveChat, originalPostImageUrl: string | null) {
|
||||
const targets: Array<{ id: string; label: string; urlPath: string; target: ImageReplacementTarget }> = [];
|
||||
let conversationImageNumber = 0;
|
||||
if (originalPostImageUrl) {
|
||||
targets.push({
|
||||
id: 'original-post',
|
||||
label: 'Original post image',
|
||||
urlPath: originalPostImageUrl,
|
||||
target: { kind: 'remote', url: originalPostImageUrl, userAgent: ctx.config.reddit.userAgent },
|
||||
});
|
||||
}
|
||||
(conversation.messages ?? []).forEach((message, messageIndex) => {
|
||||
(message.media ?? []).forEach((media, mediaIndex) => {
|
||||
conversationImageNumber++;
|
||||
targets.push({
|
||||
id: `message-${messageIndex}-${mediaIndex}`,
|
||||
label: `${message.isSelf ? 'Your' : `u/${conversation.username}'s`} conversation image ${conversationImageNumber}`,
|
||||
urlPath: `/media/${media.localPath.split(/[\\/]/).map(encodeURIComponent).join('/')}`,
|
||||
target: { kind: 'conversation-media', localPath: media.localPath },
|
||||
});
|
||||
});
|
||||
});
|
||||
return targets;
|
||||
}
|
||||
|
||||
router.get('/conversations', async (req, res) => {
|
||||
const chatService = ctx.services.chat;
|
||||
if (!chatService) {
|
||||
@@ -1007,6 +1048,67 @@ export function createConversationsRouter(ctx: WebContext): Router {
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/conversations/:id/image-replacement-options', async (req, res) => {
|
||||
try {
|
||||
const resolved = await findConversation(decodeURIComponent(req.params.id));
|
||||
if (!resolved) return res.status(404).json({ error: 'Conversation not found' });
|
||||
const library = ctx.services.mediaLibrary;
|
||||
if (!library || !ctx.services.imageReplacement) {
|
||||
return res.status(503).json({ error: 'Image replacement is not initialized' });
|
||||
}
|
||||
const targets = replacementTargets(
|
||||
resolved.conversation,
|
||||
triggeringImageUrl(resolved.conversation, resolved.runtime),
|
||||
);
|
||||
const sources = library.list().map(toMediaLibraryResponse);
|
||||
const reason = sources.length === 0
|
||||
? 'Upload a green-screen source image to the exchange library first.'
|
||||
: targets.length === 0
|
||||
? 'This conversation has no original-post or conversation images to use as the replacement.'
|
||||
: null;
|
||||
return res.json({
|
||||
sources,
|
||||
targets: targets.map(({ target: _target, ...publicTarget }) => publicTarget),
|
||||
reason,
|
||||
});
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to list image replacement options:', error);
|
||||
return res.status(500).json({ error: 'Failed to list image replacement options' });
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/conversations/:id/generate-image-replacement', async (req, res) => {
|
||||
const parsed = GenerateConversationImageReplacementRequestSchema.safeParse(req.body);
|
||||
if (!parsed.success) return res.status(400).json({ error: 'Select a valid source and target image' });
|
||||
try {
|
||||
const resolved = await findConversation(decodeURIComponent(req.params.id));
|
||||
if (!resolved) return res.status(404).json({ error: 'Conversation not found' });
|
||||
const library = ctx.services.mediaLibrary;
|
||||
const generator = ctx.services.imageReplacement;
|
||||
if (!library || !generator) return res.status(503).json({ error: 'Image replacement is not initialized' });
|
||||
const source = library.get(parsed.data.sourceAssetId);
|
||||
if (!source) return res.status(404).json({ error: 'Source image not found in the exchange library' });
|
||||
const targets = replacementTargets(
|
||||
resolved.conversation,
|
||||
triggeringImageUrl(resolved.conversation, resolved.runtime),
|
||||
);
|
||||
const selectedTarget = targets.find((target) => target.id === parsed.data.targetId);
|
||||
if (!selectedTarget) return res.status(409).json({ error: 'The selected target image is no longer available' });
|
||||
const result = await generator.generate(library.resolvePath(source), selectedTarget.target);
|
||||
return res.json({
|
||||
imageDataUrl: `data:${result.mimeType};base64,${result.bytes.toString('base64')}`,
|
||||
mimeType: result.mimeType,
|
||||
fileName: `replacement-${resolved.conversation.username.replace(/[^a-zA-Z0-9_-]/g, '_')}.jpg`,
|
||||
width: result.width,
|
||||
height: result.height,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof ImageReplacementInputError) return res.status(400).json({ error: error.message });
|
||||
ctx.logger.error('Failed to generate conversation image replacement:', error);
|
||||
return res.status(500).json({ error: 'Failed to generate image replacement' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Start a brand-new DM with a Reddit user and send the opening message.
|
||||
* Body: { username: string, message: string }
|
||||
|
||||
+5
-1
@@ -56,6 +56,7 @@ import type { AccountBrowsingSimulationRepository } from '../database/browsing-s
|
||||
import { createGrowthRouter } from './routes/growth.js';
|
||||
import { createLearningRouter } from './routes/learning.js';
|
||||
import { allowsSending, type BotMode } from '../services/bot-mode.js';
|
||||
import { GreenScreenImageReplacementService } from '../services/image-replacement.js';
|
||||
|
||||
export type { BotStats, ActivityItem, LogEntry, RuntimeConfig } from './types.js';
|
||||
|
||||
@@ -77,7 +78,7 @@ export class WebServer {
|
||||
private runtimeConfig: RuntimeConfig;
|
||||
private onConfigChange: ((config: RuntimeConfig) => void) | null = null;
|
||||
|
||||
private services: WebContext['services'] = {};
|
||||
private services: WebContext['services'];
|
||||
|
||||
constructor(
|
||||
private port: number,
|
||||
@@ -94,6 +95,9 @@ export class WebServer {
|
||||
private readonly persistBotMode: (mode: BotMode) => void = () => undefined,
|
||||
private readonly accountContentRepo?: AccountContentRepository,
|
||||
) {
|
||||
this.services = {
|
||||
imageReplacement: new GreenScreenImageReplacementService(path.join(this.workspaceDirectory, 'data', 'media')),
|
||||
};
|
||||
this.botMode = initialBotMode;
|
||||
this.broadcaster = broadcaster ?? new LogBroadcaster();
|
||||
this.activity = new ActivityLog(activityLogRepo);
|
||||
|
||||
@@ -468,6 +468,53 @@ describe('ConversationsPage', () => {
|
||||
expect(screen.getByText('Share a saved image')).toHaveFocus();
|
||||
});
|
||||
|
||||
it('creates a blurred green-screen preview from a saved source and conversation target', async () => {
|
||||
installConversationList();
|
||||
const asset = {
|
||||
id: '11111111-1111-4111-8111-111111111111', tags: ['selfie'], safety: 'sfw',
|
||||
classificationStatus: 'ready', description: 'green phone', confidence: 1,
|
||||
fileName: 'green-phone.jpg', mimeType: 'image/jpeg', sizeBytes: 1024, width: 800, height: 1000,
|
||||
enabled: true, consentConfirmedAt: 1, createdAt: 1, updatedAt: 1,
|
||||
sendCount: 0, lastSentAt: null, urlPath: '/library-media/green-phone.jpg',
|
||||
};
|
||||
server.use(
|
||||
http.get('/api/conversations/room-alice/image-replacement-options', () => HttpResponse.json({
|
||||
sources: [asset],
|
||||
targets: [
|
||||
{ id: 'original-post', label: 'Original post image', urlPath: 'https://i.redd.it/source-post.jpg' },
|
||||
{ id: 'message-1-0', label: "u/alice's conversation image 2", urlPath: '/media/alice/image.jpg' },
|
||||
],
|
||||
reason: null,
|
||||
})),
|
||||
http.post('/api/conversations/room-alice/generate-image-replacement', async ({ request }) => {
|
||||
expect(await request.json()).toEqual({ sourceAssetId: asset.id, targetId: 'original-post' });
|
||||
return HttpResponse.json({
|
||||
imageDataUrl: 'data:image/jpeg;base64,Z2VuZXJhdGVk', mimeType: 'image/jpeg',
|
||||
fileName: 'replacement-alice.jpg', width: 800, height: 1000,
|
||||
});
|
||||
}),
|
||||
);
|
||||
const user = userEvent.setup();
|
||||
renderConversations();
|
||||
|
||||
await user.click(await screen.findByRole('button', { name: /u\/alice/ }));
|
||||
await user.click(await screen.findByText('Create a green-screen image'));
|
||||
expect(await screen.findByRole('img', { name: 'Green-screen source green-phone.jpg' })).toHaveClass('is-blurred');
|
||||
await user.click(screen.getByRole('button', { name: 'Reveal source images' }));
|
||||
await user.click(screen.getByRole('radio', { name: /green-phone\.jpg/ }));
|
||||
await user.click(screen.getByRole('button', { name: 'Reveal conversation choices' }));
|
||||
await user.click(screen.getByRole('radio', { name: 'Original post image' }));
|
||||
await user.click(screen.getByRole('button', { name: 'Create image' }));
|
||||
|
||||
const generated = await screen.findByRole('img', { name: 'Generated green-screen replacement' });
|
||||
expect(generated).toHaveClass('is-blurred');
|
||||
await user.click(screen.getByRole('button', { name: 'Reveal generated image' }));
|
||||
expect(generated).not.toHaveClass('is-blurred');
|
||||
expect(screen.getByRole('link', { name: 'Download generated image' })).toHaveAttribute(
|
||||
'download', 'replacement-alice.jpg',
|
||||
);
|
||||
});
|
||||
|
||||
it('shows unused turns separately from a safety handoff', async () => {
|
||||
server.use(
|
||||
http.get('/api/conversations', () => HttpResponse.json([alice])),
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import sharp from 'sharp';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { ImageReplacementInputError, replaceGreenScreen } from '../src/services/image-replacement.js';
|
||||
|
||||
describe('replaceGreenScreen', () => {
|
||||
it('warps a replacement into the detected green region and preserves the surrounding source', async () => {
|
||||
const source = await sharp({
|
||||
create: { width: 200, height: 160, channels: 3, background: '#20252a' },
|
||||
}).composite([{ input: Buffer.from(
|
||||
'<svg width="200" height="160"><polygon points="45,30 165,42 150,135 35,120" fill="#00ff00"/></svg>',
|
||||
) }]).png().toBuffer();
|
||||
const target = await sharp({
|
||||
create: { width: 80, height: 120, channels: 3, background: '#ee2233' },
|
||||
}).png().toBuffer();
|
||||
|
||||
const result = await replaceGreenScreen(source, target);
|
||||
const decoded = await sharp(result.bytes).raw().toBuffer({ resolveWithObject: true });
|
||||
const pixel = (x: number, y: number) => {
|
||||
const offset = (y * decoded.info.width + x) * decoded.info.channels;
|
||||
return [...decoded.data.subarray(offset, offset + 3)];
|
||||
};
|
||||
|
||||
expect(result).toMatchObject({ mimeType: 'image/jpeg', width: 200, height: 160 });
|
||||
expect(pixel(100, 80)[0]).toBeGreaterThan(150);
|
||||
expect(pixel(100, 80)[1]).toBeLessThan(100);
|
||||
expect(pixel(10, 10)[0]).toBeLessThan(70);
|
||||
});
|
||||
|
||||
it('rejects a source without a usable green-screen area', async () => {
|
||||
const source = await sharp({ create: { width: 100, height: 100, channels: 3, background: '#223344' } }).png().toBuffer();
|
||||
const target = await sharp({ create: { width: 20, height: 20, channels: 3, background: '#ffffff' } }).png().toBuffer();
|
||||
|
||||
await expect(replaceGreenScreen(source, target)).rejects.toThrow(ImageReplacementInputError);
|
||||
});
|
||||
});
|
||||
@@ -37,6 +37,81 @@ afterEach(async () => {
|
||||
});
|
||||
|
||||
describe('conversations route', () => {
|
||||
it('lists authoritative replacement targets and generates a private preview', async () => {
|
||||
const accountId = '@t2_one:reddit.com';
|
||||
const assetId = '11111111-1111-4111-8111-111111111111';
|
||||
const conversation = {
|
||||
username: 'alice', roomName: 'alice', roomId: 'room-one', url: '/room/room-one',
|
||||
lastMessageTime: 2, lastMessageSnippet: '[Image]', isFromUs: false, unread: true,
|
||||
messages: [{
|
||||
sender: 'alice', text: '[Image]', timestamp: 2, isSelf: false,
|
||||
media: [{ originalUrl: 'mxc://reddit.com/one', localPath: 'account/one.jpg', fileName: 'one.jpg' }],
|
||||
}],
|
||||
};
|
||||
const asset = {
|
||||
id: assetId, tags: ['selfie'], safety: 'sfw', classificationStatus: 'ready', description: 'phone template', confidence: 1,
|
||||
fileName: 'green-phone.jpg', relativePath: 'green-phone.jpg', mimeType: 'image/jpeg', sizeBytes: 4,
|
||||
width: 10, height: 10, enabled: true, consentConfirmedAt: 1, createdAt: 1, updatedAt: 1,
|
||||
sendCount: 0, lastSentAt: null,
|
||||
};
|
||||
const imageReplacement = {
|
||||
generate: vi.fn().mockResolvedValue({ bytes: Buffer.from('generated'), mimeType: 'image/jpeg', width: 100, height: 200 }),
|
||||
};
|
||||
const runtime = {
|
||||
snapshot: () => ({ state: 'running' }),
|
||||
resources: {
|
||||
chatService: { getCachedConversations: vi.fn().mockResolvedValue([conversation]) },
|
||||
messagesRepo: {
|
||||
getByChatRoomId: vi.fn().mockReturnValue({ postImageUrl: 'https://i.redd.it/original.jpg' }),
|
||||
getLatestByRecipient: vi.fn(),
|
||||
},
|
||||
},
|
||||
};
|
||||
const context = {
|
||||
config: { reddit: { userAgent: 'test-agent' } },
|
||||
logger: { error: vi.fn() },
|
||||
repos: { messages: {} },
|
||||
services: {
|
||||
chat: {}, imageReplacement,
|
||||
mediaLibrary: {
|
||||
list: vi.fn().mockReturnValue([asset]),
|
||||
get: vi.fn().mockReturnValue(asset),
|
||||
resolvePath: vi.fn().mockReturnValue('/private/green-phone.jpg'),
|
||||
},
|
||||
accountRuntimes: {
|
||||
list: () => [{ accountId, redditUsername: 'owner-one', state: 'running' }],
|
||||
get: (requested: string) => requested === accountId ? runtime : null,
|
||||
},
|
||||
},
|
||||
} as unknown as WebContext;
|
||||
const app = express().use(express.json()).use('/api', createConversationsRouter(context));
|
||||
const ownedId = encodeURIComponent(`${encodeURIComponent(accountId)}~room-one`);
|
||||
|
||||
const optionsResponse = await request(app, `/api/conversations/${ownedId}/image-replacement-options`);
|
||||
expect(optionsResponse.status).toBe(200);
|
||||
expect(await optionsResponse.json()).toMatchObject({
|
||||
reason: null,
|
||||
sources: [{ id: assetId, urlPath: '/library-media/green-phone.jpg' }],
|
||||
targets: [
|
||||
{ id: 'original-post', label: 'Original post image', urlPath: 'https://i.redd.it/original.jpg' },
|
||||
{ id: 'message-0-0', urlPath: '/media/account/one.jpg' },
|
||||
],
|
||||
});
|
||||
|
||||
const generateResponse = await request(app, `/api/conversations/${ownedId}/generate-image-replacement`, {
|
||||
method: 'POST', headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ sourceAssetId: assetId, targetId: 'message-0-0' }),
|
||||
});
|
||||
expect(generateResponse.status).toBe(200);
|
||||
expect(await generateResponse.json()).toEqual({
|
||||
imageDataUrl: `data:image/jpeg;base64,${Buffer.from('generated').toString('base64')}`,
|
||||
mimeType: 'image/jpeg', fileName: 'replacement-alice.jpg', width: 100, height: 200,
|
||||
});
|
||||
expect(imageReplacement.generate).toHaveBeenCalledWith('/private/green-phone.jpg', {
|
||||
kind: 'conversation-media', localPath: 'account/one.jpg',
|
||||
});
|
||||
});
|
||||
|
||||
it('aggregates simultaneous account runtimes and labels identical room IDs with their owner', async () => {
|
||||
const chat = (username: string) => ({
|
||||
username, roomName: username, roomId: 'same-room', lastMessageTime: 1,
|
||||
|
||||
Reference in new issue
Block a user