Add conversation image replacement

This commit is contained in:
jjbrunton committed 2026-08-14 12:56:00 +01:00
1 parent 2500f0a23c
commit 8523873831
18 files changed
+812 -3

No files matched your search

+301
View File
@@ -0,0 +1,301 @@
import fs from 'node:fs/promises';
import path from 'node:path';
import sharp from 'sharp';
import { downloadImage } from '../utils/image.js';
const MAX_INPUT_PIXELS = 25_000_000;
const DETECTION_MAX_DIMENSION = 900;
export interface ImageReplacementResult {
bytes: Buffer;
mimeType: 'image/jpeg';
width: number;
height: number;
}
export type ImageReplacementTarget =
| { kind: 'conversation-media'; localPath: string }
| { kind: 'remote'; url: string; userAgent: string };
export interface ImageReplacementGenerator {
generate(sourcePath: string, target: ImageReplacementTarget): Promise<ImageReplacementResult>;
}
export class ImageReplacementInputError extends Error {}
type RawImage = { data: Buffer; width: number; height: number; channels: 3 | 4 };
type Point = { x: number; y: number };
/**
* Ports the green-screen replacement prototype into the Node runtime. The
* source is an operator-owned exchange-library image; the replacement is a
* conversation image selected from authoritative server data.
*/
export class GreenScreenImageReplacementService implements ImageReplacementGenerator {
constructor(private readonly mediaRoot = path.join(process.cwd(), 'data', 'media')) {}
async generate(sourcePath: string, target: ImageReplacementTarget): Promise<ImageReplacementResult> {
const sourceBytes = await fs.readFile(sourcePath);
const targetBytes = target.kind === 'remote'
? (await downloadImage(target.url, target.userAgent)).buffer
: await fs.readFile(resolveInside(this.mediaRoot, target.localPath));
return replaceGreenScreen(sourceBytes, targetBytes);
}
}
export async function replaceGreenScreen(sourceBytes: Buffer, targetBytes: Buffer): Promise<ImageReplacementResult> {
const [source, detection] = await Promise.all([
decode(sourceBytes),
decode(sourceBytes, DETECTION_MAX_DIMENSION),
]).catch((error: unknown) => {
throw new ImageReplacementInputError(`Unable to read the selected image: ${safeError(error)}`);
});
const smallMask = greenMask(detection);
const component = largestComponent(smallMask, detection.width, detection.height);
if (component.length < Math.max(64, Math.round(detection.width * detection.height * 0.002))) {
throw new ImageReplacementInputError('No usable green-screen region was found in the source image');
}
const smallQuad = componentExtremes(component, detection.width);
const scaleX = source.width / detection.width;
const scaleY = source.height / detection.height;
const quad = smallQuad.map(({ x, y }) => ({ x: x * scaleX, y: y * scaleY }));
const screenWidth = Math.max(1, Math.round(Math.max(distance(quad[0], quad[1]), distance(quad[3], quad[2]))));
const screenHeight = Math.max(1, Math.round(Math.max(distance(quad[0], quad[3]), distance(quad[1], quad[2]))));
const fitted = await fitCover(targetBytes, screenWidth, screenHeight).catch((error: unknown) => {
throw new ImageReplacementInputError(`Unable to read the selected replacement image: ${safeError(error)}`);
});
const homography = solveHomography(quad, screenWidth, screenHeight);
const rawMask = greenMask(source);
const clippedMask = Buffer.alloc(source.width * source.height);
let luminanceTotal = 0;
let maskedPixels = 0;
for (let y = 0; y < source.height; y++) {
for (let x = 0; x < source.width; x++) {
const index = y * source.width + x;
if (rawMask[index] && pointInQuad(x, y, quad)) {
clippedMask[index] = 255;
const offset = index * source.channels;
luminanceTotal += luminance(source.data[offset], source.data[offset + 1], source.data[offset + 2]);
maskedPixels++;
}
}
}
if (maskedPixels === 0) throw new ImageReplacementInputError('The green-screen region could not be composited');
// Expand by one pixel before feathering so antialiased green edge pixels do
// not survive as a bright fringe around the insert.
const expandedMask = dilateMask(clippedMask, source.width, source.height);
const featheredMask = await sharp(expandedMask, { raw: { width: source.width, height: source.height, channels: 1 } })
.blur(1.5)
.greyscale()
.raw()
.toBuffer();
const referenceLuminance = luminanceTotal / maskedPixels;
const output = Buffer.from(source.data);
const bounds = quadBounds(quad, source.width, source.height);
for (let y = bounds.top; y <= bounds.bottom; y++) {
for (let x = bounds.left; x <= bounds.right; x++) {
const pixel = y * source.width + x;
const alpha = featheredMask[pixel] / 255;
if (alpha <= 0.002) continue;
const mapped = mapPoint(homography, x, y);
if (mapped.x < 0 || mapped.y < 0 || mapped.x >= screenWidth || mapped.y >= screenHeight) continue;
const sampled = bilinearSample(fitted, mapped.x, mapped.y);
const offset = pixel * source.channels;
const sourceLuminance = luminance(source.data[offset], source.data[offset + 1], source.data[offset + 2]);
const gain = clamp(1 + 0.35 * (sourceLuminance / Math.max(referenceLuminance, 1) - 1), 0.55, 1.65);
for (let channel = 0; channel < 3; channel++) {
const replacement = clamp(sampled[channel] * gain, 0, 255);
output[offset + channel] = Math.round(source.data[offset + channel] * (1 - alpha) + replacement * alpha);
}
}
}
const bytes = await sharp(output, {
raw: { width: source.width, height: source.height, channels: source.channels },
}).jpeg({ quality: 92, mozjpeg: true }).toBuffer();
return { bytes, mimeType: 'image/jpeg', width: source.width, height: source.height };
}
async function decode(bytes: Buffer, maxDimension?: number): Promise<RawImage> {
let pipeline = sharp(bytes, { limitInputPixels: MAX_INPUT_PIXELS, animated: false }).rotate().removeAlpha().toColourspace('srgb');
if (maxDimension) pipeline = pipeline.resize({ width: maxDimension, height: maxDimension, fit: 'inside', withoutEnlargement: true });
const decoded = await pipeline.raw().toBuffer({ resolveWithObject: true });
if (decoded.info.channels < 3) throw new ImageReplacementInputError('Selected images must contain RGB colour data');
return { data: decoded.data, width: decoded.info.width, height: decoded.info.height, channels: decoded.info.channels as 3 | 4 };
}
async function fitCover(bytes: Buffer, width: number, height: number): Promise<RawImage> {
const decoded = await sharp(bytes, { limitInputPixels: MAX_INPUT_PIXELS, animated: false })
.rotate().removeAlpha().toColourspace('srgb')
.resize(width, height, { fit: 'cover', position: 'centre' })
.raw().toBuffer({ resolveWithObject: true });
return { data: decoded.data, width, height, channels: decoded.info.channels as 3 | 4 };
}
function greenMask(image: RawImage): Uint8Array {
const mask = new Uint8Array(image.width * image.height);
for (let pixel = 0; pixel < mask.length; pixel++) {
const offset = pixel * image.channels;
const r = image.data[offset] / 255;
const g = image.data[offset + 1] / 255;
const b = image.data[offset + 2] / 255;
const max = Math.max(r, g, b);
const min = Math.min(r, g, b);
const saturation = max === 0 ? 0 : (max - min) / max;
let hue = 0;
if (max !== min) {
if (max === r) hue = 60 * (((g - b) / (max - min)) % 6);
else if (max === g) hue = 60 * ((b - r) / (max - min) + 2);
else hue = 60 * ((r - g) / (max - min) + 4);
if (hue < 0) hue += 360;
}
if (hue >= 70 && hue <= 180 && saturation >= 70 / 255 && max >= 50 / 255) mask[pixel] = 1;
}
return mask;
}
function largestComponent(mask: Uint8Array, width: number, height: number): number[] {
const seen = new Uint8Array(mask.length);
const queue = new Int32Array(mask.length);
let largest: number[] = [];
for (let start = 0; start < mask.length; start++) {
if (!mask[start] || seen[start]) continue;
let head = 0;
let tail = 0;
queue[tail++] = start;
seen[start] = 1;
const component: number[] = [];
while (head < tail) {
const pixel = queue[head++];
component.push(pixel);
const x = pixel % width;
const y = Math.floor(pixel / width);
const neighbours = [pixel - width, pixel + width, pixel - 1, pixel + 1];
for (let index = 0; index < neighbours.length; index++) {
const neighbour = neighbours[index];
if (neighbour < 0 || neighbour >= mask.length) continue;
if (index === 2 && x === 0) continue;
if (index === 3 && x === width - 1) continue;
if ((index === 0 && y === 0) || (index === 1 && y === height - 1)) continue;
if (mask[neighbour] && !seen[neighbour]) {
seen[neighbour] = 1;
queue[tail++] = neighbour;
}
}
}
if (component.length > largest.length) largest = component;
}
return largest;
}
function dilateMask(mask: Buffer, width: number, height: number): Buffer {
const expanded = Buffer.from(mask);
for (let y = 1; y < height - 1; y++) {
for (let x = 1; x < width - 1; x++) {
const pixel = y * width + x;
if (!mask[pixel]) continue;
for (let offsetY = -1; offsetY <= 1; offsetY++) {
for (let offsetX = -1; offsetX <= 1; offsetX++) expanded[pixel + offsetY * width + offsetX] = 255;
}
}
}
return expanded;
}
function componentExtremes(component: number[], width: number): [Point, Point, Point, Point] {
const points = component.map((pixel) => ({ x: pixel % width, y: Math.floor(pixel / width) }));
return [
points.reduce((best, point) => point.x + point.y < best.x + best.y ? point : best),
points.reduce((best, point) => point.x - point.y > best.x - best.y ? point : best),
points.reduce((best, point) => point.x + point.y > best.x + best.y ? point : best),
points.reduce((best, point) => point.y - point.x > best.y - best.x ? point : best),
];
}
function solveHomography(quad: Point[], width: number, height: number): number[] {
const destinations = [{ x: 0, y: 0 }, { x: width - 1, y: 0 }, { x: width - 1, y: height - 1 }, { x: 0, y: height - 1 }];
const matrix: number[][] = [];
for (let index = 0; index < 4; index++) {
const { x, y } = quad[index];
const { x: u, y: v } = destinations[index];
matrix.push([x, y, 1, 0, 0, 0, -u * x, -u * y, u]);
matrix.push([0, 0, 0, x, y, 1, -v * x, -v * y, v]);
}
for (let column = 0; column < 8; column++) {
let pivot = column;
for (let row = column + 1; row < 8; row++) if (Math.abs(matrix[row][column]) > Math.abs(matrix[pivot][column])) pivot = row;
[matrix[column], matrix[pivot]] = [matrix[pivot], matrix[column]];
const divisor = matrix[column][column];
if (Math.abs(divisor) < 1e-9) throw new ImageReplacementInputError('The detected green-screen shape is not usable');
for (let cell = column; cell < 9; cell++) matrix[column][cell] /= divisor;
for (let row = 0; row < 8; row++) {
if (row === column) continue;
const factor = matrix[row][column];
for (let cell = column; cell < 9; cell++) matrix[row][cell] -= factor * matrix[column][cell];
}
}
return matrix.map((row) => row[8]);
}
function mapPoint(h: number[], x: number, y: number): Point {
const denominator = h[6] * x + h[7] * y + 1;
return { x: (h[0] * x + h[1] * y + h[2]) / denominator, y: (h[3] * x + h[4] * y + h[5]) / denominator };
}
function bilinearSample(image: RawImage, x: number, y: number): [number, number, number] {
const x0 = Math.floor(x);
const y0 = Math.floor(y);
const x1 = Math.min(x0 + 1, image.width - 1);
const y1 = Math.min(y0 + 1, image.height - 1);
const dx = x - x0;
const dy = y - y0;
const sample = (sx: number, sy: number, channel: number) => image.data[(sy * image.width + sx) * image.channels + channel];
return [0, 1, 2].map((channel) => (
sample(x0, y0, channel) * (1 - dx) * (1 - dy)
+ sample(x1, y0, channel) * dx * (1 - dy)
+ sample(x0, y1, channel) * (1 - dx) * dy
+ sample(x1, y1, channel) * dx * dy
)) as [number, number, number];
}
function pointInQuad(x: number, y: number, quad: Point[]): boolean {
let sign = 0;
for (let index = 0; index < 4; index++) {
const a = quad[index];
const b = quad[(index + 1) % 4];
const cross = (b.x - a.x) * (y - a.y) - (b.y - a.y) * (x - a.x);
if (Math.abs(cross) < 1e-6) continue;
const nextSign = Math.sign(cross);
if (sign !== 0 && nextSign !== sign) return false;
sign = nextSign;
}
return true;
}
function quadBounds(quad: Point[], width: number, height: number) {
const xs = quad.map((point) => point.x);
const ys = quad.map((point) => point.y);
return {
left: clamp(Math.floor(Math.min(...xs)) - 3, 0, width - 1),
right: clamp(Math.ceil(Math.max(...xs)) + 3, 0, width - 1),
top: clamp(Math.floor(Math.min(...ys)) - 3, 0, height - 1),
bottom: clamp(Math.ceil(Math.max(...ys)) + 3, 0, height - 1),
};
}
function resolveInside(root: string, relativePath: string): string {
const resolvedRoot = path.resolve(root);
const resolved = path.resolve(resolvedRoot, relativePath);
if (resolved === resolvedRoot || !resolved.startsWith(`${resolvedRoot}${path.sep}`)) {
throw new ImageReplacementInputError('Unsafe conversation media path');
}
return resolved;
}
function distance(a: Point, b: Point): number { return Math.hypot(a.x - b.x, a.y - b.y); }
function luminance(r: number, g: number, b: number): number { return 0.2126 * r + 0.7152 * g + 0.0722 * b; }
function clamp(value: number, minimum: number, maximum: number): number { return Math.min(maximum, Math.max(minimum, value)); }
function safeError(error: unknown): string { return error instanceof Error ? error.message : String(error); }
+13
View File
@@ -107,6 +107,19 @@ export const API_OPERATIONS: readonly ApiOperation[] = [
operation('sendReply', 'POST', '/api/conversations/{id}/send-reply', 'Send a conversation reply', ['Conversations']),
operation('getConversationMediaOptions', 'GET', '/api/conversations/{id}/shareable-media', 'List every enabled exchange-library image not yet shared in this conversation', ['Conversations']),
operation('shareConversationMedia', 'POST', '/api/conversations/{id}/share-media', 'Send a selected exchange-library image', ['Conversations']),
operation('getConversationImageReplacementOptions', 'GET', '/api/conversations/{id}/image-replacement-options', 'List green-screen sources and conversation-owned replacement targets', ['Conversations'], {
responseSchema: z.object({ sources: z.array(z.unknown()), targets: z.array(z.unknown()), reason: z.string().nullable() }),
}),
operation('generateConversationImageReplacement', 'POST', '/api/conversations/{id}/generate-image-replacement', 'Composite a selected conversation image into a green-screen source', ['Conversations'], {
requestSchema: z.strictObject({ sourceAssetId: z.string().uuid(), targetId: z.string().min(1) }),
responseSchema: z.strictObject({
imageDataUrl: z.string().startsWith('data:image/jpeg;base64,'),
mimeType: z.literal('image/jpeg'),
fileName: z.string(),
width: z.number().int().positive(),
height: z.number().int().positive(),
}),
}),
operation('startConversation', 'POST', '/api/conversations/start', 'Start a new conversation', ['Conversations']),
operation('markConversationRead', 'POST', '/api/conversations/{id}/read', 'Advance the operator read watermark', ['Conversations'], {
requestSchema: z.strictObject({ throughTimestamp: z.number().int().nonnegative().optional() }),
+2
View File
@@ -31,6 +31,7 @@ import type { AccountGrowthRepository } from '../database/growth-repository.js';
import type { AccountBrowsingSimulationRepository } from '../database/browsing-simulation-repository.js';
import type { PushNotifier } from '../services/push-notifier.js';
import type { BotMode } from '../services/bot-mode.js';
import type { ImageReplacementGenerator } from '../services/image-replacement.js';
/** Bag of everything the route modules might need. */
export interface WebContext {
@@ -69,6 +70,7 @@ export interface WebContext {
browsing?: AccountBrowsingSimulationRepository;
mediaLibrary?: MediaLibraryService;
pushNotifier?: PushNotifier;
imageReplacement?: ImageReplacementGenerator;
};
/** Operator's run/scan/stop intent, read by the main loop. */
+102
View File
@@ -23,8 +23,13 @@ import type {
ConversationReadStateRepository,
} from '../../database/repositories.js';
import type { RedditRecipientAccountStatusRecord } from '../../database/account-repositories.js';
import { ImageReplacementInputError, type ImageReplacementTarget } from '../../services/image-replacement.js';
const ShareConversationMediaRequestSchema = z.strictObject({ assetId: z.string().uuid() });
const GenerateConversationImageReplacementRequestSchema = z.strictObject({
sourceAssetId: z.string().uuid(),
targetId: z.string().min(1),
});
const MarkReadRequestSchema = z.strictObject({
throughTimestamp: z.number().int().nonnegative().optional(),
});
@@ -259,6 +264,42 @@ export function createConversationsRouter(ctx: WebContext): Router {
};
}
function triggeringImageUrl(
conversation: ActiveChat,
runtime: NonNullable<ReturnType<NonNullable<WebContext['services']['accountRuntimes']>['get']>> | null,
): string | null {
const messagesRepo = runtime?.resources.messagesRepo ?? ctx.repos.messages;
const sent = conversation.roomId
? messagesRepo.getByChatRoomId(conversation.roomId) ?? messagesRepo.getLatestByRecipient(conversation.username)
: messagesRepo.getLatestByRecipient(conversation.username);
return sent?.postImageUrl ?? null;
}
function replacementTargets(conversation: ActiveChat, originalPostImageUrl: string | null) {
const targets: Array<{ id: string; label: string; urlPath: string; target: ImageReplacementTarget }> = [];
let conversationImageNumber = 0;
if (originalPostImageUrl) {
targets.push({
id: 'original-post',
label: 'Original post image',
urlPath: originalPostImageUrl,
target: { kind: 'remote', url: originalPostImageUrl, userAgent: ctx.config.reddit.userAgent },
});
}
(conversation.messages ?? []).forEach((message, messageIndex) => {
(message.media ?? []).forEach((media, mediaIndex) => {
conversationImageNumber++;
targets.push({
id: `message-${messageIndex}-${mediaIndex}`,
label: `${message.isSelf ? 'Your' : `u/${conversation.username}'s`} conversation image ${conversationImageNumber}`,
urlPath: `/media/${media.localPath.split(/[\\/]/).map(encodeURIComponent).join('/')}`,
target: { kind: 'conversation-media', localPath: media.localPath },
});
});
});
return targets;
}
router.get('/conversations', async (req, res) => {
const chatService = ctx.services.chat;
if (!chatService) {
@@ -1007,6 +1048,67 @@ export function createConversationsRouter(ctx: WebContext): Router {
}
});
router.get('/conversations/:id/image-replacement-options', async (req, res) => {
try {
const resolved = await findConversation(decodeURIComponent(req.params.id));
if (!resolved) return res.status(404).json({ error: 'Conversation not found' });
const library = ctx.services.mediaLibrary;
if (!library || !ctx.services.imageReplacement) {
return res.status(503).json({ error: 'Image replacement is not initialized' });
}
const targets = replacementTargets(
resolved.conversation,
triggeringImageUrl(resolved.conversation, resolved.runtime),
);
const sources = library.list().map(toMediaLibraryResponse);
const reason = sources.length === 0
? 'Upload a green-screen source image to the exchange library first.'
: targets.length === 0
? 'This conversation has no original-post or conversation images to use as the replacement.'
: null;
return res.json({
sources,
targets: targets.map(({ target: _target, ...publicTarget }) => publicTarget),
reason,
});
} catch (error) {
ctx.logger.error('Failed to list image replacement options:', error);
return res.status(500).json({ error: 'Failed to list image replacement options' });
}
});
router.post('/conversations/:id/generate-image-replacement', async (req, res) => {
const parsed = GenerateConversationImageReplacementRequestSchema.safeParse(req.body);
if (!parsed.success) return res.status(400).json({ error: 'Select a valid source and target image' });
try {
const resolved = await findConversation(decodeURIComponent(req.params.id));
if (!resolved) return res.status(404).json({ error: 'Conversation not found' });
const library = ctx.services.mediaLibrary;
const generator = ctx.services.imageReplacement;
if (!library || !generator) return res.status(503).json({ error: 'Image replacement is not initialized' });
const source = library.get(parsed.data.sourceAssetId);
if (!source) return res.status(404).json({ error: 'Source image not found in the exchange library' });
const targets = replacementTargets(
resolved.conversation,
triggeringImageUrl(resolved.conversation, resolved.runtime),
);
const selectedTarget = targets.find((target) => target.id === parsed.data.targetId);
if (!selectedTarget) return res.status(409).json({ error: 'The selected target image is no longer available' });
const result = await generator.generate(library.resolvePath(source), selectedTarget.target);
return res.json({
imageDataUrl: `data:${result.mimeType};base64,${result.bytes.toString('base64')}`,
mimeType: result.mimeType,
fileName: `replacement-${resolved.conversation.username.replace(/[^a-zA-Z0-9_-]/g, '_')}.jpg`,
width: result.width,
height: result.height,
});
} catch (error) {
if (error instanceof ImageReplacementInputError) return res.status(400).json({ error: error.message });
ctx.logger.error('Failed to generate conversation image replacement:', error);
return res.status(500).json({ error: 'Failed to generate image replacement' });
}
});
/**
* Start a brand-new DM with a Reddit user and send the opening message.
* Body: { username: string, message: string }
+5 -1
View File
@@ -56,6 +56,7 @@ import type { AccountBrowsingSimulationRepository } from '../database/browsing-s
import { createGrowthRouter } from './routes/growth.js';
import { createLearningRouter } from './routes/learning.js';
import { allowsSending, type BotMode } from '../services/bot-mode.js';
import { GreenScreenImageReplacementService } from '../services/image-replacement.js';
export type { BotStats, ActivityItem, LogEntry, RuntimeConfig } from './types.js';
@@ -77,7 +78,7 @@ export class WebServer {
private runtimeConfig: RuntimeConfig;
private onConfigChange: ((config: RuntimeConfig) => void) | null = null;
private services: WebContext['services'] = {};
private services: WebContext['services'];
constructor(
private port: number,
@@ -94,6 +95,9 @@ export class WebServer {
private readonly persistBotMode: (mode: BotMode) => void = () => undefined,
private readonly accountContentRepo?: AccountContentRepository,
) {
this.services = {
imageReplacement: new GreenScreenImageReplacementService(path.join(this.workspaceDirectory, 'data', 'media')),
};
this.botMode = initialBotMode;
this.broadcaster = broadcaster ?? new LogBroadcaster();
this.activity = new ActivityLog(activityLogRepo);