/** * Matrix-API prototype CLI. * * Usage: * tsx scripts/matrix-test.ts # capture token + list rooms * tsx scripts/matrix-test.ts --refresh # force re-extract token * tsx scripts/matrix-test.ts --room # dump message history for a room * tsx scripts/matrix-test.ts --send "hello" # send a test message * tsx scripts/matrix-test.ts --dump # download every room's full * # history + images to disk * * Reuses the bot's persistent browser session at ./data/browser-session, * so you must already be logged into Reddit there (run `npm run login` first * if you haven't). */ import fs from 'fs'; import path from 'path'; import { chromium } from 'playwright'; import { createLogger } from '../src/utils/logger.js'; import { MatrixClient, MatrixAuthError } from '../src/services/chat/matrix/client.js'; import { extractMatrixCredentials, loadCachedCredentials, } from '../src/services/chat/matrix/token-extractor.js'; import type { MatrixTimelineEvent } from '../src/services/chat/matrix/types.js'; const SESSION_DIR = path.resolve(process.cwd(), './data/browser-session'); interface Args { refresh: boolean; roomId: string | null; sendBody: string | null; sendRoomId: string | null; dump: boolean; sync: boolean; forceBackfill: boolean; dumpDir: string; maxPages: number; } function parseArgs(argv: string[]): Args { const out: Args = { refresh: false, roomId: null, sendBody: null, sendRoomId: null, dump: false, sync: false, forceBackfill: false, dumpDir: 'data/matrix-dump', maxPages: 200, }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === '--refresh') out.refresh = true; else if (a === '--room') out.roomId = argv[++i] ?? null; else if (a === '--dump') out.dump = true; else if (a === '--sync') out.sync = true; else if (a === '--force-backfill') out.forceBackfill = true; else if (a === '--out') out.dumpDir = argv[++i] ?? out.dumpDir; else if (a === '--max-pages') out.maxPages = Number(argv[++i] ?? '200'); else if (a === '--send') { out.sendRoomId = argv[++i] ?? null; out.sendBody = argv[++i] ?? null; } } return out; } function extOfContentType(ct: string): string { const base = ct.split(';')[0].trim().toLowerCase(); const map: Record = { 'image/jpeg': 'jpg', 'image/jpg': 'jpg', 'image/png': 'png', 'image/webp': 'webp', 'image/gif': 'gif', 'image/heic': 'heic', 'image/avif': 'avif', 'video/mp4': 'mp4', 'video/webm': 'webm', 'video/quicktime': 'mov', 'audio/mpeg': 'mp3', 'audio/ogg': 'ogg', 'application/octet-stream': 'bin', }; return map[base] ?? base.split('/')[1] ?? 'bin'; } function safeSlug(s: string): string { return s.replace(/[^A-Za-z0-9._-]/g, '_').slice(0, 80); } /** * Build a unique directory name for a room: `__`. * Many Reddit chat rooms share names like "Stunning" — without the suffix * they collide on disk and silently overwrite each other's manifests. */ function roomDirSlug(roomId: string, displayName: string | null): string { // Strip `!` and `:reddit.com` so the hash is short but stable. const idCore = roomId.replace(/^!/, '').split(':')[0].slice(0, 8); const base = displayName ? safeSlug(displayName) : 'room'; return `${base}__${idCore}`; } /** Run `worker` over `items` with at most `limit` in flight. Preserves order. */ async function pMap( items: T[], limit: number, worker: (item: T, index: number) => Promise, ): Promise { const results: R[] = new Array(items.length); let next = 0; const runners = Array.from({ length: Math.min(limit, items.length) }, async () => { while (true) { const idx = next++; if (idx >= items.length) return; results[idx] = await worker(items[idx], idx); } }); await Promise.all(runners); return results; } const ROOM_CONCURRENCY = 6; const IMAGE_CONCURRENCY = 6; async function paginateAllMessages( client: MatrixClient, roomId: string, maxPages: number, ): Promise { const all: MatrixTimelineEvent[] = []; let from: string | undefined = undefined; for (let i = 0; i < maxPages; i++) { const res = await client.getRoomMessages(roomId, { from, limit: 100, dir: 'b' }); all.push(...res.chunk); if (!res.end || res.end === from || res.chunk.length === 0) break; from = res.end; } return all; } interface SyncState { nextBatch: string | null; updatedAt: number; rooms: Record; } function loadSyncState(dir: string): SyncState | null { const p = path.join(dir, '_state.json'); if (!fs.existsSync(p)) return null; try { return JSON.parse(fs.readFileSync(p, 'utf-8')) as SyncState; } catch { return null; } } function saveSyncState(dir: string, state: SyncState): void { fs.writeFileSync(path.join(dir, '_state.json'), JSON.stringify(state, null, 2), 'utf-8'); } function eventToManifestLine(e: MatrixTimelineEvent, userId: string): string { return JSON.stringify({ event_id: e.event_id, ts: e.origin_server_ts, iso: new Date(e.origin_server_ts).toISOString(), sender: e.sender, isSelf: e.sender === userId, msgtype: e.content.msgtype ?? null, body: e.content.body ?? null, mxc: typeof e.content.url === 'string' ? e.content.url : null, }); } function isImageEvent(e: MatrixTimelineEvent): boolean { if (e.type !== 'm.room.message') return false; return e.content.msgtype === 'm.image' || typeof e.content.url === 'string'; } async function downloadEventImages( client: MatrixClient, events: MatrixTimelineEvent[], roomDir: string, logger: ReturnType, ): Promise<{ downloaded: number; failed: number }> { const targets = events.filter(isImageEvent); let downloaded = 0; let failed = 0; await pMap(targets, IMAGE_CONCURRENCY, async (evt) => { const mxc = evt.content.url as string; try { // Skip download if any file for this event already exists (delta re-runs). const prefix = `${evt.origin_server_ts}_${safeSlug(evt.event_id)}.`; const existing = fs.readdirSync(roomDir).find((f) => f.startsWith(prefix)); if (existing) return; const media = await client.downloadMedia(mxc); if (!media) { failed++; return; } const ext = extOfContentType(media.contentType); fs.writeFileSync(path.join(roomDir, prefix + ext), media.bytes); downloaded++; } catch (err) { logger.warn(`[media] failed ${mxc}: ${(err as Error).message}`); failed++; } }); return { downloaded, failed }; } async function resolveRoomDisplayName( client: MatrixClient, roomId: string, logger: ReturnType, ): Promise { // 1. Explicit name event try { const name = await client.getRoomName(roomId); if (name) return name; } catch (err) { logger.debug(`[name] getRoomName failed for ${roomId}: ${(err as Error).message}`); } // 2. Fall back to other joined members (DM heuristic) try { const members = await client.getJoinedMembers(roomId); const others = Object.keys(members).filter((m) => m !== client.userId); if (others.length > 0) { return others.map((m) => m.match(/^@([^:]+):/)?.[1] ?? m).join(', '); } } catch (err) { logger.debug(`[name] getJoinedMembers failed for ${roomId}: ${(err as Error).message}`); } return null; } async function backfillAll( client: MatrixClient, args: Args, logger: ReturnType, outDir: string, ): Promise { const t0 = Date.now(); // One /sync up-front: gives us heroes / m.room.name for every active room // and (crucially) the cursor for the future delta runs. This lets us avoid // per-room name lookups for the common case. const seed = await client.sync({ timeoutMs: 0 }); const seedJoined = seed.rooms?.join ?? {}; const seedNames = new Map(); for (const [roomId, room] of Object.entries(seedJoined)) { const nameEvent = room.state?.events?.find((e) => e.type === 'm.room.name'); if (nameEvent && typeof nameEvent.content.name === 'string') { seedNames.set(roomId, nameEvent.content.name as string); continue; } const heroes = (room.summary?.['m.heroes'] ?? []).filter((h) => h !== client.userId); if (heroes.length > 0) { seedNames.set(roomId, heroes.map((h) => h.match(/^@([^:]+):/)?.[1] ?? h).join(', ')); } else { seedNames.set(roomId, null); } } const joined = await client.getJoinedRooms(); logger.info( `[backfill] /joined_rooms → ${joined.length} rooms (${seedNames.size} pre-named from /sync)`, ); const state: SyncState = { nextBatch: null, updatedAt: Date.now(), rooms: {} }; let totalEvents = 0; let totalImages = 0; let totalDownloaded = 0; let totalFailed = 0; let done = 0; await pMap(joined, ROOM_CONCURRENCY, async (roomId) => { let displayName: string | null; if (seedNames.has(roomId)) { displayName = seedNames.get(roomId) ?? null; } else { displayName = await resolveRoomDisplayName(client, roomId, logger); } const slug = roomDirSlug(roomId, displayName); const roomDir = path.join(outDir, slug); fs.mkdirSync(roomDir, { recursive: true }); const events = await paginateAllMessages(client, roomId, args.maxPages); const messages = events.filter((e) => e.type === 'm.room.message'); const lines = messages .slice() .reverse() .map((e) => eventToManifestLine(e, client.userId)); fs.writeFileSync( path.join(roomDir, 'messages.jsonl'), lines.join('\n') + (lines.length ? '\n' : ''), 'utf-8', ); const { downloaded, failed } = await downloadEventImages(client, events, roomDir, logger); const imgCount = events.filter(isImageEvent).length; totalEvents += events.length; totalImages += imgCount; totalDownloaded += downloaded; totalFailed += failed; state.rooms[roomId] = { slug, displayName }; done++; logger.info( `[backfill ${done}/${joined.length}] ${slug}: events=${events.length} msgs=${messages.length} imgs=${imgCount} ok=${downloaded} fail=${failed}`, ); }); // The seed /sync's cursor is the right one to delta from — all events // returned in that response are already on disk via the per-room walk. state.nextBatch = seed.next_batch; state.updatedAt = Date.now(); saveSyncState(outDir, state); const elapsedSec = ((Date.now() - t0) / 1000).toFixed(1); logger.info( `[backfill] DONE in ${elapsedSec}s — rooms=${joined.length} events=${totalEvents} imgs=${totalImages} ok=${totalDownloaded} fail=${totalFailed}`, ); logger.info(`[backfill] cursor stored: ${state.nextBatch?.slice(0, 32)}…`); return state; } async function deltaSync( client: MatrixClient, args: Args, logger: ReturnType, outDir: string, state: SyncState, ): Promise { logger.info(`[delta] resuming from cursor ${state.nextBatch?.slice(0, 32)}…`); const sync = await client.sync({ since: state.nextBatch ?? undefined, timeoutMs: 0 }); const joined = sync.rooms?.join ?? {}; const roomIds = Object.keys(joined); if (roomIds.length === 0) { logger.info('[delta] no new events since last sync'); state.nextBatch = sync.next_batch; state.updatedAt = Date.now(); saveSyncState(outDir, state); return state; } let totalNew = 0; let totalImages = 0; let totalDownloaded = 0; for (const roomId of roomIds) { const room = joined[roomId]; const events = room.timeline?.events ?? []; if (events.length === 0) continue; // Discover (or rediscover) display name for previously-unseen rooms. let info = state.rooms[roomId]; if (!info) { const displayName = await resolveRoomDisplayName(client, roomId, logger); info = { slug: roomDirSlug(roomId, displayName), displayName }; state.rooms[roomId] = info; logger.info(`[delta] new room discovered: ${info.slug}`); } const roomDir = path.join(outDir, info.slug); fs.mkdirSync(roomDir, { recursive: true }); const messageEvents = events.filter((e) => e.type === 'm.room.message'); if (messageEvents.length > 0) { const lines = messageEvents.map((e) => eventToManifestLine(e, client.userId)); fs.appendFileSync(path.join(roomDir, 'messages.jsonl'), lines.join('\n') + '\n', 'utf-8'); } const { downloaded } = await downloadEventImages(client, events, roomDir, logger); const imgCount = events.filter(isImageEvent).length; totalNew += messageEvents.length; totalImages += imgCount; totalDownloaded += downloaded; logger.info( `[delta] ${info.slug}: +${messageEvents.length} msgs +${imgCount} imgs (downloaded ${downloaded})`, ); } state.nextBatch = sync.next_batch; state.updatedAt = Date.now(); saveSyncState(outDir, state); logger.info( `[delta] DONE — rooms_touched=${roomIds.length} new_msgs=${totalNew} new_imgs=${totalImages} downloaded=${totalDownloaded}`, ); logger.info(`[delta] cursor updated: ${state.nextBatch?.slice(0, 32)}…`); return state; } async function syncAll( client: MatrixClient, args: Args, logger: ReturnType, ) { const outDir = path.resolve(process.cwd(), args.dumpDir); if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true }); logger.info(`[sync] output dir: ${outDir}`); const existing = loadSyncState(outDir); if (!existing?.nextBatch || args.forceBackfill) { if (args.forceBackfill) logger.info('[sync] --force-backfill set: ignoring existing cursor'); else logger.info('[sync] no cursor on disk → running full backfill'); await backfillAll(client, args, logger, outDir); } else { await deltaSync(client, args, logger, outDir, existing); } } async function dumpAll(client: MatrixClient, args: Args, logger: ReturnType) { const outDir = path.resolve(process.cwd(), args.dumpDir); if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true }); logger.info(`[dump] output dir: ${outDir}`); const rooms = await client.listRooms(); logger.info(`[dump] ${rooms.length} rooms to walk`); const overallSummary: Array<{ roomId: string; displayName: string | null; eventCount: number; messageCount: number; imageCount: number; downloaded: number; failed: number; }> = []; let totalImages = 0; let totalDownloaded = 0; let totalFailed = 0; for (const room of rooms) { const slug = safeSlug(room.displayName ?? room.roomId); const roomDir = path.join(outDir, slug); fs.mkdirSync(roomDir, { recursive: true }); const events = await paginateAllMessages(client, room.roomId, args.maxPages); const messages = events.filter((e) => e.type === 'm.room.message'); const images = messages.filter( (e) => e.content.msgtype === 'm.image' || typeof e.content.url === 'string', ); // Write the messages.jsonl manifest. const manifestPath = path.join(roomDir, 'messages.jsonl'); const lines = messages .slice() .reverse() // chronological for human reading .map((e) => JSON.stringify({ event_id: e.event_id, ts: e.origin_server_ts, iso: new Date(e.origin_server_ts).toISOString(), sender: e.sender, isSelf: e.sender === client.userId, msgtype: e.content.msgtype ?? null, body: e.content.body ?? null, mxc: typeof e.content.url === 'string' ? e.content.url : null, }), ); fs.writeFileSync(manifestPath, lines.join('\n') + '\n', 'utf-8'); let downloaded = 0; let failed = 0; for (const evt of images) { const mxc = evt.content.url as string | undefined; if (!mxc) continue; try { const media = await client.downloadMedia(mxc); if (!media) { failed++; continue; } const ext = extOfContentType(media.contentType); const fname = `${evt.origin_server_ts}_${safeSlug(evt.event_id)}.${ext}`; fs.writeFileSync(path.join(roomDir, fname), media.bytes); downloaded++; } catch (err) { logger.warn(`[dump] failed ${mxc} in ${room.roomId}: ${(err as Error).message}`); failed++; } } totalImages += images.length; totalDownloaded += downloaded; totalFailed += failed; logger.info( `[dump] ${slug}: events=${events.length} msgs=${messages.length} imgs=${images.length} ok=${downloaded} fail=${failed}`, ); overallSummary.push({ roomId: room.roomId, displayName: room.displayName, eventCount: events.length, messageCount: messages.length, imageCount: images.length, downloaded, failed, }); } fs.writeFileSync( path.join(outDir, '_summary.json'), JSON.stringify( { capturedAt: new Date().toISOString(), userId: client.userId, homeserver: client.homeserver, rooms: overallSummary, totals: { rooms: rooms.length, images: totalImages, downloaded: totalDownloaded, failed: totalFailed, }, }, null, 2, ), 'utf-8', ); logger.info( `[dump] DONE — rooms=${rooms.length} images=${totalImages} downloaded=${totalDownloaded} failed=${totalFailed}`, ); logger.info(`[dump] manifest: ${path.join(outDir, '_summary.json')}`); } async function getCredentials(logger: ReturnType, refresh: boolean) { if (!refresh) { const cached = await loadCachedCredentials(); if (cached) { logger.info(`[matrix] using cached credentials (captured ${ageStr(cached.capturedAt)} ago)`); return cached; } } logger.info('[matrix] opening browser to capture access token...'); const context = await chromium.launchPersistentContext(SESSION_DIR, { headless: false, viewport: { width: 1280, height: 720 }, }); try { return await extractMatrixCredentials(context, logger); } finally { await context.close(); } } function ageStr(ts: number): string { const sec = Math.max(0, Math.floor((Date.now() - ts) / 1000)); if (sec < 60) return `${sec}s`; if (sec < 3600) return `${Math.floor(sec / 60)}m`; if (sec < 86400) return `${Math.floor(sec / 3600)}h`; return `${Math.floor(sec / 86400)}d`; } async function main() { const logger = createLogger(); const args = parseArgs(process.argv.slice(2)); let creds = await getCredentials(logger, args.refresh); let client = new MatrixClient(creds, logger); const reExtract = async () => { logger.warn('[matrix] token rejected — re-extracting'); creds = await getCredentials(logger, true); client = new MatrixClient(creds, logger); }; // --- sync mode (backfill on first run, delta thereafter) ----------- if (args.sync) { try { await syncAll(client, args, logger); } catch (err) { if (err instanceof MatrixAuthError) { await reExtract(); await syncAll(client, args, logger); } else throw err; } return; } // --- dump mode (snapshot of /sync — recent rooms only) ------------- if (args.dump) { try { await dumpAll(client, args, logger); } catch (err) { if (err instanceof MatrixAuthError) { await reExtract(); await dumpAll(client, args, logger); } else throw err; } return; } // --- send mode ----------------------------------------------------- if (args.sendRoomId && args.sendBody) { try { const eventId = await client.sendTextMessage(args.sendRoomId, args.sendBody); console.log(`Sent: event_id=${eventId}`); } catch (err) { if (err instanceof MatrixAuthError) { await reExtract(); const eventId = await client.sendTextMessage(args.sendRoomId, args.sendBody); console.log(`Sent (after re-auth): event_id=${eventId}`); } else throw err; } return; } // --- single-room history mode -------------------------------------- if (args.roomId) { let res; try { res = await client.getRoomMessages(args.roomId, { limit: 30 }); } catch (err) { if (err instanceof MatrixAuthError) { await reExtract(); res = await client.getRoomMessages(args.roomId, { limit: 30 }); } else throw err; } console.log(`\nRoom ${args.roomId} — ${res.chunk.length} events (most recent first):\n`); for (const event of res.chunk) { if (event.type !== 'm.room.message') { console.log(` [${event.type}] from ${event.sender}`); continue; } const who = event.sender === client.userId ? 'ME' : event.sender; const when = new Date(event.origin_server_ts).toISOString(); const body = (event.content.body as string | undefined) ?? ''; const mediaTag = event.content.url ? ` [media: ${event.content.url}]` : ''; console.log(` ${when} ${who}: ${body.slice(0, 120)}${body.length > 120 ? '…' : ''}${mediaTag}`); } return; } // --- list rooms (default) ------------------------------------------ let rooms; try { rooms = await client.listRooms(); } catch (err) { if (err instanceof MatrixAuthError) { await reExtract(); rooms = await client.listRooms(); } else throw err; } console.log(`\nLogged in as: ${client.userId}`); console.log(`Homeserver: ${client.homeserver}`); console.log(`Rooms: ${rooms.length}\n`); for (const r of rooms.slice(0, 50)) { const unread = r.unreadCount > 0 ? ` (${r.unreadCount} unread)` : ''; const last = r.lastEvent ? `${new Date(r.lastEvent.origin_server_ts).toISOString()} ${(r.lastEvent.content.body as string | undefined)?.slice(0, 80) ?? `[${r.lastEvent.type}]`}` : ''; console.log(`• ${r.displayName ?? r.roomId}${unread}`); console.log(` roomId: ${r.roomId}`); console.log(` last: ${last}\n`); } } main().catch((err) => { console.error('FATAL:', err); process.exit(1); });