Restructure chat service, web routes, and add test suite

Split monolithic chat service into modular structure with Matrix client
support, extract web server into per-resource route modules, introduce
bootstrap/main-loop separation, and add comprehensive vitest coverage
across services, repositories, and utilities. Adds project docs and
CLAUDE.md agent instructions.
This commit is contained in:
jjbrunton committed 2026-05-21 01:32:10 +01:00
1 parent dfa0580859
commit 633bb0ec04
84 files changed
+19278 -3888

No files matched your search

-249
View File
@@ -1,249 +0,0 @@
/**
* Diagnostic script: inspect the actual DOM structure of Reddit chat
* to understand how media images are rendered.
*
* Usage: npx tsx scripts/inspect-chat-dom.ts
*/
import { chromium } from 'playwright';
import path from 'path';
async function main() {
const sessionDir = '/tmp/reddit-session-copy';
console.log('Launching browser with session copy...');
const context = await chromium.launchPersistentContext(sessionDir, {
headless: false,
viewport: { width: 1280, height: 720 },
args: ['--no-sandbox', '--disable-blink-features=AutomationControlled'],
});
const page = await context.newPage();
await page.goto('https://chat.reddit.com/', {
waitUntil: 'domcontentloaded',
timeout: 30000,
});
await page.waitForTimeout(5000);
console.log('\n=== Finding chat rooms ===');
// Find the first chat room and click into it
const firstChatUrl = await page.evaluate(`
(function() {
var rsApp = document.querySelector('rs-app');
if (!rsApp || !rsApp.shadowRoot) return null;
// Look for room links
function findLinks(root) {
if (!root) return [];
var links = Array.from(root.querySelectorAll('a[href*="/room/"]'));
var allElements = root.querySelectorAll('*');
for (var i = 0; i < allElements.length; i++) {
if (allElements[i].shadowRoot) {
links = links.concat(findLinks(allElements[i].shadowRoot));
}
}
return links;
}
var links = findLinks(rsApp.shadowRoot);
if (links.length === 0) return null;
return links[0].href;
})()
`);
console.log('First chat URL:', firstChatUrl);
if (firstChatUrl) {
await page.goto(firstChatUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
await page.waitForTimeout(3000);
}
console.log('\n=== Inspecting message area DOM structure ===');
const domInfo = await page.evaluate(`
(function() {
var results = {
rsApp: false,
rsRoom: false,
rsRoomShadow: false,
customElements: [],
allImages: [],
messageStructure: [],
shadowRootChain: [],
};
var rsApp = document.querySelector('rs-app');
if (!rsApp || !rsApp.shadowRoot) {
results.error = 'No rs-app shadow root found';
return results;
}
results.rsApp = true;
var rsRoom = rsApp.shadowRoot.querySelector('rs-room');
if (!rsRoom) {
results.error = 'No rs-room found';
return results;
}
results.rsRoom = true;
results.rsRoomShadow = !!rsRoom.shadowRoot;
// Find ALL custom elements and their shadow roots recursively
function mapShadowDom(root, depth, path) {
if (!root || depth > 8) return;
var allEls = root.querySelectorAll('*');
for (var i = 0; i < allEls.length; i++) {
var el = allEls[i];
var tag = el.tagName.toLowerCase();
if (tag.includes('-') || el.shadowRoot) {
var entry = {
tag: tag,
depth: depth,
path: path + ' > ' + tag,
hasShadow: !!el.shadowRoot,
childCount: el.childElementCount,
className: (el.className || '').toString().slice(0, 100),
};
results.customElements.push(entry);
if (el.shadowRoot) {
mapShadowDom(el.shadowRoot, depth + 1, path + ' > ' + tag + ' [shadow]');
}
}
}
}
var searchRoot = rsRoom.shadowRoot || rsRoom;
mapShadowDom(searchRoot, 0, 'rs-room');
// Find ALL images recursively through shadow DOMs
function findAllImages(root, depth, path) {
if (!root || depth > 8) return;
var imgs = root.querySelectorAll('img');
for (var i = 0; i < imgs.length; i++) {
var img = imgs[i];
results.allImages.push({
src: (img.getAttribute('src') || '').slice(0, 200),
alt: img.getAttribute('alt') || '',
width: img.width,
height: img.height,
naturalWidth: img.naturalWidth,
naturalHeight: img.naturalHeight,
className: (img.className || '').toString().slice(0, 100),
parentTag: img.parentElement ? img.parentElement.tagName.toLowerCase() : 'none',
parentClass: img.parentElement ? (img.parentElement.className || '').toString().slice(0, 100) : '',
grandparentTag: img.parentElement && img.parentElement.parentElement ? img.parentElement.parentElement.tagName.toLowerCase() : 'none',
path: path,
depth: depth,
});
}
var allEls = root.querySelectorAll('*');
for (var j = 0; j < allEls.length; j++) {
if (allEls[j].shadowRoot) {
findAllImages(allEls[j].shadowRoot, depth + 1, path + ' > ' + allEls[j].tagName.toLowerCase() + '[shadow]');
}
}
}
findAllImages(searchRoot, 0, 'rs-room');
// Try to understand message structure: look for repeated patterns
// Walk the shadow DOM and find elements that look like messages
function findMessageElements(root, depth, path) {
if (!root || depth > 8) return;
var allEls = root.querySelectorAll('*');
for (var i = 0; i < allEls.length; i++) {
var el = allEls[i];
var tag = el.tagName.toLowerCase();
var cls = (el.className || '').toString().toLowerCase();
var text = (el.textContent || '').trim();
// Look for elements that contain message-like content
if (tag.includes('message') || cls.includes('message') ||
tag.includes('msg') || cls.includes('msg') ||
tag.includes('chat') || cls.includes('bubble')) {
results.messageStructure.push({
tag: tag,
className: cls.slice(0, 150),
path: path,
depth: depth,
textSnippet: text.slice(0, 100),
childCount: el.childElementCount,
hasImages: el.querySelectorAll('img').length,
innerHTML: el.innerHTML.slice(0, 300),
});
}
if (el.shadowRoot) {
findMessageElements(el.shadowRoot, depth + 1, path + ' > ' + tag + '[shadow]');
}
}
}
findMessageElements(searchRoot, 0, 'rs-room');
return results;
})()
`);
console.log('\n--- Shadow DOM Structure ---');
console.log('rs-app:', domInfo.rsApp);
console.log('rs-room:', domInfo.rsRoom);
console.log('rs-room has shadow:', domInfo.rsRoomShadow);
if (domInfo.error) console.log('ERROR:', domInfo.error);
console.log('\n--- Custom Elements with Shadow Roots ---');
for (const el of domInfo.customElements || []) {
console.log(`${' '.repeat(el.depth)}${el.tag} (shadow: ${el.hasShadow}, children: ${el.childCount}) ${el.className ? '[class: ' + el.className + ']' : ''}`);
}
console.log('\n--- All Images Found ---');
for (const img of domInfo.allImages || []) {
console.log(` [depth=${img.depth}] ${img.src}`);
console.log(` size: ${img.width}x${img.height} (natural: ${img.naturalWidth}x${img.naturalHeight})`);
console.log(` class: ${img.className}`);
console.log(` parent: <${img.parentTag} class="${img.parentClass}">`);
console.log(` path: ${img.path}`);
console.log('');
}
console.log('\n--- Message-like Elements ---');
for (const msg of domInfo.messageStructure || []) {
console.log(` [depth=${msg.depth}] <${msg.tag}> images=${msg.hasImages}`);
console.log(` class: ${msg.className}`);
console.log(` text: ${msg.textSnippet}`);
console.log(` path: ${msg.path}`);
console.log(` html: ${msg.innerHTML}`);
console.log('');
}
// Also dump a broader view - find what's directly inside rs-room shadow
console.log('\n=== Direct children of message area ===');
const directChildren = await page.evaluate(`
(function() {
var rsApp = document.querySelector('rs-app');
if (!rsApp || !rsApp.shadowRoot) return 'no rs-app';
var rsRoom = rsApp.shadowRoot.querySelector('rs-room');
if (!rsRoom) return 'no rs-room';
var root = rsRoom.shadowRoot || rsRoom;
var children = [];
for (var i = 0; i < root.children.length; i++) {
var child = root.children[i];
children.push({
tag: child.tagName.toLowerCase(),
className: (child.className || '').toString().slice(0, 200),
id: child.id || '',
childCount: child.childElementCount,
hasShadow: !!child.shadowRoot,
textSnippet: (child.textContent || '').trim().slice(0, 150),
});
}
return children;
})()
`);
console.log(JSON.stringify(directChildren, null, 2));
console.log('\nDone. Closing browser...');
await context.close();
}
main().catch(console.error);
+217
View File
@@ -0,0 +1,217 @@
/**
* Matrix network sniffer.
*
* Watches every request to matrix.redditspace.com made by chat.reddit.com,
* captures method/URL/body for both request and response, and writes the
* sequence to data/matrix-sniff-<timestamp>.json when you exit (Ctrl-C).
*
* Usage:
* tsx scripts/matrix-sniff.ts
*
* Then in the browser that pops up: log in if needed → click "Start chat"
* → pick a recipient → send a message → come back here and press Ctrl-C.
*
* The script does NOT initiate any Matrix calls itself. It only observes
* what the UI sends, so there's no risk of an accidental outbound message.
*/
import fs from 'fs';
import path from 'path';
import { chromium, type Request, type Response } from 'playwright';
import { createLogger } from '../src/utils/logger.js';
const SESSION_DIR = path.resolve(process.cwd(), './data/browser-session');
const OUT_DIR = path.resolve(process.cwd(), './data');
const MATRIX_HOST_RE = /matrix\.(redditspace|reddit)\.com$/i;
interface CapturedCall {
index: number;
startedAt: string;
method: string;
url: string;
pathLabel: string;
requestHeaders: Record<string, string>;
requestBody: string | null;
requestBodyJson: unknown | null;
responseStatus: number | null;
responseHeaders: Record<string, string> | null;
responseBody: string | null;
responseBodyJson: unknown | null;
durationMs: number | null;
}
/** Best-effort short label so the live log is readable. */
function labelFor(method: string, urlStr: string): string {
try {
const url = new URL(urlStr);
// Strip /_matrix/client/v3 prefix for readability.
let p = url.pathname.replace(/^\/_matrix\/(client|media)\/(v3|v1)/, '');
// Truncate long IDs in path segments.
p = p
.split('/')
.map((seg) => (seg.length > 24 ? seg.slice(0, 12) + '…' + seg.slice(-6) : seg))
.join('/');
return `${method} ${p}`;
} catch {
return `${method} ${urlStr}`;
}
}
function safeParseJson(s: string | null): unknown | null {
if (!s) return null;
try {
return JSON.parse(s);
} catch {
return null;
}
}
async function main() {
const logger = createLogger();
if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
const outFile = path.join(OUT_DIR, `matrix-sniff-${new Date().toISOString().replace(/[:.]/g, '-')}.json`);
logger.info('[sniff] launching browser (using persistent session)...');
const context = await chromium.launchPersistentContext(SESSION_DIR, {
headless: false,
viewport: { width: 1280, height: 800 },
});
const calls: CapturedCall[] = [];
// Map request → index so we can hydrate it when the response arrives.
const reqIndex = new WeakMap<Request, number>();
const reqStart = new WeakMap<Request, number>();
let serial = 0;
const onRequest = (request: Request) => {
try {
const u = new URL(request.url());
if (!MATRIX_HOST_RE.test(u.hostname)) return;
} catch {
return;
}
const idx = serial++;
reqIndex.set(request, idx);
reqStart.set(request, Date.now());
const body = request.postData();
const call: CapturedCall = {
index: idx,
startedAt: new Date().toISOString(),
method: request.method(),
url: request.url(),
pathLabel: labelFor(request.method(), request.url()),
requestHeaders: scrubHeaders(request.headers()),
requestBody: body ?? null,
requestBodyJson: safeParseJson(body ?? null),
responseStatus: null,
responseHeaders: null,
responseBody: null,
responseBodyJson: null,
durationMs: null,
};
calls[idx] = call;
logger.info(`[sniff] #${idx} → ${call.pathLabel}`);
if (body && body.length < 600) {
logger.info(` req body: ${body}`);
} else if (body) {
logger.info(` req body: ${body.length}B (truncated in log; full saved to file)`);
}
};
const onResponse = async (response: Response) => {
const request = response.request();
const idx = reqIndex.get(request);
if (idx === undefined) return;
const call = calls[idx];
if (!call) return;
call.responseStatus = response.status();
call.responseHeaders = scrubHeaders(response.headers());
const started = reqStart.get(request);
call.durationMs = started ? Date.now() - started : null;
// Only capture text bodies, and skip large binaries.
const ct = (response.headers()['content-type'] || '').toLowerCase();
if (ct.includes('json') || ct.includes('text')) {
try {
const text = await response.text();
if (text.length <= 100_000) {
call.responseBody = text;
call.responseBodyJson = safeParseJson(text);
} else {
call.responseBody = `[truncated: ${text.length}B ${ct}]`;
}
} catch (err) {
call.responseBody = `[read error: ${(err as Error).message}]`;
}
} else if (ct) {
call.responseBody = `[binary: ${ct}]`;
}
logger.info(`[sniff] #${idx} ← ${call.responseStatus} (${call.durationMs}ms)`);
};
context.on('request', onRequest);
context.on('response', onResponse);
const page = context.pages()[0] ?? (await context.newPage());
await page.goto('https://chat.reddit.com/', { waitUntil: 'domcontentloaded' });
logger.info('');
logger.info('═══════════════════════════════════════════════════════════');
logger.info(' Ready. In the browser:');
logger.info(' 1. Click "Start chat" (or the + icon)');
logger.info(' 2. Pick a recipient and send a short message');
logger.info(' 3. Come back here and press Ctrl-C to save & exit.');
logger.info('═══════════════════════════════════════════════════════════');
logger.info('');
const flush = async () => {
logger.info(`[sniff] writing ${calls.length} calls to ${outFile}`);
fs.writeFileSync(outFile, JSON.stringify({ capturedAt: new Date().toISOString(), calls }, null, 2));
try {
await context.close();
} catch {
/* ignore */
}
};
// Save on Ctrl-C.
process.on('SIGINT', async () => {
logger.info('[sniff] SIGINT — flushing...');
await flush();
process.exit(0);
});
// Also save if the user closes the browser window.
context.on('close', async () => {
logger.info('[sniff] browser closed — flushing...');
await flush();
process.exit(0);
});
}
/**
* Remove the bearer token before writing to disk — the captured JSON file
* is going to sit in the repo's data dir, no need to leave creds in it.
*/
function scrubHeaders(headers: Record<string, string>): Record<string, string> {
const out: Record<string, string> = {};
for (const [k, v] of Object.entries(headers)) {
if (k.toLowerCase() === 'authorization') {
out[k] = v.toLowerCase().startsWith('bearer ') ? 'Bearer <redacted>' : '<redacted>';
} else if (k.toLowerCase() === 'cookie') {
out[k] = '<redacted>';
} else {
out[k] = v;
}
}
return out;
}
main().catch((err) => {
console.error('FATAL:', err);
process.exit(1);
});
+686
View File
@@ -0,0 +1,686 @@
/**
* Matrix-API prototype CLI.
*
* Usage:
* tsx scripts/matrix-test.ts # capture token + list rooms
* tsx scripts/matrix-test.ts --refresh # force re-extract token
* tsx scripts/matrix-test.ts --room <id> # dump message history for a room
* tsx scripts/matrix-test.ts --send <id> "hello" # send a test message
* tsx scripts/matrix-test.ts --dump # download every room's full
* # history + images to disk
*
* Reuses the bot's persistent browser session at ./data/browser-session,
* so you must already be logged into Reddit there (run `npm run login` first
* if you haven't).
*/
import fs from 'fs';
import path from 'path';
import { chromium } from 'playwright';
import { createLogger } from '../src/utils/logger.js';
import { MatrixClient, MatrixAuthError } from '../src/services/chat/matrix/client.js';
import {
extractMatrixCredentials,
loadCachedCredentials,
} from '../src/services/chat/matrix/token-extractor.js';
import type { MatrixTimelineEvent } from '../src/services/chat/matrix/types.js';
const SESSION_DIR = path.resolve(process.cwd(), './data/browser-session');
interface Args {
refresh: boolean;
roomId: string | null;
sendBody: string | null;
sendRoomId: string | null;
dump: boolean;
sync: boolean;
forceBackfill: boolean;
dumpDir: string;
maxPages: number;
}
function parseArgs(argv: string[]): Args {
const out: Args = {
refresh: false,
roomId: null,
sendBody: null,
sendRoomId: null,
dump: false,
sync: false,
forceBackfill: false,
dumpDir: 'data/matrix-dump',
maxPages: 200,
};
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a === '--refresh') out.refresh = true;
else if (a === '--room') out.roomId = argv[++i] ?? null;
else if (a === '--dump') out.dump = true;
else if (a === '--sync') out.sync = true;
else if (a === '--force-backfill') out.forceBackfill = true;
else if (a === '--out') out.dumpDir = argv[++i] ?? out.dumpDir;
else if (a === '--max-pages') out.maxPages = Number(argv[++i] ?? '200');
else if (a === '--send') {
out.sendRoomId = argv[++i] ?? null;
out.sendBody = argv[++i] ?? null;
}
}
return out;
}
function extOfContentType(ct: string): string {
const base = ct.split(';')[0].trim().toLowerCase();
const map: Record<string, string> = {
'image/jpeg': 'jpg',
'image/jpg': 'jpg',
'image/png': 'png',
'image/webp': 'webp',
'image/gif': 'gif',
'image/heic': 'heic',
'image/avif': 'avif',
'video/mp4': 'mp4',
'video/webm': 'webm',
'video/quicktime': 'mov',
'audio/mpeg': 'mp3',
'audio/ogg': 'ogg',
'application/octet-stream': 'bin',
};
return map[base] ?? base.split('/')[1] ?? 'bin';
}
function safeSlug(s: string): string {
return s.replace(/[^A-Za-z0-9._-]/g, '_').slice(0, 80);
}
/**
* Build a unique directory name for a room: `<displayName>__<short room-id>`.
* Many Reddit chat rooms share names like "Stunning" — without the suffix
* they collide on disk and silently overwrite each other's manifests.
*/
function roomDirSlug(roomId: string, displayName: string | null): string {
// Strip `!` and `:reddit.com` so the hash is short but stable.
const idCore = roomId.replace(/^!/, '').split(':')[0].slice(0, 8);
const base = displayName ? safeSlug(displayName) : 'room';
return `${base}__${idCore}`;
}
/** Run `worker` over `items` with at most `limit` in flight. Preserves order. */
async function pMap<T, R>(
items: T[],
limit: number,
worker: (item: T, index: number) => Promise<R>,
): Promise<R[]> {
const results: R[] = new Array(items.length);
let next = 0;
const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
while (true) {
const idx = next++;
if (idx >= items.length) return;
results[idx] = await worker(items[idx], idx);
}
});
await Promise.all(runners);
return results;
}
const ROOM_CONCURRENCY = 6;
const IMAGE_CONCURRENCY = 6;
async function paginateAllMessages(
client: MatrixClient,
roomId: string,
maxPages: number,
): Promise<MatrixTimelineEvent[]> {
const all: MatrixTimelineEvent[] = [];
let from: string | undefined = undefined;
for (let i = 0; i < maxPages; i++) {
const res = await client.getRoomMessages(roomId, { from, limit: 100, dir: 'b' });
all.push(...res.chunk);
if (!res.end || res.end === from || res.chunk.length === 0) break;
from = res.end;
}
return all;
}
interface SyncState {
nextBatch: string | null;
updatedAt: number;
rooms: Record<string, { slug: string; displayName: string | null }>;
}
function loadSyncState(dir: string): SyncState | null {
const p = path.join(dir, '_state.json');
if (!fs.existsSync(p)) return null;
try {
return JSON.parse(fs.readFileSync(p, 'utf-8')) as SyncState;
} catch {
return null;
}
}
function saveSyncState(dir: string, state: SyncState): void {
fs.writeFileSync(path.join(dir, '_state.json'), JSON.stringify(state, null, 2), 'utf-8');
}
function eventToManifestLine(e: MatrixTimelineEvent, userId: string): string {
return JSON.stringify({
event_id: e.event_id,
ts: e.origin_server_ts,
iso: new Date(e.origin_server_ts).toISOString(),
sender: e.sender,
isSelf: e.sender === userId,
msgtype: e.content.msgtype ?? null,
body: e.content.body ?? null,
mxc: typeof e.content.url === 'string' ? e.content.url : null,
});
}
function isImageEvent(e: MatrixTimelineEvent): boolean {
if (e.type !== 'm.room.message') return false;
return e.content.msgtype === 'm.image' || typeof e.content.url === 'string';
}
async function downloadEventImages(
client: MatrixClient,
events: MatrixTimelineEvent[],
roomDir: string,
logger: ReturnType<typeof createLogger>,
): Promise<{ downloaded: number; failed: number }> {
const targets = events.filter(isImageEvent);
let downloaded = 0;
let failed = 0;
await pMap(targets, IMAGE_CONCURRENCY, async (evt) => {
const mxc = evt.content.url as string;
try {
// Skip download if any file for this event already exists (delta re-runs).
const prefix = `${evt.origin_server_ts}_${safeSlug(evt.event_id)}.`;
const existing = fs.readdirSync(roomDir).find((f) => f.startsWith(prefix));
if (existing) return;
const media = await client.downloadMedia(mxc);
if (!media) {
failed++;
return;
}
const ext = extOfContentType(media.contentType);
fs.writeFileSync(path.join(roomDir, prefix + ext), media.bytes);
downloaded++;
} catch (err) {
logger.warn(`[media] failed ${mxc}: ${(err as Error).message}`);
failed++;
}
});
return { downloaded, failed };
}
async function resolveRoomDisplayName(
client: MatrixClient,
roomId: string,
logger: ReturnType<typeof createLogger>,
): Promise<string | null> {
// 1. Explicit name event
try {
const name = await client.getRoomName(roomId);
if (name) return name;
} catch (err) {
logger.debug(`[name] getRoomName failed for ${roomId}: ${(err as Error).message}`);
}
// 2. Fall back to other joined members (DM heuristic)
try {
const members = await client.getJoinedMembers(roomId);
const others = Object.keys(members).filter((m) => m !== client.userId);
if (others.length > 0) {
return others.map((m) => m.match(/^@([^:]+):/)?.[1] ?? m).join(', ');
}
} catch (err) {
logger.debug(`[name] getJoinedMembers failed for ${roomId}: ${(err as Error).message}`);
}
return null;
}
async function backfillAll(
client: MatrixClient,
args: Args,
logger: ReturnType<typeof createLogger>,
outDir: string,
): Promise<SyncState> {
const t0 = Date.now();
// One /sync up-front: gives us heroes / m.room.name for every active room
// and (crucially) the cursor for the future delta runs. This lets us avoid
// per-room name lookups for the common case.
const seed = await client.sync({ timeoutMs: 0 });
const seedJoined = seed.rooms?.join ?? {};
const seedNames = new Map<string, string | null>();
for (const [roomId, room] of Object.entries(seedJoined)) {
const nameEvent = room.state?.events?.find((e) => e.type === 'm.room.name');
if (nameEvent && typeof nameEvent.content.name === 'string') {
seedNames.set(roomId, nameEvent.content.name as string);
continue;
}
const heroes = (room.summary?.['m.heroes'] ?? []).filter((h) => h !== client.userId);
if (heroes.length > 0) {
seedNames.set(roomId, heroes.map((h) => h.match(/^@([^:]+):/)?.[1] ?? h).join(', '));
} else {
seedNames.set(roomId, null);
}
}
const joined = await client.getJoinedRooms();
logger.info(
`[backfill] /joined_rooms → ${joined.length} rooms (${seedNames.size} pre-named from /sync)`,
);
const state: SyncState = { nextBatch: null, updatedAt: Date.now(), rooms: {} };
let totalEvents = 0;
let totalImages = 0;
let totalDownloaded = 0;
let totalFailed = 0;
let done = 0;
await pMap(joined, ROOM_CONCURRENCY, async (roomId) => {
let displayName: string | null;
if (seedNames.has(roomId)) {
displayName = seedNames.get(roomId) ?? null;
} else {
displayName = await resolveRoomDisplayName(client, roomId, logger);
}
const slug = roomDirSlug(roomId, displayName);
const roomDir = path.join(outDir, slug);
fs.mkdirSync(roomDir, { recursive: true });
const events = await paginateAllMessages(client, roomId, args.maxPages);
const messages = events.filter((e) => e.type === 'm.room.message');
const lines = messages
.slice()
.reverse()
.map((e) => eventToManifestLine(e, client.userId));
fs.writeFileSync(
path.join(roomDir, 'messages.jsonl'),
lines.join('\n') + (lines.length ? '\n' : ''),
'utf-8',
);
const { downloaded, failed } = await downloadEventImages(client, events, roomDir, logger);
const imgCount = events.filter(isImageEvent).length;
totalEvents += events.length;
totalImages += imgCount;
totalDownloaded += downloaded;
totalFailed += failed;
state.rooms[roomId] = { slug, displayName };
done++;
logger.info(
`[backfill ${done}/${joined.length}] ${slug}: events=${events.length} msgs=${messages.length} imgs=${imgCount} ok=${downloaded} fail=${failed}`,
);
});
// The seed /sync's cursor is the right one to delta from — all events
// returned in that response are already on disk via the per-room walk.
state.nextBatch = seed.next_batch;
state.updatedAt = Date.now();
saveSyncState(outDir, state);
const elapsedSec = ((Date.now() - t0) / 1000).toFixed(1);
logger.info(
`[backfill] DONE in ${elapsedSec}s — rooms=${joined.length} events=${totalEvents} imgs=${totalImages} ok=${totalDownloaded} fail=${totalFailed}`,
);
logger.info(`[backfill] cursor stored: ${state.nextBatch?.slice(0, 32)}…`);
return state;
}
async function deltaSync(
client: MatrixClient,
args: Args,
logger: ReturnType<typeof createLogger>,
outDir: string,
state: SyncState,
): Promise<SyncState> {
logger.info(`[delta] resuming from cursor ${state.nextBatch?.slice(0, 32)}…`);
const sync = await client.sync({ since: state.nextBatch ?? undefined, timeoutMs: 0 });
const joined = sync.rooms?.join ?? {};
const roomIds = Object.keys(joined);
if (roomIds.length === 0) {
logger.info('[delta] no new events since last sync');
state.nextBatch = sync.next_batch;
state.updatedAt = Date.now();
saveSyncState(outDir, state);
return state;
}
let totalNew = 0;
let totalImages = 0;
let totalDownloaded = 0;
for (const roomId of roomIds) {
const room = joined[roomId];
const events = room.timeline?.events ?? [];
if (events.length === 0) continue;
// Discover (or rediscover) display name for previously-unseen rooms.
let info = state.rooms[roomId];
if (!info) {
const displayName = await resolveRoomDisplayName(client, roomId, logger);
info = { slug: roomDirSlug(roomId, displayName), displayName };
state.rooms[roomId] = info;
logger.info(`[delta] new room discovered: ${info.slug}`);
}
const roomDir = path.join(outDir, info.slug);
fs.mkdirSync(roomDir, { recursive: true });
const messageEvents = events.filter((e) => e.type === 'm.room.message');
if (messageEvents.length > 0) {
const lines = messageEvents.map((e) => eventToManifestLine(e, client.userId));
fs.appendFileSync(path.join(roomDir, 'messages.jsonl'), lines.join('\n') + '\n', 'utf-8');
}
const { downloaded } = await downloadEventImages(client, events, roomDir, logger);
const imgCount = events.filter(isImageEvent).length;
totalNew += messageEvents.length;
totalImages += imgCount;
totalDownloaded += downloaded;
logger.info(
`[delta] ${info.slug}: +${messageEvents.length} msgs +${imgCount} imgs (downloaded ${downloaded})`,
);
}
state.nextBatch = sync.next_batch;
state.updatedAt = Date.now();
saveSyncState(outDir, state);
logger.info(
`[delta] DONE — rooms_touched=${roomIds.length} new_msgs=${totalNew} new_imgs=${totalImages} downloaded=${totalDownloaded}`,
);
logger.info(`[delta] cursor updated: ${state.nextBatch?.slice(0, 32)}…`);
return state;
}
async function syncAll(
client: MatrixClient,
args: Args,
logger: ReturnType<typeof createLogger>,
) {
const outDir = path.resolve(process.cwd(), args.dumpDir);
if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
logger.info(`[sync] output dir: ${outDir}`);
const existing = loadSyncState(outDir);
if (!existing?.nextBatch || args.forceBackfill) {
if (args.forceBackfill) logger.info('[sync] --force-backfill set: ignoring existing cursor');
else logger.info('[sync] no cursor on disk → running full backfill');
await backfillAll(client, args, logger, outDir);
} else {
await deltaSync(client, args, logger, outDir, existing);
}
}
async function dumpAll(client: MatrixClient, args: Args, logger: ReturnType<typeof createLogger>) {
const outDir = path.resolve(process.cwd(), args.dumpDir);
if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
logger.info(`[dump] output dir: ${outDir}`);
const rooms = await client.listRooms();
logger.info(`[dump] ${rooms.length} rooms to walk`);
const overallSummary: Array<{
roomId: string;
displayName: string | null;
eventCount: number;
messageCount: number;
imageCount: number;
downloaded: number;
failed: number;
}> = [];
let totalImages = 0;
let totalDownloaded = 0;
let totalFailed = 0;
for (const room of rooms) {
const slug = safeSlug(room.displayName ?? room.roomId);
const roomDir = path.join(outDir, slug);
fs.mkdirSync(roomDir, { recursive: true });
const events = await paginateAllMessages(client, room.roomId, args.maxPages);
const messages = events.filter((e) => e.type === 'm.room.message');
const images = messages.filter(
(e) => e.content.msgtype === 'm.image' || typeof e.content.url === 'string',
);
// Write the messages.jsonl manifest.
const manifestPath = path.join(roomDir, 'messages.jsonl');
const lines = messages
.slice()
.reverse() // chronological for human reading
.map((e) =>
JSON.stringify({
event_id: e.event_id,
ts: e.origin_server_ts,
iso: new Date(e.origin_server_ts).toISOString(),
sender: e.sender,
isSelf: e.sender === client.userId,
msgtype: e.content.msgtype ?? null,
body: e.content.body ?? null,
mxc: typeof e.content.url === 'string' ? e.content.url : null,
}),
);
fs.writeFileSync(manifestPath, lines.join('\n') + '\n', 'utf-8');
let downloaded = 0;
let failed = 0;
for (const evt of images) {
const mxc = evt.content.url as string | undefined;
if (!mxc) continue;
try {
const media = await client.downloadMedia(mxc);
if (!media) {
failed++;
continue;
}
const ext = extOfContentType(media.contentType);
const fname = `${evt.origin_server_ts}_${safeSlug(evt.event_id)}.${ext}`;
fs.writeFileSync(path.join(roomDir, fname), media.bytes);
downloaded++;
} catch (err) {
logger.warn(`[dump] failed ${mxc} in ${room.roomId}: ${(err as Error).message}`);
failed++;
}
}
totalImages += images.length;
totalDownloaded += downloaded;
totalFailed += failed;
logger.info(
`[dump] ${slug}: events=${events.length} msgs=${messages.length} imgs=${images.length} ok=${downloaded} fail=${failed}`,
);
overallSummary.push({
roomId: room.roomId,
displayName: room.displayName,
eventCount: events.length,
messageCount: messages.length,
imageCount: images.length,
downloaded,
failed,
});
}
fs.writeFileSync(
path.join(outDir, '_summary.json'),
JSON.stringify(
{
capturedAt: new Date().toISOString(),
userId: client.userId,
homeserver: client.homeserver,
rooms: overallSummary,
totals: {
rooms: rooms.length,
images: totalImages,
downloaded: totalDownloaded,
failed: totalFailed,
},
},
null,
2,
),
'utf-8',
);
logger.info(
`[dump] DONE — rooms=${rooms.length} images=${totalImages} downloaded=${totalDownloaded} failed=${totalFailed}`,
);
logger.info(`[dump] manifest: ${path.join(outDir, '_summary.json')}`);
}
async function getCredentials(logger: ReturnType<typeof createLogger>, refresh: boolean) {
if (!refresh) {
const cached = await loadCachedCredentials();
if (cached) {
logger.info(`[matrix] using cached credentials (captured ${ageStr(cached.capturedAt)} ago)`);
return cached;
}
}
logger.info('[matrix] opening browser to capture access token...');
const context = await chromium.launchPersistentContext(SESSION_DIR, {
headless: false,
viewport: { width: 1280, height: 720 },
});
try {
return await extractMatrixCredentials(context, logger);
} finally {
await context.close();
}
}
function ageStr(ts: number): string {
const sec = Math.max(0, Math.floor((Date.now() - ts) / 1000));
if (sec < 60) return `${sec}s`;
if (sec < 3600) return `${Math.floor(sec / 60)}m`;
if (sec < 86400) return `${Math.floor(sec / 3600)}h`;
return `${Math.floor(sec / 86400)}d`;
}
async function main() {
const logger = createLogger();
const args = parseArgs(process.argv.slice(2));
let creds = await getCredentials(logger, args.refresh);
let client = new MatrixClient(creds, logger);
const reExtract = async () => {
logger.warn('[matrix] token rejected — re-extracting');
creds = await getCredentials(logger, true);
client = new MatrixClient(creds, logger);
};
// --- sync mode (backfill on first run, delta thereafter) -----------
if (args.sync) {
try {
await syncAll(client, args, logger);
} catch (err) {
if (err instanceof MatrixAuthError) {
await reExtract();
await syncAll(client, args, logger);
} else throw err;
}
return;
}
// --- dump mode (snapshot of /sync — recent rooms only) -------------
if (args.dump) {
try {
await dumpAll(client, args, logger);
} catch (err) {
if (err instanceof MatrixAuthError) {
await reExtract();
await dumpAll(client, args, logger);
} else throw err;
}
return;
}
// --- send mode -----------------------------------------------------
if (args.sendRoomId && args.sendBody) {
try {
const eventId = await client.sendTextMessage(args.sendRoomId, args.sendBody);
console.log(`Sent: event_id=${eventId}`);
} catch (err) {
if (err instanceof MatrixAuthError) {
await reExtract();
const eventId = await client.sendTextMessage(args.sendRoomId, args.sendBody);
console.log(`Sent (after re-auth): event_id=${eventId}`);
} else throw err;
}
return;
}
// --- single-room history mode --------------------------------------
if (args.roomId) {
let res;
try {
res = await client.getRoomMessages(args.roomId, { limit: 30 });
} catch (err) {
if (err instanceof MatrixAuthError) {
await reExtract();
res = await client.getRoomMessages(args.roomId, { limit: 30 });
} else throw err;
}
console.log(`\nRoom ${args.roomId} — ${res.chunk.length} events (most recent first):\n`);
for (const event of res.chunk) {
if (event.type !== 'm.room.message') {
console.log(` [${event.type}] from ${event.sender}`);
continue;
}
const who = event.sender === client.userId ? 'ME' : event.sender;
const when = new Date(event.origin_server_ts).toISOString();
const body = (event.content.body as string | undefined) ?? '<no body>';
const mediaTag = event.content.url ? ` [media: ${event.content.url}]` : '';
console.log(` ${when} ${who}: ${body.slice(0, 120)}${body.length > 120 ? '…' : ''}${mediaTag}`);
}
return;
}
// --- list rooms (default) ------------------------------------------
let rooms;
try {
rooms = await client.listRooms();
} catch (err) {
if (err instanceof MatrixAuthError) {
await reExtract();
rooms = await client.listRooms();
} else throw err;
}
console.log(`\nLogged in as: ${client.userId}`);
console.log(`Homeserver: ${client.homeserver}`);
console.log(`Rooms: ${rooms.length}\n`);
for (const r of rooms.slice(0, 50)) {
const unread = r.unreadCount > 0 ? ` (${r.unreadCount} unread)` : '';
const last = r.lastEvent
? `${new Date(r.lastEvent.origin_server_ts).toISOString()} ${(r.lastEvent.content.body as string | undefined)?.slice(0, 80) ?? `[${r.lastEvent.type}]`}`
: '<no events in window>';
console.log(`• ${r.displayName ?? r.roomId}${unread}`);
console.log(` roomId: ${r.roomId}`);
console.log(` last: ${last}\n`);
}
}
main().catch((err) => {
console.error('FATAL:', err);
process.exit(1);
});