Update source code and remove runtime data from tracking
Add gitignore rules for browser session data, SQLite databases, media downloads, and build artifacts. Remove previously tracked runtime files from the repository. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jjbruntonandClaude Opus 4.6 committed
1 parent
e90213d1ff
commit
dfa0580859
23 files changed
+5324
-347
No files matched your search
@@ -0,0 +1,254 @@
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import crypto from 'crypto';
|
||||
import { z } from 'zod';
|
||||
import type { Logger } from '../utils/logger.js';
|
||||
import type { ActiveChat, ChatMedia, ChatMessage } from './chat.js';
|
||||
|
||||
export interface MediaDownloadResult {
|
||||
buffer: Buffer;
|
||||
contentType: string | null;
|
||||
}
|
||||
|
||||
export interface MediaDownloader {
|
||||
fetch(url: string): Promise<MediaDownloadResult | null>;
|
||||
}
|
||||
|
||||
export interface MediaRecord {
|
||||
id: string;
|
||||
chatUsername: string | null;
|
||||
chatRoomId: string | null;
|
||||
sender: string;
|
||||
timestamp: number;
|
||||
originalUrl: string;
|
||||
fileName: string;
|
||||
relativePath: string;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
const MediaIndexSchema = z.object({
|
||||
version: z.number(),
|
||||
savedAt: z.number(),
|
||||
items: z.array(
|
||||
z.object({
|
||||
id: z.string(),
|
||||
chatUsername: z.string().nullable(),
|
||||
chatRoomId: z.string().nullable(),
|
||||
sender: z.string(),
|
||||
timestamp: z.number(),
|
||||
originalUrl: z.string(),
|
||||
fileName: z.string(),
|
||||
relativePath: z.string(),
|
||||
createdAt: z.number(),
|
||||
})
|
||||
),
|
||||
});
|
||||
|
||||
type MediaIndex = z.infer<typeof MediaIndexSchema>;
|
||||
|
||||
interface ScrapedChatMessage extends ChatMessage {
|
||||
mediaUrls: string[];
|
||||
}
|
||||
|
||||
export class MediaService {
|
||||
private readonly mediaRoot: string;
|
||||
private readonly indexPath: string;
|
||||
private index: MediaIndex | null = null;
|
||||
|
||||
constructor(
|
||||
private downloader: MediaDownloader,
|
||||
private logger: Logger
|
||||
) {
|
||||
this.mediaRoot = path.join(process.cwd(), 'data', 'media');
|
||||
this.indexPath = path.join(this.mediaRoot, 'index.json');
|
||||
}
|
||||
|
||||
async attachMedia(chat: ActiveChat, messages: ScrapedChatMessage[]): Promise<ChatMessage[]> {
|
||||
await this.loadIndex();
|
||||
const updated: ChatMessage[] = [];
|
||||
|
||||
for (const message of messages) {
|
||||
const media = await this.downloadMessageMedia(chat, message);
|
||||
updated.push({
|
||||
sender: message.sender,
|
||||
text: message.text,
|
||||
timestamp: message.timestamp,
|
||||
isSelf: message.isSelf,
|
||||
media: media.length > 0 ? media : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
await this.saveIndex();
|
||||
return updated;
|
||||
}
|
||||
|
||||
async getMediaItems(): Promise<MediaRecord[]> {
|
||||
await this.loadIndex();
|
||||
return (this.index?.items || []).slice().sort((a, b) => b.createdAt - a.createdAt);
|
||||
}
|
||||
|
||||
private async downloadMessageMedia(chat: ActiveChat, message: ScrapedChatMessage): Promise<ChatMedia[]> {
|
||||
if (!message.mediaUrls || message.mediaUrls.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const mediaItems: ChatMedia[] = [];
|
||||
|
||||
for (const url of message.mediaUrls) {
|
||||
const record = await this.downloadMedia(chat, message, url);
|
||||
if (record) {
|
||||
mediaItems.push({
|
||||
originalUrl: record.originalUrl,
|
||||
localPath: record.relativePath,
|
||||
fileName: record.fileName,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return mediaItems;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Reddit Matrix blurred thumbnail URLs to full-resolution download URLs.
|
||||
* Blurred: https://matrix.redditspace.com/_matrix/media/v3/thumbnail/reddit.com/blur_XXXXX?height=300
|
||||
* Full: https://matrix.redditspace.com/_matrix/media/v3/download/reddit.com/XXXXX
|
||||
*/
|
||||
private resolveFullMediaUrl(url: string): string {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
if (parsed.hostname === 'matrix.redditspace.com' && parsed.pathname.includes('/thumbnail/')) {
|
||||
// Strip blur_ prefix from the media ID
|
||||
const fullPath = parsed.pathname.replace('/thumbnail/', '/download/').replace('/blur_', '/');
|
||||
return `${parsed.origin}${fullPath}`;
|
||||
}
|
||||
} catch { /* keep original */ }
|
||||
return url;
|
||||
}
|
||||
|
||||
private async downloadMedia(chat: ActiveChat, message: ScrapedChatMessage, url: string): Promise<MediaRecord | null> {
|
||||
if (!url || url.startsWith('data:')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Try the full-resolution URL first, fall back to original
|
||||
const fullUrl = this.resolveFullMediaUrl(url);
|
||||
const id = crypto.createHash('sha1').update(fullUrl).digest('hex');
|
||||
const existing = this.index?.items.find((item) => item.id === id);
|
||||
if (existing) {
|
||||
// Verify the file still exists on disk
|
||||
const existingPath = path.join(this.mediaRoot, existing.fileName);
|
||||
try {
|
||||
await fs.access(existingPath);
|
||||
return existing;
|
||||
} catch {
|
||||
// File missing — remove stale record and re-download
|
||||
this.logger.debug(`Media file missing on disk, re-downloading: ${existing.fileName}`);
|
||||
this.index!.items = this.index!.items.filter((item) => item.id !== id);
|
||||
}
|
||||
}
|
||||
|
||||
let download = await this.downloader.fetch(fullUrl);
|
||||
if (!download && fullUrl !== url) {
|
||||
this.logger.debug(`Full-res download failed, trying original URL: ${url}`);
|
||||
download = await this.downloader.fetch(url);
|
||||
}
|
||||
if (!download) {
|
||||
this.logger.warn(`Failed to download chat media from ${fullUrl}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const extension = this.pickExtension(url, download.contentType);
|
||||
if (!extension) {
|
||||
this.logger.debug(`Skipping non-image media from ${url}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const fileName = `${id}${extension}`;
|
||||
const relativePath = fileName;
|
||||
const fullPath = path.join(this.mediaRoot, fileName);
|
||||
|
||||
await fs.mkdir(this.mediaRoot, { recursive: true });
|
||||
await fs.writeFile(fullPath, download.buffer);
|
||||
|
||||
const record: MediaRecord = {
|
||||
id,
|
||||
chatUsername: chat.username || null,
|
||||
chatRoomId: chat.roomId || null,
|
||||
sender: message.sender,
|
||||
timestamp: message.timestamp,
|
||||
originalUrl: fullUrl,
|
||||
fileName,
|
||||
relativePath,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
|
||||
if (!this.index) {
|
||||
this.index = { version: 1, savedAt: Date.now(), items: [record] };
|
||||
} else {
|
||||
this.index.items.push(record);
|
||||
this.index.savedAt = Date.now();
|
||||
}
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
private pickExtension(url: string, contentType: string | null): string | null {
|
||||
let urlExt = '';
|
||||
try {
|
||||
urlExt = path.extname(new URL(url).pathname || '').toLowerCase();
|
||||
} catch {
|
||||
urlExt = path.extname(url.split('?')[0] || '').toLowerCase();
|
||||
}
|
||||
if (urlExt && urlExt.length <= 5) {
|
||||
return urlExt;
|
||||
}
|
||||
|
||||
if (!contentType) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (contentType.includes('image/jpeg')) return '.jpg';
|
||||
if (contentType.includes('image/png')) return '.png';
|
||||
if (contentType.includes('image/gif')) return '.gif';
|
||||
if (contentType.includes('image/webp')) return '.webp';
|
||||
if (contentType.includes('image/avif')) return '.avif';
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private async loadIndex(): Promise<void> {
|
||||
if (this.index) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const raw = await fs.readFile(this.indexPath, 'utf-8');
|
||||
const parsed = MediaIndexSchema.safeParse(JSON.parse(raw));
|
||||
if (!parsed.success) {
|
||||
this.logger.warn('Media index is invalid, starting fresh.');
|
||||
this.index = { version: 1, savedAt: Date.now(), items: [] };
|
||||
return;
|
||||
}
|
||||
|
||||
this.index = parsed.data;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
this.logger.warn('Failed to load media index, starting fresh:', error);
|
||||
}
|
||||
this.index = { version: 1, savedAt: Date.now(), items: [] };
|
||||
}
|
||||
}
|
||||
|
||||
private async saveIndex(): Promise<void> {
|
||||
if (!this.index) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.mkdir(this.mediaRoot, { recursive: true });
|
||||
await fs.writeFile(this.indexPath, JSON.stringify(this.index, null, 2), 'utf-8');
|
||||
} catch (error) {
|
||||
this.logger.warn('Failed to save media index:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user