mirror of
https://github.com/jjbrunton/Sauci.git
synced 2026-11-02 02:19:43 +00:00
feat(backend): add standalone data plane
Replace the Supabase application data plane with the Node/PostgreSQL API, migrate mobile, admin, web, and MCP callers, add controlled cutover tooling, and define hardened Dokploy infrastructure.
This commit is contained in:
1 parent
01bb8225ae
commit
006761d3b0
294 files changed
+16259
-6138
No files matched your search
@@ -0,0 +1,24 @@
|
||||
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { MigrationCheckpoint } from './types.js';
|
||||
|
||||
export async function loadCheckpoint(path: string, finalSync: boolean): Promise<MigrationCheckpoint> {
|
||||
if (!finalSync) {
|
||||
try {
|
||||
const parsed = JSON.parse(await readFile(path, 'utf8')) as MigrationCheckpoint;
|
||||
if (parsed.version === 1 && parsed.mode === 'initial') return parsed;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
}
|
||||
}
|
||||
return { version: 1, runId: randomUUID(), mode: finalSync ? 'final' : 'initial', completedTables: [], storage: {}, updatedAt: new Date().toISOString() };
|
||||
}
|
||||
|
||||
export async function saveCheckpoint(path: string, checkpoint: MigrationCheckpoint): Promise<void> {
|
||||
checkpoint.updatedAt = new Date().toISOString();
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
const temporary = `${path}.${process.pid}.tmp`;
|
||||
await writeFile(temporary, `${JSON.stringify(checkpoint, null, 2)}\n`, { mode: 0o600 });
|
||||
await rename(temporary, path);
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import type { Pool, PoolClient } from 'pg';
|
||||
import type { DatabaseMigrationOptions, TableResult } from './types.js';
|
||||
|
||||
interface TableSpec { name: string; key: string[]; replace?: boolean }
|
||||
export const TABLES: TableSpec[] = [
|
||||
{ name: 'couples', key: ['id'] }, { name: 'profiles', key: ['id'] },
|
||||
{ name: 'admin_users', key: ['id'] }, { name: 'master_keys', key: ['id'] },
|
||||
{ name: 'categories', key: ['id'] }, { name: 'question_packs', key: ['id'] },
|
||||
{ name: 'topics', key: ['id'] }, { name: 'pack_topics', key: ['pack_id', 'topic_id'] },
|
||||
{ name: 'questions', key: ['id'] }, { name: 'app_config', key: ['id'], replace: true },
|
||||
{ name: 'ai_config', key: ['id'], replace: true },
|
||||
{ name: 'dare_packs', key: ['id'] }, { name: 'dares', key: ['id'] },
|
||||
{ name: 'sent_dares', key: ['id'] }, { name: 'dare_messages', key: ['id'] },
|
||||
{ name: 'couple_packs', key: ['couple_id', 'pack_id'] }, { name: 'responses', key: ['id'] },
|
||||
{ name: 'matches', key: ['id'] }, { name: 'couple_streaks', key: ['couple_id'] },
|
||||
{ name: 'messages', key: ['id'] }, { name: 'match_archives', key: ['id'] },
|
||||
{ name: 'message_deletions', key: ['message_id', 'user_id'] }, { name: 'message_reports', key: ['id'] },
|
||||
{ name: 'notification_preferences', key: ['user_id'] }, { name: 'feedback', key: ['id'] },
|
||||
{ name: 'subscriptions', key: ['id'] }, { name: 'revenuecat_webhook_events', key: ['id'] },
|
||||
{ name: 'redemption_codes', key: ['id'] }, { name: 'code_redemptions', key: ['id'] },
|
||||
{ name: 'feature_interests', key: ['user_id', 'feature'] }, { name: 'live_draw_sessions', key: ['couple_id'] },
|
||||
{ name: 'catchup_reminder_tracking', key: ['user_id'] }, { name: 'audit_logs', key: ['id'] },
|
||||
];
|
||||
|
||||
const quote = (value: string) => `"${value.replaceAll('"', '""')}"`;
|
||||
async function columns(pool: Pool | PoolClient, table: string): Promise<string[]> {
|
||||
const result = await pool.query<{ column_name: string }>(`select column_name from information_schema.columns where table_schema=current_schema() and table_name=$1 order by ordinal_position`, [table]);
|
||||
return result.rows.map((row) => row.column_name);
|
||||
}
|
||||
async function count(pool: Pool | PoolClient, table: string): Promise<number> {
|
||||
return Number((await pool.query<{ count: string }>(`select count(*) count from ${quote(table)}`)).rows[0]?.count ?? 0);
|
||||
}
|
||||
|
||||
async function upsertRows(client: PoolClient, spec: TableSpec, names: string[], rows: Record<string, unknown>[]): Promise<number> {
|
||||
if (spec.replace) await client.query(`delete from ${quote(spec.name)}`);
|
||||
if (!rows.length) return 0;
|
||||
const update = names.filter((name) => !spec.key.includes(name));
|
||||
const conflict = spec.key.map(quote).join(',');
|
||||
const action = update.length ? `do update set ${update.map((name) => `${quote(name)}=excluded.${quote(name)}`).join(',')}` : 'do nothing';
|
||||
for (const row of rows) {
|
||||
await client.query(`insert into ${quote(spec.name)} (${names.map(quote).join(',')}) values (${names.map((_, index) => `$${index + 1}`).join(',')}) on conflict (${conflict}) ${action}`, names.map((name) => row[name]));
|
||||
}
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
async function pruneRows(client: PoolClient, spec: TableSpec, sourceRows: Record<string, unknown>[]): Promise<number> {
|
||||
if (spec.replace || !spec.key.length) return 0;
|
||||
const before = await count(client, spec.name);
|
||||
if (!sourceRows.length) await client.query(`delete from ${quote(spec.name)}`);
|
||||
else {
|
||||
const values: unknown[] = [];
|
||||
const tuples = sourceRows.map((row) => `(${spec.key.map((key) => { values.push(row[key]); return `$${values.length}`; }).join(',')})`).join(',');
|
||||
const keyTuple = `(${spec.key.map(quote).join(',')})`;
|
||||
await client.query(`delete from ${quote(spec.name)} where ${keyTuple} not in (${tuples})`, values);
|
||||
}
|
||||
return before - await count(client, spec.name);
|
||||
}
|
||||
|
||||
export async function migrateDatabase(source: Pool, target: Pool, options: DatabaseMigrationOptions): Promise<TableResult[]> {
|
||||
const results: TableResult[] = [];
|
||||
for (const spec of TABLES) {
|
||||
const [sourceColumns, targetColumns] = await Promise.all([columns(source, spec.name), columns(target, spec.name)]);
|
||||
if (!sourceColumns.length || !targetColumns.length) {
|
||||
results.push({ table: spec.name, sourceRows: 0, targetRows: targetColumns.length ? await count(target, spec.name) : 0, importedRows: 0, prunedRows: 0, status: 'skipped', note: !sourceColumns.length ? 'source table absent' : 'target table absent' });
|
||||
continue;
|
||||
}
|
||||
const common = targetColumns.filter((name) => sourceColumns.includes(name));
|
||||
const sourceOnly = sourceColumns.filter((name) => !targetColumns.includes(name));
|
||||
if (!spec.key.every((key) => common.includes(key))) throw new Error(`${spec.name} is missing required migration key columns`);
|
||||
const sourceRows = (await source.query<Record<string, unknown>>(`select ${common.map(quote).join(',')} from ${quote(spec.name)}`)).rows;
|
||||
let importedRows = 0; let prunedRows = 0;
|
||||
const skip = !options.finalSync && options.checkpoint.completedTables.includes(spec.name);
|
||||
if (!options.dryRun && !skip) {
|
||||
const client = await target.connect();
|
||||
try {
|
||||
await client.query('begin');
|
||||
// Historical imports must not enqueue user notifications, classification,
|
||||
// Discord events, or other operational side effects.
|
||||
await client.query("set local sauci.suppress_operations = 'on'");
|
||||
importedRows = await upsertRows(client, spec, common, sourceRows);
|
||||
if (options.finalSync && options.prune) prunedRows = await pruneRows(client, spec, sourceRows);
|
||||
await client.query('commit');
|
||||
} catch (error) { await client.query('rollback'); throw error; } finally { client.release(); }
|
||||
options.checkpoint.completedTables = [...new Set([...options.checkpoint.completedTables, spec.name])];
|
||||
await options.saveCheckpoint?.(options.checkpoint);
|
||||
}
|
||||
const targetRows = await count(target, spec.name);
|
||||
const notes = [skip ? 'resumed from checkpoint' : '', sourceOnly.length ? `legacy-only columns intentionally omitted: ${sourceOnly.join(', ')}` : ''].filter(Boolean);
|
||||
results.push({ table: spec.name, sourceRows: sourceRows.length, targetRows, importedRows, prunedRows, status: sourceRows.length === targetRows ? 'equal' : 'different', ...(notes.length ? { note: notes.join('; ') } : {}) });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
export async function relationalParity(target: Pool): Promise<Record<string, number>> {
|
||||
const checks: Record<string, string> = {
|
||||
profiles_without_couple: 'select count(*) count from profiles p left join couples c on c.id=p.couple_id where p.couple_id is not null and c.id is null',
|
||||
responses_with_missing_owner: 'select count(*) count from responses r left join profiles p on p.id=r.user_id left join couples c on c.id=r.couple_id left join questions q on q.id=r.question_id where p.id is null or c.id is null or q.id is null',
|
||||
matches_with_missing_relation: 'select count(*) count from matches m left join couples c on c.id=m.couple_id left join questions q on q.id=m.question_id where c.id is null or q.id is null',
|
||||
messages_with_missing_relation: 'select count(*) count from messages m left join matches x on x.id=m.match_id left join profiles p on p.id=m.user_id where x.id is null or p.id is null',
|
||||
couple_mismatch_responses: 'select count(*) count from responses r join profiles p on p.id=r.user_id where p.couple_id is distinct from r.couple_id',
|
||||
};
|
||||
const result: Record<string, number> = {};
|
||||
for (const [name, sql] of Object.entries(checks)) result[name] = Number((await target.query<{ count: string }>(sql)).rows[0]?.count ?? 0);
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { Pool } from 'pg';
|
||||
import { loadCheckpoint, saveCheckpoint } from './checkpoint.js';
|
||||
import { migrateDatabase, relationalParity } from './database.js';
|
||||
import { validateEndpoints, redactDatabaseUrl } from './safety.js';
|
||||
import { createSupabaseDownloader, migrateStorage } from './storage.js';
|
||||
import type { ParityReport } from './types.js';
|
||||
|
||||
const args = new Set(process.argv.slice(2));
|
||||
const dryRun = args.has('--dry-run'); const finalSync = args.has('--final-sync'); const prune = args.has('--prune'); const includeStorage = args.has('--storage');
|
||||
if (prune && (!finalSync || dryRun)) throw new Error('--prune requires a non-dry-run --final-sync with the legacy source stopped');
|
||||
if (includeStorage && !process.env.MEDIA_ROOT) throw new Error('MEDIA_ROOT is required with --storage');
|
||||
const endpoints = validateEndpoints(process.env.SOURCE_DATABASE_URL, process.env.TARGET_DATABASE_URL, process.env.MIGRATION_TARGET_HOST_ALLOWLIST);
|
||||
const checkpointPath = resolve(process.env.MIGRATION_CHECKPOINT_FILE ?? '.migration-state/backend-cutover.json');
|
||||
const reportPath = resolve(process.env.MIGRATION_REPORT_FILE ?? '.migration-state/backend-parity.json');
|
||||
const checkpoint = await loadCheckpoint(checkpointPath, finalSync);
|
||||
const source = new Pool({ connectionString: endpoints.source, max: 2 }); const target = new Pool({ connectionString: endpoints.target, max: 2 });
|
||||
const startedAt = new Date().toISOString();
|
||||
console.log(`Migration ${checkpoint.runId}: ${redactDatabaseUrl(endpoints.source)} -> ${redactDatabaseUrl(endpoints.target)} (${dryRun ? 'dry-run' : finalSync ? 'final-sync' : 'initial'})`);
|
||||
try {
|
||||
const save = (value: typeof checkpoint) => saveCheckpoint(checkpointPath, value);
|
||||
const tables = await migrateDatabase(source, target, { dryRun, finalSync, prune, checkpoint, saveCheckpoint: save });
|
||||
const storage = includeStorage ? await migrateStorage(source, target, {
|
||||
dryRun, prune: finalSync && prune, mediaRoot: process.env.MEDIA_ROOT ?? '', checkpoint, saveCheckpoint: save,
|
||||
download: createSupabaseDownloader(process.env.SOURCE_STORAGE_URL ?? '', process.env.SOURCE_STORAGE_SERVICE_ROLE_KEY ?? ''),
|
||||
}) : null;
|
||||
const relational = await relationalParity(target);
|
||||
const parity = tables.filter((row) => row.status !== 'skipped').every((row) => row.status === 'equal') && Object.values(relational).every((count) => count === 0) && (!storage || (storage.sourceFiles === storage.targetFiles && storage.failedFiles === 0 && storage.missingOnDisk.length === 0));
|
||||
const report: ParityReport = { runId: checkpoint.runId, dryRun, finalSync, startedAt, finishedAt: new Date().toISOString(), tables, storage, relational, parity };
|
||||
await mkdir(dirname(reportPath), { recursive: true }); await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 });
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
if (!parity && finalSync) process.exitCode = 2;
|
||||
} finally { await Promise.all([source.end(), target.end()]); }
|
||||
@@ -0,0 +1,30 @@
|
||||
const LOCAL_HOSTS = new Set(['localhost', '127.0.0.1', '::1', 'host.docker.internal']);
|
||||
|
||||
function parseDatabaseUrl(label: string, value: string | undefined): URL {
|
||||
if (!value) throw new Error(`${label} is required`);
|
||||
let parsed: URL;
|
||||
try { parsed = new URL(value); } catch { throw new Error(`${label} must be a valid PostgreSQL URL`); }
|
||||
if (!['postgres:', 'postgresql:'].includes(parsed.protocol)) throw new Error(`${label} must use postgres:// or postgresql://`);
|
||||
if (!parsed.hostname || !parsed.pathname.slice(1)) throw new Error(`${label} must identify a host and database`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function databaseIdentity(url: URL): string {
|
||||
return `${url.hostname.toLowerCase()}:${url.port || '5432'}/${decodeURIComponent(url.pathname.slice(1))}`;
|
||||
}
|
||||
|
||||
export function validateEndpoints(sourceValue: string | undefined, targetValue: string | undefined, allowlistValue?: string): { source: string; target: string } {
|
||||
const source = parseDatabaseUrl('SOURCE_DATABASE_URL', sourceValue);
|
||||
const target = parseDatabaseUrl('TARGET_DATABASE_URL', targetValue);
|
||||
if (databaseIdentity(source) === databaseIdentity(target)) throw new Error('Source and target must be different PostgreSQL databases');
|
||||
const allowlist = new Set((allowlistValue ?? '').split(',').map((item) => item.trim().toLowerCase()).filter(Boolean));
|
||||
if (!LOCAL_HOSTS.has(target.hostname.toLowerCase()) && !allowlist.has(target.hostname.toLowerCase())) {
|
||||
throw new Error(`Target host is not allowlisted; add it explicitly to MIGRATION_TARGET_HOST_ALLOWLIST`);
|
||||
}
|
||||
return { source: source.toString(), target: target.toString() };
|
||||
}
|
||||
|
||||
export function redactDatabaseUrl(value: string): string {
|
||||
const parsed = new URL(value); parsed.username = parsed.username ? '***' : ''; parsed.password = parsed.password ? '***' : '';
|
||||
return parsed.toString();
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
|
||||
import { dirname, resolve, sep } from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { Pool } from 'pg';
|
||||
import type { StorageMigrationOptions, StorageResult } from './types.js';
|
||||
|
||||
interface StorageObject {
|
||||
id: string;
|
||||
bucket_id: string;
|
||||
name: string;
|
||||
owner_id: string | null;
|
||||
metadata: Record<string, unknown> | null;
|
||||
created_at: string;
|
||||
updated_at: string | null;
|
||||
}
|
||||
const BUCKET_KIND: Record<string, 'avatar' | 'response' | 'chat' | 'feedback'> = {
|
||||
avatars: 'avatar', 'response-media': 'response', 'chat-media': 'chat', 'feedback-screenshots': 'feedback',
|
||||
};
|
||||
const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
const quote = (value: string) => `"${value.replaceAll('"', '""')}"`;
|
||||
|
||||
function safeStorageKey(bucket: string, name: string): string {
|
||||
const normalized = `${bucket}/${name}`.replaceAll('\\', '/');
|
||||
if (!BUCKET_KIND[bucket] || normalized.startsWith('/') || normalized.split('/').some((part) => !part || part === '.' || part === '..')) throw new Error(`Unsafe storage object path: ${bucket}/${name}`);
|
||||
return normalized;
|
||||
}
|
||||
function destination(rootValue: string, key: string): string {
|
||||
const root = resolve(rootValue); const path = resolve(root, key);
|
||||
if (!path.startsWith(`${root}${sep}`)) throw new Error(`Storage path escapes MEDIA_ROOT: ${key}`);
|
||||
return path;
|
||||
}
|
||||
function metadataString(metadata: Record<string, unknown> | null, ...keys: string[]): string | null {
|
||||
for (const key of keys) if (typeof metadata?.[key] === 'string') return metadata[key] as string;
|
||||
return null;
|
||||
}
|
||||
function metadataSize(metadata: Record<string, unknown> | null): number {
|
||||
const value = metadata?.size ?? metadata?.contentLength;
|
||||
return typeof value === 'number' && value >= 0 ? value : Number(value ?? 0) || 0;
|
||||
}
|
||||
|
||||
export function createSupabaseDownloader(baseUrl: string, serviceRoleKey: string): (bucket: string, name: string) => Promise<Uint8Array> {
|
||||
const base = baseUrl.replace(/\/$/, '');
|
||||
if (!/^https:\/\//.test(base) && !/^http:\/\/(localhost|127\.0\.0\.1)(:|\/)/.test(base)) throw new Error('SOURCE_STORAGE_URL must use HTTPS unless it is localhost');
|
||||
if (!serviceRoleKey) throw new Error('SOURCE_STORAGE_SERVICE_ROLE_KEY is required for storage copy');
|
||||
return async (bucket, name) => {
|
||||
const path = [bucket, ...name.split('/')].map(encodeURIComponent).join('/');
|
||||
const response = await fetch(`${base}/storage/v1/object/authenticated/${path}`, { headers: { authorization: `Bearer ${serviceRoleKey}`, apikey: serviceRoleKey } });
|
||||
if (!response.ok) throw new Error(`Storage download failed for ${bucket}/${name}: HTTP ${response.status}`);
|
||||
return new Uint8Array(await response.arrayBuffer());
|
||||
};
|
||||
}
|
||||
|
||||
async function mediaRelation(target: Pool, object: StorageObject, key: string) {
|
||||
const first = object.name.split('/')[0];
|
||||
const owner = object.owner_id && uuidPattern.test(object.owner_id) ? object.owner_id : (uuidPattern.test(first) && object.bucket_id !== 'chat-media' ? first : null);
|
||||
if (!owner) throw new Error(`Cannot determine owner for ${key}`);
|
||||
const profile = await target.query<{ couple_id: string | null }>('select couple_id from profiles where id=$1', [owner]);
|
||||
if (!profile.rows[0]) throw new Error(`Storage owner does not exist in target for ${key}`);
|
||||
let matchId: string | null = null; let questionId: string | null = null;
|
||||
if (object.bucket_id === 'chat-media' && uuidPattern.test(first)) matchId = first;
|
||||
if (object.bucket_id === 'response-media') {
|
||||
const candidate = object.name.split('/').at(-1)?.split('_')[0] ?? '';
|
||||
if (uuidPattern.test(candidate)) questionId = candidate;
|
||||
}
|
||||
return { ownerId: owner, coupleId: profile.rows[0].couple_id, matchId, questionId };
|
||||
}
|
||||
|
||||
async function rewriteLegacyReference(target: Pool, object: StorageObject): Promise<void> {
|
||||
const reference = `media:${object.id}`;
|
||||
const legacyName = object.name;
|
||||
const legacyKey = `${object.bucket_id}/${legacyName}`;
|
||||
const matchesLegacy = (column: string) => `(
|
||||
${column}=$2 or ${column}=$3 or
|
||||
(right(${column},length($2))=$2 and position('/${object.bucket_id}/' in ${column})>0)
|
||||
)`;
|
||||
if (object.bucket_id === 'avatars') {
|
||||
await target.query(`update profiles set avatar_url=$1 where ${matchesLegacy('avatar_url')}`, [reference, legacyName, legacyKey]);
|
||||
} else if (object.bucket_id === 'chat-media') {
|
||||
await target.query(`update messages set media_path=$1 where ${matchesLegacy('media_path')}`, [reference, legacyName, legacyKey]);
|
||||
} else if (object.bucket_id === 'feedback-screenshots') {
|
||||
await target.query(`update feedback set screenshot_url=$1 where ${matchesLegacy('screenshot_url')}`, [reference, legacyName, legacyKey]);
|
||||
} else if (object.bucket_id === 'response-media') {
|
||||
await target.query(`
|
||||
update responses
|
||||
set response_data=jsonb_set(response_data,'{media_path}',to_jsonb($1::text),true)
|
||||
where response_data is not null and ${matchesLegacy("response_data->>'media_path'")}
|
||||
`, [reference, legacyName, legacyKey]);
|
||||
}
|
||||
}
|
||||
|
||||
export async function migrateStorage(source: Pool, target: Pool, options: StorageMigrationOptions): Promise<StorageResult> {
|
||||
const schema = options.storageSchema ?? 'storage';
|
||||
if (!/^[a-z_][a-z0-9_]*$/i.test(schema)) throw new Error('Invalid storage schema');
|
||||
const objects = await source.query<StorageObject>(`select id,bucket_id,name,owner_id,metadata,created_at,updated_at from ${quote(schema)}.objects where bucket_id=any($1::text[]) order by bucket_id,name`, [Object.keys(BUCKET_KIND)]);
|
||||
const result: StorageResult = { sourceFiles: objects.rowCount ?? objects.rows.length, targetFiles: 0, copiedFiles: 0, skippedFiles: 0, prunedFiles: 0, failedFiles: 0, sourceBytes: 0, copiedBytes: 0, missingOnDisk: [], failures: [] };
|
||||
const sourceKeys = new Set<string>();
|
||||
for (const object of objects.rows) {
|
||||
const key = safeStorageKey(object.bucket_id, object.name); const expectedSize = metadataSize(object.metadata); result.sourceBytes += expectedSize;
|
||||
sourceKeys.add(key);
|
||||
const updatedAt = object.updated_at ? new Date(object.updated_at).toISOString() : null;
|
||||
const marker = options.checkpoint.storage[key];
|
||||
if (marker && marker.updatedAt === updatedAt && (!expectedSize || marker.byteSize === expectedSize)) {
|
||||
if (options.dryRun) { result.skippedFiles += 1; continue; }
|
||||
const registered = await target.query('select 1 from media_objects where id=$1 and storage_key=$2 and byte_size=$3', [object.id, key, marker.byteSize]);
|
||||
try {
|
||||
if (registered.rowCount && (await stat(destination(options.mediaRoot, key))).size === marker.byteSize) {
|
||||
await rewriteLegacyReference(target, object);
|
||||
result.skippedFiles += 1;
|
||||
continue;
|
||||
}
|
||||
} catch { /* checkpoint is incomplete; recopy below */ }
|
||||
}
|
||||
if (options.dryRun) continue;
|
||||
try {
|
||||
const bytes = await options.download(object.bucket_id, object.name);
|
||||
if (expectedSize && bytes.byteLength !== expectedSize) throw new Error(`Size mismatch for ${key}: expected ${expectedSize}, received ${bytes.byteLength}`);
|
||||
const relation = await mediaRelation(target, object, key);
|
||||
const path = destination(options.mediaRoot, key); await mkdir(dirname(path), { recursive: true });
|
||||
const temporary = `${path}.${process.pid}.tmp`; await writeFile(temporary, bytes, { mode: 0o600 }); await rename(temporary, path);
|
||||
const mime = metadataString(object.metadata, 'mimetype', 'contentType') ?? 'application/octet-stream';
|
||||
await target.query(`insert into media_objects(id,owner_id,couple_id,kind,storage_key,mime_type,byte_size,question_id,match_id,created_at)
|
||||
values($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) on conflict(id) do update set owner_id=excluded.owner_id,couple_id=excluded.couple_id,kind=excluded.kind,storage_key=excluded.storage_key,mime_type=excluded.mime_type,byte_size=excluded.byte_size,question_id=excluded.question_id,match_id=excluded.match_id,created_at=excluded.created_at`,
|
||||
[object.id, relation.ownerId, relation.coupleId, BUCKET_KIND[object.bucket_id], key, mime, bytes.byteLength, relation.questionId, relation.matchId, object.created_at]);
|
||||
await rewriteLegacyReference(target, object);
|
||||
options.checkpoint.storage[key] = { updatedAt, byteSize: bytes.byteLength };
|
||||
await options.saveCheckpoint?.(options.checkpoint); result.copiedFiles += 1; result.copiedBytes += bytes.byteLength;
|
||||
} catch (error) {
|
||||
result.failedFiles += 1;
|
||||
result.failures.push({ storageKey: key, error: error instanceof Error ? error.message : 'Unknown storage migration failure' });
|
||||
}
|
||||
}
|
||||
if (!options.dryRun) {
|
||||
if (options.prune) {
|
||||
const existing = await target.query<{ id: string; storage_key: string }>("select id,storage_key from media_objects where kind=any($1::text[])", [Object.values(BUCKET_KIND)]);
|
||||
for (const row of existing.rows) if (!sourceKeys.has(row.storage_key)) {
|
||||
await rm(destination(options.mediaRoot, row.storage_key), { force: true });
|
||||
await target.query('delete from media_objects where id=$1', [row.id]);
|
||||
delete options.checkpoint.storage[row.storage_key]; result.prunedFiles += 1;
|
||||
}
|
||||
await options.saveCheckpoint?.(options.checkpoint);
|
||||
}
|
||||
const registered = await target.query<{ storage_key: string; byte_size: number }>("select storage_key,byte_size from media_objects where deleted_at is null and kind=any($1::text[])", [Object.values(BUCKET_KIND)]);
|
||||
result.targetFiles = registered.rows.length;
|
||||
for (const row of registered.rows) {
|
||||
try { if ((await stat(destination(options.mediaRoot, row.storage_key))).size !== row.byte_size) result.missingOnDisk.push(row.storage_key); }
|
||||
catch { result.missingOnDisk.push(row.storage_key); }
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function fileDigest(path: string): Promise<string> {
|
||||
return createHash('sha256').update(await readFile(path)).digest('hex');
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import type { Pool } from 'pg';
|
||||
|
||||
export interface MigrationCheckpoint {
|
||||
version: 1;
|
||||
runId: string;
|
||||
mode: 'initial' | 'final';
|
||||
completedTables: string[];
|
||||
storage: Record<string, { updatedAt: string | null; byteSize: number }>;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface TableResult {
|
||||
table: string;
|
||||
sourceRows: number;
|
||||
targetRows: number;
|
||||
importedRows: number;
|
||||
prunedRows: number;
|
||||
status: 'equal' | 'different' | 'skipped';
|
||||
note?: string;
|
||||
}
|
||||
|
||||
export interface StorageResult {
|
||||
sourceFiles: number;
|
||||
targetFiles: number;
|
||||
copiedFiles: number;
|
||||
skippedFiles: number;
|
||||
prunedFiles: number;
|
||||
failedFiles: number;
|
||||
sourceBytes: number;
|
||||
copiedBytes: number;
|
||||
missingOnDisk: string[];
|
||||
failures: Array<{ storageKey: string; error: string }>;
|
||||
}
|
||||
|
||||
export interface ParityReport {
|
||||
runId: string;
|
||||
dryRun: boolean;
|
||||
finalSync: boolean;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
tables: TableResult[];
|
||||
storage: StorageResult | null;
|
||||
relational: Record<string, number>;
|
||||
parity: boolean;
|
||||
}
|
||||
|
||||
export interface DatabaseMigrationOptions {
|
||||
dryRun: boolean;
|
||||
finalSync: boolean;
|
||||
prune: boolean;
|
||||
checkpoint: MigrationCheckpoint;
|
||||
saveCheckpoint?: (value: MigrationCheckpoint) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface StorageMigrationOptions {
|
||||
dryRun: boolean;
|
||||
prune?: boolean;
|
||||
mediaRoot: string;
|
||||
checkpoint: MigrationCheckpoint;
|
||||
storageSchema?: string;
|
||||
download: (bucket: string, name: string) => Promise<Uint8Array>;
|
||||
saveCheckpoint?: (value: MigrationCheckpoint) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface MigrationPools { source: Pool; target: Pool }
|
||||
Reference in new issue
Block a user