claudetools/projects/dataforth-dos/datasheet-pipeline/scmvas-hvas-research/existing-database/migrate-data.js

/**
 * SQLite to PostgreSQL Data Migration
 *
 * Streams all data from the SQLite testdata.db into PostgreSQL.
 * Uses batch INSERTs for performance.
 *
 * Usage:
 *   node migrate-data.js                  Migrate all tables
 *   node migrate-data.js --skip-tsvector  Skip tsvector rebuild (faster, trigger handles it)
 *   node migrate-data.js --table test_records  Migrate only one table
 */

const path = require('path');
const Database = require('better-sqlite3');
const db = require('./db');

const SQLITE_PATH = path.join(__dirname, 'testdata.db');
const BATCH_SIZE = 5000;

async function migrateTestRecords(sqlite) {
    console.log('\n--- Migrating test_records ---');

    const total = sqlite.prepare('SELECT COUNT(*) as cnt FROM test_records').get().cnt;
    console.log(`  Source records: ${total.toLocaleString()}`);

    // Disable triggers during bulk load for performance
    await db.execute('ALTER TABLE test_records DISABLE TRIGGER trg_search_vector');

    const stmt = sqlite.prepare('SELECT * FROM test_records ORDER BY id');
    let migrated = 0;
    let batch = [];

    for (const row of stmt.iterate()) {
        batch.push(row);

        if (batch.length >= BATCH_SIZE) {
            await insertTestRecordsBatch(batch);
            migrated += batch.length;
            batch = [];
            process.stdout.write(`\r  Migrated: ${migrated.toLocaleString()} / ${total.toLocaleString()}`);
        }
    }

    // Flush remaining
    if (batch.length > 0) {
        await insertTestRecordsBatch(batch);
        migrated += batch.length;
    }

    console.log(`\r  Migrated: ${migrated.toLocaleString()} / ${total.toLocaleString()}`);

    // Rebuild search_vector for all rows
    console.log('  Rebuilding search_vector (this may take a few minutes)...');
    await db.execute(`
        UPDATE test_records SET search_vector = to_tsvector('english',
            COALESCE(serial_number, '') || ' ' ||
            COALESCE(model_number, '') || ' ' ||
            COALESCE(raw_data, '')
        )
    `);
    console.log('  search_vector rebuilt.');

    // Re-enable trigger
    await db.execute('ALTER TABLE test_records ENABLE TRIGGER trg_search_vector');

    // Reset sequence to max id
    await db.execute(`SELECT setval('test_records_id_seq', (SELECT COALESCE(MAX(id), 1) FROM test_records))`);

    return migrated;
}

async function insertTestRecordsBatch(batch) {
    // Build a multi-row INSERT
    const cols = ['id', 'log_type', 'model_number', 'serial_number', 'test_date',
                  'test_station', 'overall_result', 'raw_data', 'source_file',
                  'import_date', 'datasheet_exported_at', 'forweb_exported_at', 'work_order'];

    const values = [];
    const params = [];
    let paramIdx = 0;

    for (const row of batch) {
        const placeholders = cols.map(() => {
            paramIdx++;
            return `$${paramIdx}`;
        });
        values.push(`(${placeholders.join(',')})`);

        params.push(
            row.id,
            row.log_type,
            row.model_number,
            row.serial_number,
            row.test_date,
            row.test_station,
            row.overall_result,
            row.raw_data,
            row.source_file,
            row.import_date,
            row.datasheet_exported_at,
            row.forweb_exported_at,
            row.work_order
        );
    }

    const sql = `INSERT INTO test_records (${cols.join(',')})
        VALUES ${values.join(',')}
        ON CONFLICT (log_type, model_number, serial_number, test_date, test_station)
        DO NOTHING`;

    await db.execute(sql, params);
}

async function migrateWorkOrders(sqlite) {
    console.log('\n--- Migrating work_orders ---');

    const rows = sqlite.prepare('SELECT * FROM work_orders ORDER BY id').all();
    console.log(`  Source records: ${rows.length.toLocaleString()}`);

    let migrated = 0;

    const cols = ['wo_number', 'wo_date', 'program', 'version',
                  'lib_version', 'test_station', 'source_file', 'import_date'];

    for (let i = 0; i < rows.length; i += BATCH_SIZE) {
        const batch = rows.slice(i, i + BATCH_SIZE);
        const values = [];
        const params = [];
        let paramIdx = 0;

        for (const row of batch) {
            const placeholders = cols.map(() => { paramIdx++; return `$${paramIdx}`; });
            values.push(`(${placeholders.join(',')})`);
            params.push(row.wo_number, row.wo_date, row.program, row.version,
                        row.lib_version, row.test_station, row.source_file, row.import_date);
        }

        await db.execute(
            `INSERT INTO work_orders (${cols.join(',')}) VALUES ${values.join(',')}
             ON CONFLICT (wo_number, test_station) DO NOTHING`,
            params
        );
        migrated += batch.length;
    }

    console.log(`  Migrated: ${migrated.toLocaleString()}`);
    return migrated;
}

async function migrateWorkOrderLines(sqlite) {
    console.log('\n--- Migrating work_order_lines ---');

    const total = sqlite.prepare('SELECT COUNT(*) as cnt FROM work_order_lines').get().cnt;
    console.log(`  Source records: ${total.toLocaleString()}`);

    const stmt = sqlite.prepare('SELECT * FROM work_order_lines ORDER BY id');
    let migrated = 0;
    let batch = [];

    for (const row of stmt.iterate()) {
        batch.push(row);

        if (batch.length >= BATCH_SIZE) {
            await insertWoLinesBatch(batch);
            migrated += batch.length;
            batch = [];
            process.stdout.write(`\r  Migrated: ${migrated.toLocaleString()} / ${total.toLocaleString()}`);
        }
    }

    if (batch.length > 0) {
        await insertWoLinesBatch(batch);
        migrated += batch.length;
    }

    console.log(`\r  Migrated: ${migrated.toLocaleString()} / ${total.toLocaleString()}`);
    return migrated;
}

async function insertWoLinesBatch(batch) {
    const cols = ['wo_number', 'serial_number', 'status', 'model_number',
                  'ds_filename', 'test_date', 'test_time', 'test_station'];
    const values = [];
    const params = [];
    let paramIdx = 0;

    for (const row of batch) {
        const placeholders = cols.map(() => { paramIdx++; return `$${paramIdx}`; });
        values.push(`(${placeholders.join(',')})`);
        params.push(row.wo_number, row.serial_number, row.status,
                    row.model_number, row.ds_filename, row.test_date, row.test_time, row.test_station);
    }

    await db.execute(
        `INSERT INTO work_order_lines (${cols.join(',')}) VALUES ${values.join(',')}
         ON CONFLICT (wo_number, serial_number, test_date, test_time) DO NOTHING`,
        params
    );
}

async function main() {
    const args = process.argv.slice(2);
    const tableArg = args.indexOf('--table');
    const targetTable = tableArg >= 0 ? args[tableArg + 1] : null;

    console.log('========================================');
    console.log('SQLite -> PostgreSQL Data Migration');
    console.log('========================================');
    console.log(`SQLite: ${SQLITE_PATH}`);
    console.log(`Start: ${new Date().toISOString()}`);

    const sqlite = new Database(SQLITE_PATH, { readonly: true });

    try {
        if (!targetTable || targetTable === 'test_records') {
            await migrateTestRecords(sqlite);
        }
        if (!targetTable || targetTable === 'work_orders') {
            await migrateWorkOrders(sqlite);
        }
        if (!targetTable || targetTable === 'work_order_lines') {
            await migrateWorkOrderLines(sqlite);
        }

        // VACUUM ANALYZE
        console.log('\n--- Running VACUUM ANALYZE ---');
        await db.execute('VACUUM ANALYZE test_records');
        await db.execute('VACUUM ANALYZE work_orders');
        await db.execute('VACUUM ANALYZE work_order_lines');
        console.log('  Done.');

        // Verify counts
        console.log('\n--- Verification ---');
        const pgTestCount = await db.queryOne('SELECT COUNT(*) as cnt FROM test_records');
        const pgWoCount = await db.queryOne('SELECT COUNT(*) as cnt FROM work_orders');
        const pgWolCount = await db.queryOne('SELECT COUNT(*) as cnt FROM work_order_lines');

        const sqliteTestCount = sqlite.prepare('SELECT COUNT(*) as cnt FROM test_records').get().cnt;
        const sqliteWoCount = sqlite.prepare('SELECT COUNT(*) as cnt FROM work_orders').get().cnt;
        const sqliteWolCount = sqlite.prepare('SELECT COUNT(*) as cnt FROM work_order_lines').get().cnt;

        console.log(`  test_records:     SQLite=${sqliteTestCount.toLocaleString()}  PG=${parseInt(pgTestCount.cnt).toLocaleString()}  ${parseInt(pgTestCount.cnt) === sqliteTestCount ? '[OK]' : '[MISMATCH]'}`);
        console.log(`  work_orders:      SQLite=${sqliteWoCount.toLocaleString()}  PG=${parseInt(pgWoCount.cnt).toLocaleString()}  ${parseInt(pgWoCount.cnt) === sqliteWoCount ? '[OK]' : '[MISMATCH]'}`);
        console.log(`  work_order_lines: SQLite=${sqliteWolCount.toLocaleString()}  PG=${parseInt(pgWolCount.cnt).toLocaleString()}  ${parseInt(pgWolCount.cnt) === sqliteWolCount ? '[OK]' : '[MISMATCH]'}`);

    } finally {
        sqlite.close();
        await db.close();
    }

    console.log(`\n========================================`);
    console.log(`Migration Complete`);
    console.log(`========================================`);
    console.log(`End: ${new Date().toISOString()}`);
}

main().catch(err => {
    console.error('Migration failed:', err);
    process.exit(1);
});