claudetools/projects/dataforth-dos/datasheet-pipeline/implementation/database/export-datasheets.js

/**
 * Export Datasheets
 *
 * Generates TXT datasheets for unexported PASS records and writes them to X:\For_Web\.
 * Updates forweb_exported_at after successful export.
 *
 * Usage:
 *   node export-datasheets.js                  Export all pending (batch mode)
 *   node export-datasheets.js --limit 100      Export up to 100 records
 *   node export-datasheets.js --file <paths>   Export records matching specific source files
 *   node export-datasheets.js --serial 178439-1  Export a specific serial number
 *   node export-datasheets.js --dry-run        Show what would be exported without writing
 */

const fs = require('fs');
const path = require('path');
const db = require('./db');

const { loadAllSpecs, getSpecs } = require('../parsers/spec-reader');
const { generateExactDatasheet } = require('../templates/datasheet-exact');

// Configuration
const OUTPUT_DIR = 'X:\\For_Web';
const BATCH_SIZE = 500;

async function run() {
    const args = process.argv.slice(2);
    const dryRun = args.includes('--dry-run');
    const limitIdx = args.indexOf('--limit');
    const limit = limitIdx >= 0 ? parseInt(args[limitIdx + 1]) : 0;
    const serialIdx = args.indexOf('--serial');
    const serial = serialIdx >= 0 ? args[serialIdx + 1] : null;
    const fileIdx = args.indexOf('--file');
    const files = fileIdx >= 0 ? args.slice(fileIdx + 1).filter(f => !f.startsWith('--')) : null;

    console.log('========================================');
    console.log('Datasheet Export');
    console.log('========================================');
    console.log(`Output: ${OUTPUT_DIR}`);
    console.log(`Dry run: ${dryRun}`);
    if (limit) console.log(`Limit: ${limit}`);
    if (serial) console.log(`Serial: ${serial}`);
    console.log(`Start: ${new Date().toISOString()}`);

    if (!dryRun && !fs.existsSync(OUTPUT_DIR)) {
        console.error(`ERROR: Output directory does not exist: ${OUTPUT_DIR}`);
        process.exit(1);
    }

    console.log('\nLoading model specs...');
    const specMap = loadAllSpecs();

    // Build query
    const conditions = [`overall_result = 'PASS'`, `forweb_exported_at IS NULL`];
    const params = [];
    let paramIdx = 0;

    if (serial) {
        paramIdx++;
        conditions.push(`serial_number = $${paramIdx}`);
        params.push(serial);
    }

    if (files && files.length > 0) {
        const placeholders = files.map(() => { paramIdx++; return `$${paramIdx}`; }).join(',');
        conditions.push(`source_file IN (${placeholders})`);
        params.push(...files);
    }

    let sql = `SELECT * FROM test_records WHERE ${conditions.join(' AND ')} ORDER BY test_date DESC`;

    if (limit) {
        paramIdx++;
        sql += ` LIMIT $${paramIdx}`;
        params.push(limit);
    }

    const records = await db.query(sql, params);
    console.log(`\nFound ${records.length} records to export`);

    if (records.length === 0) {
        console.log('Nothing to export.');
        await db.close();
        return { exported: 0, skipped: 0, errors: 0 };
    }

    let exported = 0;
    let skipped = 0;
    let errors = 0;
    let noSpecs = 0;
    let pendingUpdates = [];

    for (const record of records) {
        try {
            const filename = record.serial_number + '.TXT';
            const outputPath = path.join(OUTPUT_DIR, filename);

            // VASLOG_ENG: verbatim byte-for-byte copy of the original file.
            // Using fs.copyFileSync avoids any utf-8 round-trip that would
            // corrupt non-ASCII bytes (CP1252 etc.) in customer datasheets.
            // Fall back to writing raw_data if the source file is gone.
            if (record.log_type === 'VASLOG_ENG') {
                if (dryRun) {
                    console.log(`  [DRY RUN] Would copy: ${record.source_file} -> ${filename}`);
                    exported++;
                    continue;
                }
                if (record.source_file && fs.existsSync(record.source_file)) {
                    fs.copyFileSync(record.source_file, outputPath);
                } else {
                    console.warn(`[WARN] source file missing, writing decoded raw_data for ${record.serial_number}`);
                    if (!record.raw_data) {
                        skipped++;
                        continue;
                    }
                    fs.writeFileSync(outputPath, record.raw_data, 'utf8');
                }
                pendingUpdates.push(record.id);
                exported++;

                if (pendingUpdates.length >= BATCH_SIZE) {
                    await flushUpdates(pendingUpdates);
                    pendingUpdates = [];
                    process.stdout.write(`\r  Exported: ${exported} / ${records.length}`);
                }
                continue;
            }

            // Template-generated datasheet path.
            const specs = getSpecs(specMap, record.model_number);
            if (!specs) {
                noSpecs++;
                skipped++;
                continue;
            }
            const txt = generateExactDatasheet(record, specs);
            if (!txt) {
                skipped++;
                continue;
            }

            if (dryRun) {
                console.log(`  [DRY RUN] Would write: ${filename}`);
                exported++;
            } else {
                fs.writeFileSync(outputPath, txt, 'utf8');
                pendingUpdates.push(record.id);
                exported++;

                // Batch commit
                if (pendingUpdates.length >= BATCH_SIZE) {
                    await flushUpdates(pendingUpdates);
                    pendingUpdates = [];
                    process.stdout.write(`\r  Exported: ${exported} / ${records.length}`);
                }
            }
        } catch (err) {
            console.error(`\n  ERROR exporting ${record.serial_number}: ${err.message}`);
            errors++;
        }
    }

    // Flush remaining updates
    if (pendingUpdates.length > 0) {
        await flushUpdates(pendingUpdates);
    }

    console.log(`\n\n========================================`);
    console.log(`Export Complete`);
    console.log(`========================================`);
    console.log(`Exported:  ${exported}`);
    console.log(`Skipped:   ${skipped} (${noSpecs} missing specs)`);
    console.log(`Errors:    ${errors}`);
    console.log(`End: ${new Date().toISOString()}`);

    await db.close();
    return { exported, skipped, errors };
}

async function flushUpdates(ids) {
    const now = new Date().toISOString();
    await db.transaction(async (txClient) => {
        for (const id of ids) {
            await txClient.execute(
                'UPDATE test_records SET forweb_exported_at = $1 WHERE id = $2',
                [now, id]
            );
        }
    });
}

// Export function for use by import.js (no db argument -- uses shared pool)
async function exportNewRecords(specMap, filePaths) {
    if (!fs.existsSync(OUTPUT_DIR)) {
        console.log(`[EXPORT] Output directory not found: ${OUTPUT_DIR}`);
        return 0;
    }

    const conditions = [`overall_result = 'PASS'`, `forweb_exported_at IS NULL`];
    const params = [];
    let paramIdx = 0;

    if (filePaths && filePaths.length > 0) {
        const placeholders = filePaths.map(() => { paramIdx++; return `$${paramIdx}`; }).join(',');
        conditions.push(`source_file IN (${placeholders})`);
        params.push(...filePaths);
    }

    const sql = `SELECT * FROM test_records WHERE ${conditions.join(' AND ')}`;
    const records = await db.query(sql, params);
    if (records.length === 0) return 0;

    let exported = 0;

    await db.transaction(async (txClient) => {
        for (const record of records) {
            const filename = record.serial_number + '.TXT';
            const outputPath = path.join(OUTPUT_DIR, filename);

            try {
                // VASLOG_ENG: verbatim copy, preserving original bytes.
                if (record.log_type === 'VASLOG_ENG') {
                    if (record.source_file && fs.existsSync(record.source_file)) {
                        fs.copyFileSync(record.source_file, outputPath);
                    } else {
                        console.warn(`[WARN] source file missing, writing decoded raw_data for ${record.serial_number}`);
                        if (!record.raw_data) continue;
                        fs.writeFileSync(outputPath, record.raw_data, 'utf8');
                    }
                } else {
                    const specs = getSpecs(specMap, record.model_number);
                    if (!specs) continue;
                    const txt = generateExactDatasheet(record, specs);
                    if (!txt) continue;
                    fs.writeFileSync(outputPath, txt, 'utf8');
                }

                await txClient.execute(
                    'UPDATE test_records SET forweb_exported_at = $1 WHERE id = $2',
                    [new Date().toISOString(), record.id]
                );
                exported++;
            } catch (err) {
                console.error(`[EXPORT] Error writing ${filename}: ${err.message}`);
            }
        }
    });

    console.log(`[EXPORT] Generated ${exported} datasheet(s)`);
    return exported;
}

if (require.main === module) {
    run().catch(console.error);
}

module.exports = { exportNewRecords };