diff --git a/projects/dataforth-dos/database/import.js b/projects/dataforth-dos/database/import.js new file mode 100644 index 0000000..9015fe7 --- /dev/null +++ b/projects/dataforth-dos/database/import.js @@ -0,0 +1,304 @@ +/** + * Data Import Script + * Imports test data from DAT and SHT files into PostgreSQL database + */ + +const fs = require('fs'); +const path = require('path'); +const db = require('./db'); +const notify = require('./notify'); + +const { parseMultilineFile, extractTestStation } = require('../parsers/multiline'); +const { parseCsvFile } = require('../parsers/csvline'); +const { parseShtFile } = require('../parsers/shtfile'); +const { parseVaslogEngTxt } = require('../parsers/vaslog-engtxt'); + +// Data source paths +const TEST_PATH = 'C:/Shares/test'; +const RECOVERY_PATH = 'C:/Shares/Recovery-TEST'; +const HISTLOGS_PATH = path.join(TEST_PATH, 'Ate/HISTLOGS'); + +const LOG_TYPES = { + 'DSCLOG': { parser: 'multiline', ext: '.DAT' }, + '5BLOG': { parser: 'multiline', ext: '.DAT' }, + '8BLOG': { parser: 'multiline', ext: '.DAT' }, + 'PWRLOG': { parser: 'multiline', ext: '.DAT' }, + 'SCTLOG': { parser: 'multiline', ext: '.DAT' }, + 'VASLOG': { parser: 'multiline', ext: '.DAT', recursive: false }, + '7BLOG': { parser: 'csvline', ext: '.DAT' }, + 'VASLOG_ENG': { parser: 'vaslog-engtxt', ext: '.txt', dir: 'VASLOG/VASLOG - Engineering Tested', recursive: false } +}; + +function findFiles(dir, pattern, recursive = true) { + const results = []; + try { + if (!fs.existsSync(dir)) return results; + const items = fs.readdirSync(dir, { withFileTypes: true }); + for (const item of items) { + const fullPath = path.join(dir, item.name); + if (item.isDirectory() && recursive) { + results.push(...findFiles(fullPath, pattern, recursive)); + } else if (item.isFile()) { + if (pattern.test(item.name)) { + results.push(fullPath); + } + } + } + } catch (err) { + // Ignore permission errors + } + return results; +} + +function parseFile(filePath, logType, parser) { + const testStation = extractTestStation(filePath); + switch (parser) { + case 'multiline': + return parseMultilineFile(filePath, logType, testStation); + case 'csvline': + return parseCsvFile(filePath, testStation); + case 'shtfile': + return parseShtFile(filePath, testStation); + case 'vaslog-engtxt': + return parseVaslogEngTxt(filePath, testStation); + default: + return []; + } +} + +async function insertBatch(txClient, records) { + let imported = 0; + for (const record of records) { + try { + const result = await txClient.execute( + `INSERT INTO test_records + (log_type, model_number, serial_number, test_date, test_station, overall_result, raw_data, source_file) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (serial_number) DO UPDATE SET + log_type = EXCLUDED.log_type, + model_number = EXCLUDED.model_number, + test_date = EXCLUDED.test_date, + test_station = EXCLUDED.test_station, + overall_result = EXCLUDED.overall_result, + raw_data = EXCLUDED.raw_data, + source_file = EXCLUDED.source_file, + api_uploaded_at = NULL, + forweb_exported_at = NULL + WHERE test_records.overall_result = 'FAIL' + OR (EXCLUDED.overall_result = 'PASS' AND EXCLUDED.test_date > test_records.test_date)`, + [record.log_type, record.model_number, record.serial_number, record.test_date, + record.test_station, record.overall_result, record.raw_data, record.source_file] + ); + if (result.rowCount > 0) imported++; + } catch (err) { + // Constraint error - skip + } + } + return imported; +} + +async function importFile(txClient, filePath, logType, parser) { + let records = []; + try { + records = parseFile(filePath, logType, parser); + const imported = await insertBatch(txClient, records); + return { total: records.length, imported }; + } catch (err) { + console.error(`Error importing ${filePath}: ${err.message}`); + return { total: 0, imported: 0 }; + } +} + +async function importHistlogs(txClient) { + console.log('\n=== Importing from HISTLOGS ==='); + let totalImported = 0, totalRecords = 0; + for (const [logType, config] of Object.entries(LOG_TYPES)) { + const subdir = config.dir || logType; + const logDir = path.join(HISTLOGS_PATH, subdir); + if (!fs.existsSync(logDir)) { console.log(` ${logType}: directory not found`); continue; } + const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), config.recursive !== false); + console.log(` ${logType}: found ${files.length} files`); + for (const file of files) { + const { total, imported } = await importFile(txClient, file, logType, config.parser); + totalRecords += total; totalImported += imported; + } + } + console.log(` HISTLOGS total: ${totalImported} records imported (${totalRecords} parsed)`); + return totalImported; +} + +async function importStationLogs(txClient, basePath, label) { + console.log(`\n=== Importing from ${label} ===`); + let totalImported = 0, totalRecords = 0; + const stationPattern = /^TS-\d+[LR]?$/i; + let stations = []; + try { + const items = fs.readdirSync(basePath, { withFileTypes: true }); + stations = items.filter(i => i.isDirectory() && stationPattern.test(i.name)).map(i => i.name); + } catch (err) { + console.log(` Error reading ${basePath}: ${err.message}`); + return 0; + } + console.log(` Found stations: ${stations.join(', ')}`); + for (const station of stations) { + const logsDir = path.join(basePath, station, 'LOGS'); + if (!fs.existsSync(logsDir)) continue; + for (const [logType, config] of Object.entries(LOG_TYPES)) { + const subdir = config.dir || logType; + const logDir = path.join(logsDir, subdir); + if (!fs.existsSync(logDir)) continue; + const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), config.recursive !== false); + for (const file of files) { + const { total, imported } = await importFile(txClient, file, logType, config.parser); + totalRecords += total; totalImported += imported; + } + } + } + const shtFiles = findFiles(basePath, /\.SHT$/i, true); + console.log(` Found ${shtFiles.length} SHT files`); + for (const file of shtFiles) { + const { total, imported } = await importFile(txClient, file, 'SHT', 'shtfile'); + totalRecords += total; totalImported += imported; + } + console.log(` ${label} total: ${totalImported} records imported (${totalRecords} parsed)`); + return totalImported; +} + +async function importRecoveryBackups(txClient) { + console.log('\n=== Importing from Recovery-TEST backups ==='); + if (!fs.existsSync(RECOVERY_PATH)) { console.log(' Recovery-TEST directory not found'); return 0; } + const backups = fs.readdirSync(RECOVERY_PATH, { withFileTypes: true }) + .filter(i => i.isDirectory() && /^\d{2}-\d{2}-\d{2}$/.test(i.name)) + .map(i => i.name).sort().reverse(); + console.log(` Found backup dates: ${backups.join(', ')}`); + let totalImported = 0; + for (const backup of backups) { + const backupPath = path.join(RECOVERY_PATH, backup); + const imported = await importStationLogs(txClient, backupPath, `Recovery-TEST/${backup}`); + totalImported += imported; + } + return totalImported; +} + +async function runImport() { + console.log('========================================'); + console.log('Test Data Import'); + console.log('========================================'); + console.log(`Start time: ${new Date().toISOString()}`); + let grandTotal = 0; + await db.transaction(async (txClient) => { + grandTotal += await importHistlogs(txClient); + grandTotal += await importRecoveryBackups(txClient); + grandTotal += await importStationLogs(txClient, TEST_PATH, 'test'); + }); + const stats = await db.queryOne('SELECT COUNT(*) as count FROM test_records'); + console.log('\n========================================'); + console.log('Import Complete'); + console.log('========================================'); + console.log(`Total records in database: ${stats.count}`); + console.log(`End time: ${new Date().toISOString()}`); + await db.close(); +} + +async function importSingleFile(filePath) { + console.log(`Importing: ${filePath}`); + let logType = null, parser = null; + if (filePath.includes('VASLOG - Engineering Tested')) { + logType = 'VASLOG_ENG'; parser = LOG_TYPES['VASLOG_ENG'].parser; + } else { + for (const [type, config] of Object.entries(LOG_TYPES)) { + if (type === 'VASLOG_ENG') continue; + if (filePath.includes(type)) { logType = type; parser = config.parser; break; } + } + } + if (!logType) { + if (/\.SHT$/i.test(filePath)) { logType = 'SHT'; parser = 'shtfile'; } + else { console.log(` Unknown log type for: ${filePath}`); return { total: 0, imported: 0 }; } + } + let result; + await db.transaction(async (txClient) => { + result = await importFile(txClient, filePath, logType, parser); + }); + console.log(` Imported ${result.imported} of ${result.total} records`); + return result; +} + +async function importFiles(filePaths) { + console.log(`\n========================================`); + console.log(`Incremental Import: ${filePaths.length} files`); + console.log(`========================================`); + let totalImported = 0, totalRecords = 0; + await db.transaction(async (txClient) => { + for (const filePath of filePaths) { + let logType = null, parser = null; + if (filePath.includes('VASLOG - Engineering Tested')) { + logType = 'VASLOG_ENG'; parser = LOG_TYPES['VASLOG_ENG'].parser; + } else { + for (const [type, config] of Object.entries(LOG_TYPES)) { + if (type === 'VASLOG_ENG') continue; + if (filePath.includes(type)) { logType = type; parser = config.parser; break; } + } + } + if (!logType) { + if (/\.SHT$/i.test(filePath)) { logType = 'SHT'; parser = 'shtfile'; } + else { console.log(` Skipping unknown type: ${filePath}`); continue; } + } + const { total, imported } = await importFile(txClient, filePath, logType, parser); + totalRecords += total; totalImported += imported; + console.log(` ${path.basename(filePath)}: ${imported}/${total} records`); + } + }); + console.log(`\nTotal: ${totalImported} records imported (${totalRecords} parsed)`); + + if (totalImported > 0) { + try { + const { loadAllSpecs } = require('../parsers/spec-reader'); + const { exportNewRecords } = require('./export-datasheets'); + const specMap = loadAllSpecs(); + await exportNewRecords(specMap, filePaths); + } catch (err) { + console.error(`[EXPORT] Datasheet export failed: ${err.message}`); + notify.alert('Datasheet export failed', { + stage: 'export-datasheets (post-import)', + error: err.message, + }); + } + + try { + const { uploadNewRecords } = require('./upload-to-api'); + await uploadNewRecords(filePaths); + } catch (err) { + console.error(`[API-UPLOAD] upload after import failed: ${err.message}`); + notify.alert('Post-import API upload failed', { + stage: 'upload-to-api (post-import hook)', + error: err.message, + }); + } + } + + return { total: totalRecords, imported: totalImported }; +} + +if (require.main === module) { + const args = process.argv.slice(2); + if (args.length > 0 && args[0] === '--file') { + const files = args.slice(1); + if (files.length === 0) { + console.log('Usage: node import.js --file [file2] ...'); + process.exit(1); + } + importFiles(files).then(() => db.close()).catch(console.error); + } else if (args.length > 0 && args[0] === '--help') { + console.log('Usage:'); + console.log(' node import.js Full import from all sources'); + console.log(' node import.js --file Import specific file(s)'); + process.exit(0); + } else { + runImport().catch(err => { + console.error(err); + notify.alert('Full import failed', { stage: 'runImport', error: err.message }); + }); + } +} + +module.exports = { runImport, importSingleFile, importFiles };