Mike Swanson
db5395ebe9
Author: Mike Swanson Machine: DESKTOP-0O8A1RL Timestamp: 2026-04-22 12:29:06
305 lines
13 KiB
JavaScript
305 lines
13 KiB
JavaScript
/**
|
|
* Data Import Script
|
|
* Imports test data from DAT and SHT files into PostgreSQL database
|
|
*/
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const db = require('./db');
|
|
const notify = require('./notify');
|
|
|
|
const { parseMultilineFile, extractTestStation } = require('../parsers/multiline');
|
|
const { parseCsvFile } = require('../parsers/csvline');
|
|
const { parseShtFile } = require('../parsers/shtfile');
|
|
const { parseVaslogEngTxt } = require('../parsers/vaslog-engtxt');
|
|
|
|
// Data source paths
|
|
const TEST_PATH = 'C:/Shares/test';
|
|
const RECOVERY_PATH = 'C:/Shares/Recovery-TEST';
|
|
const HISTLOGS_PATH = path.join(TEST_PATH, 'Ate/HISTLOGS');
|
|
|
|
const LOG_TYPES = {
|
|
'DSCLOG': { parser: 'multiline', ext: '.DAT' },
|
|
'5BLOG': { parser: 'multiline', ext: '.DAT' },
|
|
'8BLOG': { parser: 'multiline', ext: '.DAT' },
|
|
'PWRLOG': { parser: 'multiline', ext: '.DAT' },
|
|
'SCTLOG': { parser: 'multiline', ext: '.DAT' },
|
|
'VASLOG': { parser: 'multiline', ext: '.DAT', recursive: false },
|
|
'7BLOG': { parser: 'csvline', ext: '.DAT' },
|
|
'VASLOG_ENG': { parser: 'vaslog-engtxt', ext: '.txt', dir: 'VASLOG/VASLOG - Engineering Tested', recursive: false }
|
|
};
|
|
|
|
function findFiles(dir, pattern, recursive = true) {
|
|
const results = [];
|
|
try {
|
|
if (!fs.existsSync(dir)) return results;
|
|
const items = fs.readdirSync(dir, { withFileTypes: true });
|
|
for (const item of items) {
|
|
const fullPath = path.join(dir, item.name);
|
|
if (item.isDirectory() && recursive) {
|
|
results.push(...findFiles(fullPath, pattern, recursive));
|
|
} else if (item.isFile()) {
|
|
if (pattern.test(item.name)) {
|
|
results.push(fullPath);
|
|
}
|
|
}
|
|
}
|
|
} catch (err) {
|
|
// Ignore permission errors
|
|
}
|
|
return results;
|
|
}
|
|
|
|
function parseFile(filePath, logType, parser) {
|
|
const testStation = extractTestStation(filePath);
|
|
switch (parser) {
|
|
case 'multiline':
|
|
return parseMultilineFile(filePath, logType, testStation);
|
|
case 'csvline':
|
|
return parseCsvFile(filePath, testStation);
|
|
case 'shtfile':
|
|
return parseShtFile(filePath, testStation);
|
|
case 'vaslog-engtxt':
|
|
return parseVaslogEngTxt(filePath, testStation);
|
|
default:
|
|
return [];
|
|
}
|
|
}
|
|
|
|
async function insertBatch(txClient, records) {
|
|
let imported = 0;
|
|
for (const record of records) {
|
|
try {
|
|
const result = await txClient.execute(
|
|
`INSERT INTO test_records
|
|
(log_type, model_number, serial_number, test_date, test_station, overall_result, raw_data, source_file)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
ON CONFLICT (serial_number) DO UPDATE SET
|
|
log_type = EXCLUDED.log_type,
|
|
model_number = EXCLUDED.model_number,
|
|
test_date = EXCLUDED.test_date,
|
|
test_station = EXCLUDED.test_station,
|
|
overall_result = EXCLUDED.overall_result,
|
|
raw_data = EXCLUDED.raw_data,
|
|
source_file = EXCLUDED.source_file,
|
|
api_uploaded_at = NULL,
|
|
forweb_exported_at = NULL
|
|
WHERE test_records.overall_result = 'FAIL'
|
|
OR (EXCLUDED.overall_result = 'PASS' AND EXCLUDED.test_date > test_records.test_date)`,
|
|
[record.log_type, record.model_number, record.serial_number, record.test_date,
|
|
record.test_station, record.overall_result, record.raw_data, record.source_file]
|
|
);
|
|
if (result.rowCount > 0) imported++;
|
|
} catch (err) {
|
|
// Constraint error - skip
|
|
}
|
|
}
|
|
return imported;
|
|
}
|
|
|
|
async function importFile(txClient, filePath, logType, parser) {
|
|
let records = [];
|
|
try {
|
|
records = parseFile(filePath, logType, parser);
|
|
const imported = await insertBatch(txClient, records);
|
|
return { total: records.length, imported };
|
|
} catch (err) {
|
|
console.error(`Error importing ${filePath}: ${err.message}`);
|
|
return { total: 0, imported: 0 };
|
|
}
|
|
}
|
|
|
|
async function importHistlogs(txClient) {
|
|
console.log('\n=== Importing from HISTLOGS ===');
|
|
let totalImported = 0, totalRecords = 0;
|
|
for (const [logType, config] of Object.entries(LOG_TYPES)) {
|
|
const subdir = config.dir || logType;
|
|
const logDir = path.join(HISTLOGS_PATH, subdir);
|
|
if (!fs.existsSync(logDir)) { console.log(` ${logType}: directory not found`); continue; }
|
|
const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), config.recursive !== false);
|
|
console.log(` ${logType}: found ${files.length} files`);
|
|
for (const file of files) {
|
|
const { total, imported } = await importFile(txClient, file, logType, config.parser);
|
|
totalRecords += total; totalImported += imported;
|
|
}
|
|
}
|
|
console.log(` HISTLOGS total: ${totalImported} records imported (${totalRecords} parsed)`);
|
|
return totalImported;
|
|
}
|
|
|
|
async function importStationLogs(txClient, basePath, label) {
|
|
console.log(`\n=== Importing from ${label} ===`);
|
|
let totalImported = 0, totalRecords = 0;
|
|
const stationPattern = /^TS-\d+[LR]?$/i;
|
|
let stations = [];
|
|
try {
|
|
const items = fs.readdirSync(basePath, { withFileTypes: true });
|
|
stations = items.filter(i => i.isDirectory() && stationPattern.test(i.name)).map(i => i.name);
|
|
} catch (err) {
|
|
console.log(` Error reading ${basePath}: ${err.message}`);
|
|
return 0;
|
|
}
|
|
console.log(` Found stations: ${stations.join(', ')}`);
|
|
for (const station of stations) {
|
|
const logsDir = path.join(basePath, station, 'LOGS');
|
|
if (!fs.existsSync(logsDir)) continue;
|
|
for (const [logType, config] of Object.entries(LOG_TYPES)) {
|
|
const subdir = config.dir || logType;
|
|
const logDir = path.join(logsDir, subdir);
|
|
if (!fs.existsSync(logDir)) continue;
|
|
const files = findFiles(logDir, new RegExp(`\\${config.ext}$`, 'i'), config.recursive !== false);
|
|
for (const file of files) {
|
|
const { total, imported } = await importFile(txClient, file, logType, config.parser);
|
|
totalRecords += total; totalImported += imported;
|
|
}
|
|
}
|
|
}
|
|
const shtFiles = findFiles(basePath, /\.SHT$/i, true);
|
|
console.log(` Found ${shtFiles.length} SHT files`);
|
|
for (const file of shtFiles) {
|
|
const { total, imported } = await importFile(txClient, file, 'SHT', 'shtfile');
|
|
totalRecords += total; totalImported += imported;
|
|
}
|
|
console.log(` ${label} total: ${totalImported} records imported (${totalRecords} parsed)`);
|
|
return totalImported;
|
|
}
|
|
|
|
async function importRecoveryBackups(txClient) {
|
|
console.log('\n=== Importing from Recovery-TEST backups ===');
|
|
if (!fs.existsSync(RECOVERY_PATH)) { console.log(' Recovery-TEST directory not found'); return 0; }
|
|
const backups = fs.readdirSync(RECOVERY_PATH, { withFileTypes: true })
|
|
.filter(i => i.isDirectory() && /^\d{2}-\d{2}-\d{2}$/.test(i.name))
|
|
.map(i => i.name).sort().reverse();
|
|
console.log(` Found backup dates: ${backups.join(', ')}`);
|
|
let totalImported = 0;
|
|
for (const backup of backups) {
|
|
const backupPath = path.join(RECOVERY_PATH, backup);
|
|
const imported = await importStationLogs(txClient, backupPath, `Recovery-TEST/${backup}`);
|
|
totalImported += imported;
|
|
}
|
|
return totalImported;
|
|
}
|
|
|
|
async function runImport() {
|
|
console.log('========================================');
|
|
console.log('Test Data Import');
|
|
console.log('========================================');
|
|
console.log(`Start time: ${new Date().toISOString()}`);
|
|
let grandTotal = 0;
|
|
await db.transaction(async (txClient) => {
|
|
grandTotal += await importHistlogs(txClient);
|
|
grandTotal += await importRecoveryBackups(txClient);
|
|
grandTotal += await importStationLogs(txClient, TEST_PATH, 'test');
|
|
});
|
|
const stats = await db.queryOne('SELECT COUNT(*) as count FROM test_records');
|
|
console.log('\n========================================');
|
|
console.log('Import Complete');
|
|
console.log('========================================');
|
|
console.log(`Total records in database: ${stats.count}`);
|
|
console.log(`End time: ${new Date().toISOString()}`);
|
|
await db.close();
|
|
}
|
|
|
|
async function importSingleFile(filePath) {
|
|
console.log(`Importing: ${filePath}`);
|
|
let logType = null, parser = null;
|
|
if (filePath.includes('VASLOG - Engineering Tested')) {
|
|
logType = 'VASLOG_ENG'; parser = LOG_TYPES['VASLOG_ENG'].parser;
|
|
} else {
|
|
for (const [type, config] of Object.entries(LOG_TYPES)) {
|
|
if (type === 'VASLOG_ENG') continue;
|
|
if (filePath.includes(type)) { logType = type; parser = config.parser; break; }
|
|
}
|
|
}
|
|
if (!logType) {
|
|
if (/\.SHT$/i.test(filePath)) { logType = 'SHT'; parser = 'shtfile'; }
|
|
else { console.log(` Unknown log type for: ${filePath}`); return { total: 0, imported: 0 }; }
|
|
}
|
|
let result;
|
|
await db.transaction(async (txClient) => {
|
|
result = await importFile(txClient, filePath, logType, parser);
|
|
});
|
|
console.log(` Imported ${result.imported} of ${result.total} records`);
|
|
return result;
|
|
}
|
|
|
|
async function importFiles(filePaths) {
|
|
console.log(`\n========================================`);
|
|
console.log(`Incremental Import: ${filePaths.length} files`);
|
|
console.log(`========================================`);
|
|
let totalImported = 0, totalRecords = 0;
|
|
await db.transaction(async (txClient) => {
|
|
for (const filePath of filePaths) {
|
|
let logType = null, parser = null;
|
|
if (filePath.includes('VASLOG - Engineering Tested')) {
|
|
logType = 'VASLOG_ENG'; parser = LOG_TYPES['VASLOG_ENG'].parser;
|
|
} else {
|
|
for (const [type, config] of Object.entries(LOG_TYPES)) {
|
|
if (type === 'VASLOG_ENG') continue;
|
|
if (filePath.includes(type)) { logType = type; parser = config.parser; break; }
|
|
}
|
|
}
|
|
if (!logType) {
|
|
if (/\.SHT$/i.test(filePath)) { logType = 'SHT'; parser = 'shtfile'; }
|
|
else { console.log(` Skipping unknown type: ${filePath}`); continue; }
|
|
}
|
|
const { total, imported } = await importFile(txClient, filePath, logType, parser);
|
|
totalRecords += total; totalImported += imported;
|
|
console.log(` ${path.basename(filePath)}: ${imported}/${total} records`);
|
|
}
|
|
});
|
|
console.log(`\nTotal: ${totalImported} records imported (${totalRecords} parsed)`);
|
|
|
|
if (totalImported > 0) {
|
|
try {
|
|
const { loadAllSpecs } = require('../parsers/spec-reader');
|
|
const { exportNewRecords } = require('./export-datasheets');
|
|
const specMap = loadAllSpecs();
|
|
await exportNewRecords(specMap, filePaths);
|
|
} catch (err) {
|
|
console.error(`[EXPORT] Datasheet export failed: ${err.message}`);
|
|
notify.alert('Datasheet export failed', {
|
|
stage: 'export-datasheets (post-import)',
|
|
error: err.message,
|
|
});
|
|
}
|
|
|
|
try {
|
|
const { uploadNewRecords } = require('./upload-to-api');
|
|
await uploadNewRecords(filePaths);
|
|
} catch (err) {
|
|
console.error(`[API-UPLOAD] upload after import failed: ${err.message}`);
|
|
notify.alert('Post-import API upload failed', {
|
|
stage: 'upload-to-api (post-import hook)',
|
|
error: err.message,
|
|
});
|
|
}
|
|
}
|
|
|
|
return { total: totalRecords, imported: totalImported };
|
|
}
|
|
|
|
if (require.main === module) {
|
|
const args = process.argv.slice(2);
|
|
if (args.length > 0 && args[0] === '--file') {
|
|
const files = args.slice(1);
|
|
if (files.length === 0) {
|
|
console.log('Usage: node import.js --file <file1> [file2] ...');
|
|
process.exit(1);
|
|
}
|
|
importFiles(files).then(() => db.close()).catch(console.error);
|
|
} else if (args.length > 0 && args[0] === '--help') {
|
|
console.log('Usage:');
|
|
console.log(' node import.js Full import from all sources');
|
|
console.log(' node import.js --file <f> Import specific file(s)');
|
|
process.exit(0);
|
|
} else {
|
|
runImport().catch(err => {
|
|
console.error(err);
|
|
notify.alert('Full import failed', { stage: 'runImport', error: err.message });
|
|
});
|
|
}
|
|
}
|
|
|
|
module.exports = { runImport, importSingleFile, importFiles };
|