sync: Dataforth sync fixes, TestDataDB stability, and client scripts
Dataforth DOS: - TestDataDB: singleton DB connection fix (crash prevention), WAL mode, WinSW service config, backup script, uncaught exception handlers - Sync-FromNAS.ps1: Get-NASFileList temp file approach to avoid SSH stdout deadlock, *> $null output suppression, 8.3 filename filter for PUSH phase, backslash-escaped SCP paths, rename-to-.synced - import.js: INSERT OR REPLACE for re-tested devices - Full import run: 1,028,275 -> 1,632,793 records, indexes added - Deploy script for sync fixes to AD2 Client scripts (temp/): - BG Builders: Lesley account check, MFA phone update - Lonestar Electrical: Kyla/Russ Google Workspace setup, 2FA bypass - AD2 diagnostics and NAS connectivity tests PENDING: Investigate why newest test_date is Jan 19 despite daily tests Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
363
projects/dataforth-dos/testdatadb-fix/routes/api.js
Normal file
363
projects/dataforth-dos/testdatadb-fix/routes/api.js
Normal file
@@ -0,0 +1,363 @@
|
||||
/**
|
||||
* API Routes for Test Data Database
|
||||
*
|
||||
* Fixed version - uses a single persistent database connection instead of
|
||||
* opening and closing on every request. WAL journal mode enabled for
|
||||
* concurrent read support. Limit parameter capped at 1000.
|
||||
*/
|
||||
|
||||
const express = require('express');
|
||||
const path = require('path');
|
||||
const Database = require('better-sqlite3');
|
||||
const { generateDatasheet } = require('../templates/datasheet');
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Singleton database connection - opened once at module load
|
||||
// ---------------------------------------------------------------------------
|
||||
const DB_PATH = path.join(__dirname, '..', 'database', 'testdata.db');
|
||||
|
||||
const db = new Database(DB_PATH, { readonly: false });
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('busy_timeout = 5000');
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
const MAX_LIMIT = 1000;
|
||||
|
||||
function clampLimit(value) {
|
||||
const parsed = parseInt(value, 10);
|
||||
if (isNaN(parsed) || parsed < 1) return 100;
|
||||
return Math.min(parsed, MAX_LIMIT);
|
||||
}
|
||||
|
||||
function clampOffset(value) {
|
||||
const parsed = parseInt(value, 10);
|
||||
if (isNaN(parsed) || parsed < 0) return 0;
|
||||
return parsed;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GET /api/search
|
||||
// Search test records
|
||||
// Query params: serial, model, from, to, result, q, station, logtype, limit, offset
|
||||
// ---------------------------------------------------------------------------
|
||||
router.get('/search', (req, res) => {
|
||||
try {
|
||||
const { serial, model, from, to, result, q, station, logtype } = req.query;
|
||||
const limit = clampLimit(req.query.limit || 100);
|
||||
const offset = clampOffset(req.query.offset || 0);
|
||||
|
||||
let sql = 'SELECT * FROM test_records WHERE 1=1';
|
||||
const params = [];
|
||||
|
||||
if (serial) {
|
||||
sql += ' AND serial_number LIKE ?';
|
||||
params.push(serial.includes('%') ? serial : `%${serial}%`);
|
||||
}
|
||||
|
||||
if (model) {
|
||||
sql += ' AND model_number LIKE ?';
|
||||
params.push(model.includes('%') ? model : `%${model}%`);
|
||||
}
|
||||
|
||||
if (from) {
|
||||
sql += ' AND test_date >= ?';
|
||||
params.push(from);
|
||||
}
|
||||
|
||||
if (to) {
|
||||
sql += ' AND test_date <= ?';
|
||||
params.push(to);
|
||||
}
|
||||
|
||||
if (result) {
|
||||
sql += ' AND overall_result = ?';
|
||||
params.push(result.toUpperCase());
|
||||
}
|
||||
|
||||
if (station) {
|
||||
sql += ' AND test_station = ?';
|
||||
params.push(station);
|
||||
}
|
||||
|
||||
if (logtype) {
|
||||
sql += ' AND log_type = ?';
|
||||
params.push(logtype);
|
||||
}
|
||||
|
||||
if (q) {
|
||||
// Full-text search - rebuild query with FTS
|
||||
sql = `SELECT test_records.* FROM test_records
|
||||
JOIN test_records_fts ON test_records.id = test_records_fts.rowid
|
||||
WHERE test_records_fts MATCH ?`;
|
||||
params.length = 0;
|
||||
params.push(q);
|
||||
|
||||
if (serial) {
|
||||
sql += ' AND serial_number LIKE ?';
|
||||
params.push(serial.includes('%') ? serial : `%${serial}%`);
|
||||
}
|
||||
if (model) {
|
||||
sql += ' AND model_number LIKE ?';
|
||||
params.push(model.includes('%') ? model : `%${model}%`);
|
||||
}
|
||||
if (station) {
|
||||
sql += ' AND test_station = ?';
|
||||
params.push(station);
|
||||
}
|
||||
if (logtype) {
|
||||
sql += ' AND log_type = ?';
|
||||
params.push(logtype);
|
||||
}
|
||||
if (result) {
|
||||
sql += ' AND overall_result = ?';
|
||||
params.push(result.toUpperCase());
|
||||
}
|
||||
if (from) {
|
||||
sql += ' AND test_date >= ?';
|
||||
params.push(from);
|
||||
}
|
||||
if (to) {
|
||||
sql += ' AND test_date <= ?';
|
||||
params.push(to);
|
||||
}
|
||||
}
|
||||
|
||||
sql += ' ORDER BY test_date DESC, serial_number';
|
||||
sql += ' LIMIT ? OFFSET ?';
|
||||
params.push(limit, offset);
|
||||
|
||||
const records = db.prepare(sql).all(...params);
|
||||
|
||||
// Get total count
|
||||
let countSql = sql.replace(/SELECT .* FROM/, 'SELECT COUNT(*) as count FROM')
|
||||
.replace(/ORDER BY.*$/, '');
|
||||
countSql = countSql.replace(/LIMIT \? OFFSET \?/, '');
|
||||
|
||||
const countParams = params.slice(0, -2);
|
||||
const total = db.prepare(countSql).get(...countParams);
|
||||
|
||||
res.json({
|
||||
records,
|
||||
total: total?.count || records.length,
|
||||
limit,
|
||||
offset
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(`[${new Date().toISOString()}] [SEARCH ERROR] ${err.message}`);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GET /api/record/:id
|
||||
// Get single record by ID
|
||||
// ---------------------------------------------------------------------------
|
||||
router.get('/record/:id', (req, res) => {
|
||||
try {
|
||||
const record = db.prepare('SELECT * FROM test_records WHERE id = ?').get(req.params.id);
|
||||
|
||||
if (!record) {
|
||||
return res.status(404).json({ error: 'Record not found' });
|
||||
}
|
||||
|
||||
res.json(record);
|
||||
} catch (err) {
|
||||
console.error(`[${new Date().toISOString()}] [RECORD ERROR] ${err.message}`);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GET /api/datasheet/:id
|
||||
// Generate datasheet for a record
|
||||
// Query params: format (html, txt)
|
||||
// ---------------------------------------------------------------------------
|
||||
router.get('/datasheet/:id', (req, res) => {
|
||||
try {
|
||||
const record = db.prepare('SELECT * FROM test_records WHERE id = ?').get(req.params.id);
|
||||
|
||||
if (!record) {
|
||||
return res.status(404).json({ error: 'Record not found' });
|
||||
}
|
||||
|
||||
const format = req.query.format || 'html';
|
||||
const datasheet = generateDatasheet(record, format);
|
||||
|
||||
if (format === 'html') {
|
||||
res.type('html').send(datasheet);
|
||||
} else {
|
||||
res.type('text/plain').send(datasheet);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[${new Date().toISOString()}] [DATASHEET ERROR] ${err.message}`);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GET /api/stats
|
||||
// Get database statistics
|
||||
// ---------------------------------------------------------------------------
|
||||
router.get('/stats', (req, res) => {
|
||||
try {
|
||||
const stats = {
|
||||
total_records: db.prepare('SELECT COUNT(*) as count FROM test_records').get().count,
|
||||
by_log_type: db.prepare(`
|
||||
SELECT log_type, COUNT(*) as count
|
||||
FROM test_records
|
||||
GROUP BY log_type
|
||||
ORDER BY count DESC
|
||||
`).all(),
|
||||
by_result: db.prepare(`
|
||||
SELECT overall_result, COUNT(*) as count
|
||||
FROM test_records
|
||||
GROUP BY overall_result
|
||||
`).all(),
|
||||
by_station: db.prepare(`
|
||||
SELECT test_station, COUNT(*) as count
|
||||
FROM test_records
|
||||
WHERE test_station IS NOT NULL AND test_station != ''
|
||||
GROUP BY test_station
|
||||
ORDER BY test_station
|
||||
`).all(),
|
||||
date_range: db.prepare(`
|
||||
SELECT MIN(test_date) as oldest, MAX(test_date) as newest
|
||||
FROM test_records
|
||||
`).get(),
|
||||
recent_serials: db.prepare(`
|
||||
SELECT DISTINCT serial_number, model_number, test_date
|
||||
FROM test_records
|
||||
ORDER BY test_date DESC
|
||||
LIMIT 10
|
||||
`).all()
|
||||
};
|
||||
|
||||
res.json(stats);
|
||||
} catch (err) {
|
||||
console.error(`[${new Date().toISOString()}] [STATS ERROR] ${err.message}`);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GET /api/filters
|
||||
// Get available filter options (test stations, log types, models)
|
||||
// ---------------------------------------------------------------------------
|
||||
router.get('/filters', (req, res) => {
|
||||
try {
|
||||
const filters = {
|
||||
stations: db.prepare(`
|
||||
SELECT DISTINCT test_station
|
||||
FROM test_records
|
||||
WHERE test_station IS NOT NULL AND test_station != ''
|
||||
ORDER BY test_station
|
||||
`).all().map(r => r.test_station),
|
||||
log_types: db.prepare(`
|
||||
SELECT DISTINCT log_type
|
||||
FROM test_records
|
||||
ORDER BY log_type
|
||||
`).all().map(r => r.log_type),
|
||||
models: db.prepare(`
|
||||
SELECT DISTINCT model_number, COUNT(*) as count
|
||||
FROM test_records
|
||||
GROUP BY model_number
|
||||
ORDER BY count DESC
|
||||
LIMIT 500
|
||||
`).all()
|
||||
};
|
||||
|
||||
res.json(filters);
|
||||
} catch (err) {
|
||||
console.error(`[${new Date().toISOString()}] [FILTERS ERROR] ${err.message}`);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GET /api/export
|
||||
// Export search results as CSV
|
||||
// ---------------------------------------------------------------------------
|
||||
router.get('/export', (req, res) => {
|
||||
try {
|
||||
const { serial, model, from, to, result, station, logtype } = req.query;
|
||||
|
||||
let sql = 'SELECT * FROM test_records WHERE 1=1';
|
||||
const params = [];
|
||||
|
||||
if (serial) {
|
||||
sql += ' AND serial_number LIKE ?';
|
||||
params.push(serial.includes('%') ? serial : `%${serial}%`);
|
||||
}
|
||||
|
||||
if (model) {
|
||||
sql += ' AND model_number LIKE ?';
|
||||
params.push(model.includes('%') ? model : `%${model}%`);
|
||||
}
|
||||
|
||||
if (from) {
|
||||
sql += ' AND test_date >= ?';
|
||||
params.push(from);
|
||||
}
|
||||
|
||||
if (to) {
|
||||
sql += ' AND test_date <= ?';
|
||||
params.push(to);
|
||||
}
|
||||
|
||||
if (result) {
|
||||
sql += ' AND overall_result = ?';
|
||||
params.push(result.toUpperCase());
|
||||
}
|
||||
|
||||
if (station) {
|
||||
sql += ' AND test_station = ?';
|
||||
params.push(station);
|
||||
}
|
||||
|
||||
if (logtype) {
|
||||
sql += ' AND log_type = ?';
|
||||
params.push(logtype);
|
||||
}
|
||||
|
||||
sql += ' ORDER BY test_date DESC, serial_number LIMIT 10000';
|
||||
|
||||
const records = db.prepare(sql).all(...params);
|
||||
|
||||
// Generate CSV
|
||||
const headers = ['id', 'log_type', 'model_number', 'serial_number', 'test_date', 'test_station', 'overall_result', 'source_file'];
|
||||
let csv = headers.join(',') + '\n';
|
||||
|
||||
for (const record of records) {
|
||||
const row = headers.map(h => {
|
||||
const val = record[h] || '';
|
||||
return `"${String(val).replace(/"/g, '""')}"`;
|
||||
});
|
||||
csv += row.join(',') + '\n';
|
||||
}
|
||||
|
||||
res.setHeader('Content-Type', 'text/csv');
|
||||
res.setHeader('Content-Disposition', 'attachment; filename=test_records.csv');
|
||||
res.send(csv);
|
||||
} catch (err) {
|
||||
console.error(`[${new Date().toISOString()}] [EXPORT ERROR] ${err.message}`);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cleanup function for graceful shutdown
|
||||
// ---------------------------------------------------------------------------
|
||||
function cleanup() {
|
||||
try {
|
||||
db.close();
|
||||
} catch (err) {
|
||||
console.error(`[${new Date().toISOString()}] [CLEANUP ERROR] ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = router;
|
||||
module.exports.cleanup = cleanup;
|
||||
Reference in New Issue
Block a user