claudetools/projects/dataforth-dos/datasheet-pipeline/implementation/investigate_skipped.py

"""Deep-dive on the 438 skipped records.

Looking for patterns: date range, test station, source file, model-family drift,
prior ship status, accuracy magnitude.
"""
import base64, json, subprocess, yaml, paramiko

NODE_SCRIPT = r'''
const db = require('./database/db');

(async () => {
    console.log('======================================================================');
    console.log('SKIPPED RECORDS INVESTIGATION');
    console.log('======================================================================');

    const WHERE_SKIPPED = "overall_result='PASS' AND forweb_exported_at IS NULL " +
        "AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%') " +
        "AND log_type='VASLOG'";

    // -----------------------------------------------------------------------
    console.log('\n--- [1] Date range of SKIPPED vs RENDERED ---');
    const dateRanges = await db.query(
        "SELECT CASE WHEN forweb_exported_at IS NULL THEN 'SKIPPED' ELSE 'RENDERED' END AS status, " +
        "MIN(test_date) mindate, MAX(test_date) maxdate, COUNT(*) cnt " +
        "FROM test_records WHERE overall_result='PASS' AND log_type='VASLOG' " +
        "AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%') " +
        "GROUP BY CASE WHEN forweb_exported_at IS NULL THEN 'SKIPPED' ELSE 'RENDERED' END"
    );
    for (const r of dateRanges) console.log('  ' + r.status.padEnd(10) + ' ' + r.mindate + ' .. ' + r.maxdate + '  (' + r.cnt + ' records)');

    // -----------------------------------------------------------------------
    console.log('\n--- [2] Test station of SKIPPED ---');
    const stations = await db.query(
        "SELECT COALESCE(test_station,'(null)') ts, COUNT(*) cnt FROM test_records " +
        "WHERE " + WHERE_SKIPPED + " GROUP BY test_station ORDER BY cnt DESC"
    );
    for (const r of stations) console.log('  ' + r.ts.padEnd(10) + ' ' + r.cnt);

    // -----------------------------------------------------------------------
    console.log('\n--- [3] Source file of SKIPPED (grouped) ---');
    const sources = await db.query(
        "SELECT source_file, COUNT(*) cnt FROM test_records " +
        "WHERE " + WHERE_SKIPPED + " GROUP BY source_file ORDER BY cnt DESC LIMIT 20"
    );
    for (const r of sources) console.log('  ' + r.cnt.toString().padEnd(6) + ' ' + r.source_file);

    // -----------------------------------------------------------------------
    console.log('\n--- [4] Year distribution: SKIPPED ---');
    const skippedYears = await db.query(
        "SELECT strftime('%Y', test_date) yr, COUNT(*) cnt FROM test_records " +
        "WHERE " + WHERE_SKIPPED + " GROUP BY yr ORDER BY yr"
    );
    for (const r of skippedYears) console.log('  ' + r.yr + ' ' + r.cnt);

    console.log('\n--- [5] Year distribution: RENDERED ---');
    const renderedYears = await db.query(
        "SELECT strftime('%Y', test_date) yr, COUNT(*) cnt FROM test_records " +
        "WHERE overall_result='PASS' AND log_type='VASLOG' AND forweb_exported_at IS NOT NULL " +
        "AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%') " +
        "GROUP BY yr ORDER BY yr"
    );
    for (const r of renderedYears) console.log('  ' + r.yr + ' ' + r.cnt);

    // -----------------------------------------------------------------------
    console.log('\n--- [6] Sample raw_data: SKIPPED vs same-model RENDERED ---');
    const pair = await db.query(
        "SELECT 'SKIPPED' AS tag, serial_number, model_number, test_date, test_station, source_file, raw_data " +
        "FROM test_records WHERE " + WHERE_SKIPPED + " AND model_number='SCMVAS-M700' LIMIT 2"
    );
    const pair2 = await db.query(
        "SELECT 'RENDERED' AS tag, serial_number, model_number, test_date, test_station, source_file, raw_data " +
        "FROM test_records WHERE overall_result='PASS' AND log_type='VASLOG' AND forweb_exported_at IS NOT NULL " +
        "AND model_number='SCMVAS-M700' LIMIT 2"
    );
    for (const r of [...pair, ...pair2]) {
        console.log('  [' + r.tag + '] sn=' + r.serial_number + '  date=' + r.test_date +
                    '  station=' + (r.test_station || '-') + '  src=' + r.source_file);
        console.log('    raw_data: ' + JSON.stringify((r.raw_data||'').replace(/\n/g,'\\n')));
    }

    // -----------------------------------------------------------------------
    console.log('\n--- [7] Accuracy-value magnitude distribution ---');
    const accMag = await db.query(
        "SELECT raw_data FROM test_records WHERE " + WHERE_SKIPPED + " LIMIT 50"
    );
    const vals = [];
    for (const r of accMag) {
        const m = (r.raw_data || '').match(/"(PASS|FAIL)\s*(-?\.?\d+\.?\d*)"/);
        if (m) vals.push(parseFloat(m[2]));
    }
    if (vals.length) {
        const abs = vals.map(Math.abs).sort((a,b)=>a-b);
        console.log('  sample count: ' + vals.length);
        console.log('  min |val|:    ' + abs[0]);
        console.log('  median |val|: ' + abs[Math.floor(abs.length/2)]);
        console.log('  max |val|:    ' + abs[abs.length-1]);
    }

    await db.close();
})();
'''

def pwd():
    r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'],
                       capture_output=True, text=True, timeout=30, check=True)
    return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','')

def ps(c, cmd, to=180):
    enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
    stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
    return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status()

c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False)
try:
    sftp = c.open_sftp()
    remote = 'C:/Shares/testdatadb/_invest.js'
    with sftp.open(remote,'w') as fh: fh.write(NODE_SCRIPT)
    sftp.close()

    out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_invest.js')
    print(out)
    if err.strip() and 'CLIXML' not in err:
        print('--- STDERR ---')
        print(err[:2000])

    sftp = c.open_sftp()
    try: sftp.remove(remote)
    except Exception: pass
    sftp.close()
finally:
    c.close()