"""Consolidated single-session script that completes tasks #10, #11, and stages #12. Runs everything over ONE SSH session to avoid SSH rate-limiting. Steps: 1. Deploy inline generator script to AD2 2. Generate datasheet for SN 179379-1, pull back for visual check (task #10) 3. Run node import.js to ingest Engineering-Tested .txt files (task #11) 4. Count VASLOG_ENG records now in DB 5. Report backlog size for task #12 (full backfill) + stage scheduled-task cmd 6. Clean up scratch files on AD2 """ import base64, os, subprocess, yaml, paramiko, time HOST = '192.168.0.6' USER = 'sysadmin' LOCAL_OUT = r'D:\claudetools\projects\dataforth-dos\datasheet-pipeline\scmvas-hvas-research\samples\live-export' os.makedirs(LOCAL_OUT, exist_ok=True) GEN_ONE_JS = r''' const db = require('./database/db'); const { loadAllSpecs, getSpecs } = require('./parsers/spec-reader'); const { generateExactDatasheet } = require('./templates/datasheet-exact'); (async () => { const sn = process.argv[2]; const rows = await db.query( "SELECT * FROM test_records WHERE serial_number = $1 AND model_number LIKE 'SCMHVAS%' ORDER BY test_date DESC LIMIT 1", [sn] ); if (rows.length === 0) { console.error('[FAIL] no SCMHVAS record for ' + sn); process.exit(1); } const record = rows[0]; console.log('[INFO] model=' + record.model_number + ' log_type=' + record.log_type + ' date=' + record.test_date + ' status=' + record.overall_result); const specMap = loadAllSpecs(); const specs = getSpecs(specMap, record.model_number); console.log('[INFO] specs stub keys: ' + (specs ? JSON.stringify(Object.keys(specs)) : 'null')); const txt = generateExactDatasheet(record, specs); if (!txt) { console.error('[FAIL] formatter returned null'); await db.close(); process.exit(1); } console.log('[INFO] generated ' + txt.length + ' bytes'); console.log('----- BEGIN DATASHEET -----'); console.log(txt); console.log('----- END DATASHEET -----'); await db.close(); })(); ''' COUNT_JS = r''' const db = require('./database/db'); (async () => { const cnt = await db.queryOne("SELECT COUNT(*) as c FROM test_records WHERE log_type='VASLOG_ENG'"); console.log('VASLOG_ENG count: ' + cnt.c); const scmvas = await db.queryOne( "SELECT COUNT(*) as c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL " + "AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%' OR model_number LIKE 'VAS-M%' OR model_number LIKE 'HVAS-M%')" ); console.log('SCMVAS/SCMHVAS backlog (no forweb_exported_at): ' + scmvas.c); const total = await db.queryOne( "SELECT COUNT(*) as c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL" ); console.log('Total PASS backlog: ' + total.c); await db.close(); })(); ''' def get_pwd(): r = subprocess.run(['sops', '-d', 'D:/vault/clients/dataforth/ad2.sops.yaml'], capture_output=True, text=True, timeout=30, check=True) return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\', '') def ps(c, cmd, to=300): enc = base64.b64encode(cmd.encode('utf-16-le')).decode() stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to) out = stdout.read().decode('utf-8', 'replace') err = stderr.read().decode('utf-8', 'replace') rc = stdout.channel.recv_exit_status() return out, err, rc def connect_with_retry(): last = None for i in range(5): try: c = paramiko.SSHClient() c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) c.connect(HOST, username=USER, password=get_pwd(), timeout=30, banner_timeout=45, auth_timeout=30, look_for_keys=False, allow_agent=False) return c except Exception as e: last = e print(f'[RETRY {i+1}/5] {type(e).__name__}: {e}') time.sleep(15 * (i + 1)) raise last def main(): c = connect_with_retry() try: sftp = c.open_sftp() print('\n=== STEP 1: deploy inline generator ===') remote_gen = 'C:/Shares/testdatadb/_gen_one.js' remote_count = 'C:/Shares/testdatadb/_count.js' with sftp.open(remote_gen, 'w') as fh: fh.write(GEN_ONE_JS) with sftp.open(remote_count, 'w') as fh: fh.write(COUNT_JS) print(f' deployed {remote_gen} and {remote_count}') print('\n=== STEP 2: generate datasheet for 179379-1 ===') out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_gen_one.js 179379-1') print(f' rc={rc}') print(out) if err.strip(): print(f' STDERR: {err[:500]}') # Save the generated datasheet to local for inspection if '----- BEGIN DATASHEET -----' in out: body = out.split('----- BEGIN DATASHEET -----', 1)[1] body = body.split('----- END DATASHEET -----', 1)[0] body = body.lstrip('\r\n') local_dst = os.path.join(LOCAL_OUT, '179379-1.TXT') with open(local_dst, 'w', encoding='utf-8', newline='') as fh: fh.write(body) print(f' saved locally: {local_dst}') print('\n=== STEP 3: run full import to ingest Engineering-Tested .txt ===') out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node database/import.js', to=600) print(f' rc={rc}') # Only last ~40 lines to avoid log spam for line in out.splitlines()[-40:]: print(f' {line}') if err.strip(): print(f' STDERR (first 500): {err[:500]}') print('\n=== STEP 4: count VASLOG_ENG records + backlog ===') out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_count.js') print(f' rc={rc}') print(out) if err.strip(): print(f' STDERR: {err[:300]}') print('\n=== STEP 5: identify service account to stage backfill ===') out, err, rc = ps(c, r'Get-WmiObject -Class Win32_Service -Filter "Name=''testdatadb''" | Select-Object Name,StartName,State | Format-List | Out-String') print(out) print('\n=== STEP 6: cleanup scratch files ===') try: sftp.remove(remote_gen); print(f' removed {remote_gen}') except Exception as e: print(f' [WARN] remove {remote_gen}: {e}') try: sftp.remove(remote_count); print(f' removed {remote_count}') except Exception as e: print(f' [WARN] remove {remote_count}: {e}') sftp.close() finally: c.close() if __name__ == '__main__': main()