"""Byte-exact verification of backfilled files via a temp copy on AD2.""" import base64, os, subprocess, yaml, paramiko LOCAL_OUT = r'D:\claudetools\projects\dataforth-dos\datasheet-pipeline\scmvas-hvas-research\samples\backfill-verify' os.makedirs(LOCAL_OUT, exist_ok=True) NODE_QUERY = r''' const db = require('./database/db'); (async () => { const rows = await db.query( "SELECT serial_number, model_number, log_type, source_file FROM test_records " + "WHERE forweb_exported_at IS NOT NULL " + "AND log_type='VASLOG_ENG' ORDER BY forweb_exported_at DESC LIMIT 3" ); console.log(JSON.stringify(rows)); await db.close(); })(); ''' def pwd(): r = subprocess.run(['sops','-d','D:/vault/clients/dataforth/ad2.sops.yaml'], capture_output=True, text=True, timeout=30, check=True) return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\','') def ps(c, cmd, to=60): enc = base64.b64encode(cmd.encode('utf-16-le')).decode() stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to) return stdout.read().decode('utf-8','replace'), stderr.read().decode('utf-8','replace'), stdout.channel.recv_exit_status() c = paramiko.SSHClient() c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) c.connect('192.168.0.6', username='sysadmin', password=pwd(), timeout=30, banner_timeout=45, look_for_keys=False, allow_agent=False) try: sftp = c.open_sftp() remote_q = 'C:/Shares/testdatadb/_q.js' with sftp.open(remote_q,'w') as fh: fh.write(NODE_QUERY) sftp.close() out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_q.js') import json rows = json.loads(out[out.find('['):out.rfind(']')+1]) print(f'[INFO] verifying {len(rows)} VASLOG_ENG records') # Copy exported files to C:\Users\sysadmin\Documents for SFTP tmp_dir = 'C:/Users/sysadmin/Documents/verify' ps(c, f'New-Item -ItemType Directory -Force -Path "{tmp_dir}" | Out-Null') sftp = c.open_sftp() for r in rows: sn = r['serial_number'] src_file = r['source_file'] # Copy exported file to tmp ps(c, fr'Copy-Item -LiteralPath "\\ad2\webshare\For_Web\{sn}.TXT" -Destination "{tmp_dir}\{sn}-exp.TXT" -Force') # Also copy source file to tmp (for byte-exact SFTP) ps(c, fr'Copy-Item -LiteralPath "{src_file}" -Destination "{tmp_dir}\{sn}-src.txt" -Force') local_exp = os.path.join(LOCAL_OUT, f'{sn}-exp.TXT') local_src = os.path.join(LOCAL_OUT, f'{sn}-src.txt') sftp.get(f'{tmp_dir}/{sn}-exp.TXT', local_exp) sftp.get(f'{tmp_dir}/{sn}-src.txt', local_src) with open(local_exp, 'rb') as f: exp = f.read() with open(local_src, 'rb') as f: src = f.read() same = exp == src print(f' {sn} ({r["model_number"]}): src={len(src)}B exp={len(exp)}B identical={same}') if not same: print(f' first diff byte: {next((i for i,(a,b) in enumerate(zip(src,exp)) if a != b), min(len(src),len(exp)))}') sftp.close() # Cleanup ps(c, fr'Remove-Item -LiteralPath "{tmp_dir}" -Recurse -Force') sftp = c.open_sftp() try: sftp.remove(remote_q) except Exception: pass sftp.close() finally: c.close()