claudetools/projects/dataforth-dos/datasheet-pipeline/implementation/import_engtxt.py

"""Targeted import of the 434 VASLOG Engineering-Tested .txt files.

Runs node import.js --file <batch> to import directly, then counts VASLOG_ENG
records in the DB. Avoids the slow full-import walk.
"""
import base64, os, subprocess, yaml, paramiko, sys

HOST = '192.168.0.6'
USER = 'sysadmin'
REMOTE_DIR = r'C:\Shares\test\TS-3R\LOGS\VASLOG\VASLOG - Engineering Tested'

def pwd():
    r = subprocess.run(['sops', '-d', 'D:/vault/clients/dataforth/ad2.sops.yaml'],
                       capture_output=True, text=True, timeout=30, check=True)
    return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\', '')

def ps(c, cmd, to=600):
    enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
    stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
    out = stdout.read().decode('utf-8', 'replace')
    err = stderr.read().decode('utf-8', 'replace')
    rc = stdout.channel.recv_exit_status()
    return out, err, rc

def main():
    c = paramiko.SSHClient()
    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    c.connect(HOST, username=USER, password=pwd(), timeout=30, banner_timeout=45,
              look_for_keys=False, allow_agent=False)
    sys.stdout.flush()
    try:
        print('[STEP 1] List Engineering-Tested .txt files on AD2', flush=True)
        out, err, rc = ps(c, f'Get-ChildItem -LiteralPath "{REMOTE_DIR}" -File -Filter *.txt | ForEach-Object {{ $_.FullName }}')
        files = [l.strip() for l in out.splitlines() if l.strip()]
        print(f'  found {len(files)} .txt files', flush=True)

        if not files:
            print('  [WARN] no files found', flush=True)
            return

        print('[STEP 2] Build PowerShell command array and invoke import.js --file', flush=True)
        # Build a PS array literal to pass to node. We chunk to avoid CLI length limits.
        CHUNK = 50
        total_imported = 0
        total_parsed = 0
        for i in range(0, len(files), CHUNK):
            batch = files[i:i+CHUNK]
            # PowerShell @() array with paths quoted
            quoted = ','.join(f'"{p}"' for p in batch)
            script = (
                r'cd C:\Shares\testdatadb; ' +
                f'$files = @({quoted}); ' +
                r'& node database/import.js --file @files 2>&1'
            )
            out, err, rc = ps(c, script, to=300)
            lines = out.splitlines()
            # Print a summary tail of each chunk
            tail = [l for l in lines if 'records' in l.lower() or 'total' in l.lower() or 'error' in l.lower()]
            print(f'  chunk {i//CHUNK + 1} ({len(batch)} files): rc={rc}', flush=True)
            for t in tail[-4:]:
                print(f'    {t}', flush=True)
            if err.strip() and 'CLIXML' not in err:
                print(f'    STDERR: {err[:400]}', flush=True)

        print('[STEP 3] Count VASLOG_ENG in DB', flush=True)
        script = (
            r'cd C:\Shares\testdatadb; & node -e "'
            r"const db=require('./database/db');"
            r"(async()=>{const r=await db.queryOne(\"SELECT COUNT(*) c FROM test_records WHERE log_type='VASLOG_ENG'\");"
            r'console.log(\"VASLOG_ENG rows: \"+r.c);await db.close();})();"'
        )
        out, err, rc = ps(c, script, to=60)
        print(out, flush=True)
        if err.strip() and 'CLIXML' not in err:
            print(f'  STDERR: {err[:400]}', flush=True)

        print('[STEP 4] Cleanup scratch files on AD2', flush=True)
        sftp = c.open_sftp()
        for scratch in ['C:/Shares/testdatadb/_gen_one.js', 'C:/Shares/testdatadb/_count.js']:
            try:
                sftp.remove(scratch)
                print(f'  removed {scratch}', flush=True)
            except Exception as e:
                print(f'  [WARN] {scratch}: {e}', flush=True)
        sftp.close()
    finally:
        c.close()

if __name__ == '__main__':
    main()