claudetools/projects/dataforth-dos/datasheet-pipeline/implementation/finish_deployment.py

"""Consolidated single-session script that completes tasks #10, #11, and stages #12.

Runs everything over ONE SSH session to avoid SSH rate-limiting.

Steps:
 1. Deploy inline generator script to AD2
 2. Generate datasheet for SN 179379-1, pull back for visual check (task #10)
 3. Run node import.js to ingest Engineering-Tested .txt files (task #11)
 4. Count VASLOG_ENG records now in DB
 5. Report backlog size for task #12 (full backfill) + stage scheduled-task cmd
 6. Clean up scratch files on AD2
"""
import base64, os, subprocess, yaml, paramiko, time

HOST = '192.168.0.6'
USER = 'sysadmin'
LOCAL_OUT = r'D:\claudetools\projects\dataforth-dos\datasheet-pipeline\scmvas-hvas-research\samples\live-export'
os.makedirs(LOCAL_OUT, exist_ok=True)

GEN_ONE_JS = r'''
const db = require('./database/db');
const { loadAllSpecs, getSpecs } = require('./parsers/spec-reader');
const { generateExactDatasheet } = require('./templates/datasheet-exact');

(async () => {
    const sn = process.argv[2];
    const rows = await db.query(
        "SELECT * FROM test_records WHERE serial_number = $1 AND model_number LIKE 'SCMHVAS%' ORDER BY test_date DESC LIMIT 1",
        [sn]
    );
    if (rows.length === 0) {
        console.error('[FAIL] no SCMHVAS record for ' + sn);
        process.exit(1);
    }
    const record = rows[0];
    console.log('[INFO] model=' + record.model_number +
                ' log_type=' + record.log_type +
                ' date=' + record.test_date +
                ' status=' + record.overall_result);

    const specMap = loadAllSpecs();
    const specs = getSpecs(specMap, record.model_number);
    console.log('[INFO] specs stub keys: ' + (specs ? JSON.stringify(Object.keys(specs)) : 'null'));

    const txt = generateExactDatasheet(record, specs);
    if (!txt) {
        console.error('[FAIL] formatter returned null');
        await db.close();
        process.exit(1);
    }
    console.log('[INFO] generated ' + txt.length + ' bytes');
    console.log('----- BEGIN DATASHEET -----');
    console.log(txt);
    console.log('----- END DATASHEET -----');
    await db.close();
})();
'''

COUNT_JS = r'''
const db = require('./database/db');
(async () => {
    const cnt = await db.queryOne("SELECT COUNT(*) as c FROM test_records WHERE log_type='VASLOG_ENG'");
    console.log('VASLOG_ENG count: ' + cnt.c);

    const scmvas = await db.queryOne(
        "SELECT COUNT(*) as c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL " +
        "AND (model_number LIKE 'SCMVAS%' OR model_number LIKE 'SCMHVAS%' OR model_number LIKE 'VAS-M%' OR model_number LIKE 'HVAS-M%')"
    );
    console.log('SCMVAS/SCMHVAS backlog (no forweb_exported_at): ' + scmvas.c);

    const total = await db.queryOne(
        "SELECT COUNT(*) as c FROM test_records WHERE overall_result='PASS' AND forweb_exported_at IS NULL"
    );
    console.log('Total PASS backlog: ' + total.c);
    await db.close();
})();
'''

def get_pwd():
    r = subprocess.run(['sops', '-d', 'D:/vault/clients/dataforth/ad2.sops.yaml'],
                       capture_output=True, text=True, timeout=30, check=True)
    return yaml.safe_load(r.stdout)['credentials']['password'].replace('\\', '')

def ps(c, cmd, to=300):
    enc = base64.b64encode(cmd.encode('utf-16-le')).decode()
    stdin, stdout, stderr = c.exec_command(f'powershell -NoProfile -EncodedCommand {enc}', timeout=to)
    out = stdout.read().decode('utf-8', 'replace')
    err = stderr.read().decode('utf-8', 'replace')
    rc = stdout.channel.recv_exit_status()
    return out, err, rc

def connect_with_retry():
    last = None
    for i in range(5):
        try:
            c = paramiko.SSHClient()
            c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            c.connect(HOST, username=USER, password=get_pwd(),
                      timeout=30, banner_timeout=45, auth_timeout=30,
                      look_for_keys=False, allow_agent=False)
            return c
        except Exception as e:
            last = e
            print(f'[RETRY {i+1}/5] {type(e).__name__}: {e}')
            time.sleep(15 * (i + 1))
    raise last

def main():
    c = connect_with_retry()
    try:
        sftp = c.open_sftp()
        print('\n=== STEP 1: deploy inline generator ===')
        remote_gen = 'C:/Shares/testdatadb/_gen_one.js'
        remote_count = 'C:/Shares/testdatadb/_count.js'
        with sftp.open(remote_gen, 'w') as fh:
            fh.write(GEN_ONE_JS)
        with sftp.open(remote_count, 'w') as fh:
            fh.write(COUNT_JS)
        print(f'  deployed {remote_gen} and {remote_count}')

        print('\n=== STEP 2: generate datasheet for 179379-1 ===')
        out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_gen_one.js 179379-1')
        print(f'  rc={rc}')
        print(out)
        if err.strip(): print(f'  STDERR: {err[:500]}')

        # Save the generated datasheet to local for inspection
        if '----- BEGIN DATASHEET -----' in out:
            body = out.split('----- BEGIN DATASHEET -----', 1)[1]
            body = body.split('----- END DATASHEET -----', 1)[0]
            body = body.lstrip('\r\n')
            local_dst = os.path.join(LOCAL_OUT, '179379-1.TXT')
            with open(local_dst, 'w', encoding='utf-8', newline='') as fh:
                fh.write(body)
            print(f'  saved locally: {local_dst}')

        print('\n=== STEP 3: run full import to ingest Engineering-Tested .txt ===')
        out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node database/import.js', to=600)
        print(f'  rc={rc}')
        # Only last ~40 lines to avoid log spam
        for line in out.splitlines()[-40:]:
            print(f'  {line}')
        if err.strip(): print(f'  STDERR (first 500): {err[:500]}')

        print('\n=== STEP 4: count VASLOG_ENG records + backlog ===')
        out, err, rc = ps(c, r'cd C:\Shares\testdatadb; & node ./_count.js')
        print(f'  rc={rc}')
        print(out)
        if err.strip(): print(f'  STDERR: {err[:300]}')

        print('\n=== STEP 5: identify service account to stage backfill ===')
        out, err, rc = ps(c, r'Get-WmiObject -Class Win32_Service -Filter "Name=''testdatadb''" | Select-Object Name,StartName,State | Format-List | Out-String')
        print(out)

        print('\n=== STEP 6: cleanup scratch files ===')
        try:
            sftp.remove(remote_gen); print(f'  removed {remote_gen}')
        except Exception as e:
            print(f'  [WARN] remove {remote_gen}: {e}')
        try:
            sftp.remove(remote_count); print(f'  removed {remote_count}')
        except Exception as e:
            print(f'  [WARN] remove {remote_count}: {e}')
        sftp.close()
    finally:
        c.close()

if __name__ == '__main__':
    main()