claudetools/verify-rollout-system.sh

#!/usr/bin/env bash
# Verification script for Safe Agent Rollout System
# Run on gururmm-build (172.16.3.30) to verify Phase 1-5 implementation
#
# =====================================================================
# [WARNING] STATUS - 2026-05-25 re-audit: the safe-rollout gating this script
# verifies is NOT live. update_rollouts / update_health_metrics are written but
# never read to gate promotion (gururmm docs/FEATURE_ROADMAP.md BUG-004); crash
# detection was dead code until the BUG-002 fix (branch fix/audit-2-remediation,
# unmerged). Promotion is currently 100% manual. This verifies INERT scaffolding
# - Phase-2 aspirational until the gating is re-spec'd and wired. Decision
# 2026-05-25 (Mike): keep the feature inert and labeled.
# =====================================================================

set -e

GREEN='\033[0;32m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
NC='\033[0m'

echo "=========================================="
echo "GuruRMM Safe Rollout System Verification"
echo "=========================================="
echo ""

# Function to check status
check() {
    if [ $? -eq 0 ]; then
        echo -e "${GREEN}[OK]${NC} $1"
        return 0
    else
        echo -e "${RED}[FAIL]${NC} $1"
        return 1
    fi
}

info() {
    echo -e "${YELLOW}[INFO]${NC} $1"
}

FAIL_COUNT=0

# ===== Phase 1: Build Scripts =====
echo "Phase 1: Build Scripts"
echo "----------------------"

if grep -q "Mark all new builds as beta" /opt/gururmm/build-linux.sh; then
    check "build-linux.sh has beta marking code"
else
    check "build-linux.sh missing beta marking code"
    ((FAIL_COUNT++))
fi

if grep -q -i "mark.*beta" /opt/gururmm/build-windows.sh; then
    check "build-windows.sh has beta marking code"
else
    check "build-windows.sh missing beta marking code"
    ((FAIL_COUNT++))
fi

# Check for actual .channel files
CHANNEL_COUNT=$(find /var/www/gururmm/downloads -name "*.channel" 2>/dev/null | wc -l)
if [ "$CHANNEL_COUNT" -gt 0 ]; then
    check ".channel files exist in downloads directory ($CHANNEL_COUNT found)"
    info "Sample: $(find /var/www/gururmm/downloads -name "*.channel" | head -1)"
    SAMPLE_FILE=$(find /var/www/gururmm/downloads -name "*.channel" | head -1)
    if [ -f "$SAMPLE_FILE" ]; then
        SAMPLE_CONTENT=$(cat "$SAMPLE_FILE")
        info "Content: $SAMPLE_CONTENT"
    fi
else
    check ".channel files in downloads directory"
    ((FAIL_COUNT++))
    info "No .channel files found - may need to trigger a build"
fi

echo ""

# ===== Phase 2: Database Migration =====
echo "Phase 2: Database Migration"
echo "---------------------------"

# Check tables exist
if sudo -u postgres psql gururmm -t -c "\d update_rollouts" &>/dev/null; then
    check "update_rollouts table exists"
else
    check "update_rollouts table exists"
    ((FAIL_COUNT++))
fi

if sudo -u postgres psql gururmm -t -c "\d update_health_metrics" &>/dev/null; then
    check "update_health_metrics table exists"
else
    check "update_health_metrics table exists"
    ((FAIL_COUNT++))
fi

if sudo -u postgres psql gururmm -t -c "\d agent_update_events" &>/dev/null; then
    check "agent_update_events table exists"
else
    check "agent_update_events table exists"
    ((FAIL_COUNT++))
fi

# Check for data
ROLLOUT_COUNT=$(sudo -u postgres psql gururmm -t -c "SELECT COUNT(*) FROM update_rollouts" 2>/dev/null | xargs)
info "Rollouts tracked: $ROLLOUT_COUNT"

EVENT_COUNT=$(sudo -u postgres psql gururmm -t -c "SELECT COUNT(*) FROM agent_update_events" 2>/dev/null | xargs)
info "Update events logged: $EVENT_COUNT"

METRIC_COUNT=$(sudo -u postgres psql gururmm -t -c "SELECT COUNT(*) FROM update_health_metrics" 2>/dev/null | xargs)
info "Health metrics tracked: $METRIC_COUNT"

echo ""

# ===== Phase 3: Health Monitoring =====
echo "Phase 3: Health Monitoring"
echo "--------------------------"

# Check source files exist
if [ -f "/home/guru/gururmm/server/src/updates/health.rs" ]; then
    check "health.rs source file exists"
else
    check "health.rs source file exists"
    ((FAIL_COUNT++))
fi

# Check if server is running
if systemctl is-active --quiet gururmm-server; then
    check "GuruRMM server is running"

    # Check for health monitor in logs
    if sudo journalctl -u gururmm-server --since "1 hour ago" | grep -q "Health monitoring task spawned"; then
        check "Health monitor task spawned (found in logs)"
    else
        echo -e "${YELLOW}[WARN]${NC} Health monitor spawn message not found in recent logs"
        info "May need to restart service if code just deployed"
    fi
else
    check "GuruRMM server is running"
    ((FAIL_COUNT++))
fi

echo ""

# ===== Phase 4: API Endpoints =====
echo "Phase 4: API Endpoints"
echo "----------------------"

if [ -f "/home/guru/gururmm/server/src/api/updates.rs" ]; then
    check "updates.rs API file exists"

    # Check for key functions
    if grep -q "pub async fn list_rollouts" /home/guru/gururmm/server/src/api/updates.rs; then
        check "list_rollouts endpoint defined"
    else
        check "list_rollouts endpoint defined"
        ((FAIL_COUNT++))
    fi

    if grep -q "pub async fn promote_version" /home/guru/gururmm/server/src/api/updates.rs; then
        check "promote_version endpoint defined"
    else
        check "promote_version endpoint defined"
        ((FAIL_COUNT++))
    fi

    if grep -q "pub async fn rollback_version" /home/guru/gururmm/server/src/api/updates.rs; then
        check "rollback_version endpoint defined"
    else
        check "rollback_version endpoint defined"
        ((FAIL_COUNT++))
    fi
else
    check "updates.rs API file exists"
    ((FAIL_COUNT++))
fi

# Check routes registered
if grep -q "updates::list_rollouts" /home/guru/gururmm/server/src/api/mod.rs; then
    check "API routes registered in mod.rs"
else
    check "API routes registered in mod.rs"
    ((FAIL_COUNT++))
fi

echo ""

# ===== Phase 5: Dashboard UI =====
echo "Phase 5: Dashboard UI"
echo "---------------------"

if [ -f "/home/guru/gururmm/dashboard/src/pages/Updates.tsx" ]; then
    check "Updates.tsx page exists"

    # Check for key components
    if grep -q "RolloutInfo" /home/guru/gururmm/dashboard/src/pages/Updates.tsx; then
        check "RolloutInfo interface defined"
    else
        check "RolloutInfo interface defined"
        ((FAIL_COUNT++))
    fi

    if grep -q "handlePromote" /home/guru/gururmm/dashboard/src/pages/Updates.tsx; then
        check "Promote functionality implemented"
    else
        check "Promote functionality implemented"
        ((FAIL_COUNT++))
    fi

    if grep -q "handleRollback" /home/guru/gururmm/dashboard/src/pages/Updates.tsx; then
        check "Rollback functionality implemented"
    else
        check "Rollback functionality implemented"
        ((FAIL_COUNT++))
    fi
else
    check "Updates.tsx page exists"
    ((FAIL_COUNT++))
fi

# Check navigation
if grep -q "/updates" /home/guru/gururmm/dashboard/src/App.tsx; then
    check "Updates route registered in App.tsx"
else
    check "Updates route registered in App.tsx"
    ((FAIL_COUNT++))
fi

if grep -q "updates" /home/guru/gururmm/dashboard/src/components/Layout.tsx; then
    check "Updates navigation link added"
else
    check "Updates navigation link added"
    ((FAIL_COUNT++))
fi

echo ""

# ===== Build Status =====
echo "Build Status"
echo "------------"

# Check server binary
if [ -f "/opt/gururmm/gururmm-server" ]; then
    SERVER_SIZE=$(stat -f%z "/opt/gururmm/gururmm-server" 2>/dev/null || stat -c%s "/opt/gururmm/gururmm-server" 2>/dev/null)
    SERVER_DATE=$(stat -f%Sm "/opt/gururmm/gururmm-server" 2>/dev/null || stat -c%y "/opt/gururmm/gururmm-server" 2>/dev/null | cut -d' ' -f1)
    check "Server binary exists (${SERVER_SIZE} bytes, ${SERVER_DATE})"
else
    check "Server binary exists"
    ((FAIL_COUNT++))
fi

# Check dashboard build
if [ -d "/home/guru/gururmm/dashboard/dist" ]; then
    check "Dashboard build exists"
else
    echo -e "${YELLOW}[WARN]${NC} Dashboard dist/ directory not found - may need to run 'npm run build'"
fi

echo ""

# ===== Summary =====
echo "=========================================="
echo "Verification Summary"
echo "=========================================="

if [ $FAIL_COUNT -eq 0 ]; then
    echo -e "${GREEN}✓ All checks passed!${NC}"
    echo ""
    echo "Safe Agent Rollout System is ready for Phase 6 testing."
    echo ""
    echo "Next steps:"
    echo "  1. Review PHASE_6_TEST_PLAN.md"
    echo "  2. Execute Test 1: Beta-first build workflow"
    echo "  3. Execute Test 2-4: Health monitoring, promotion, rollback"
    echo "  4. Execute Test 5: Dashboard UI testing"
    echo "  5. Execute Test 6: Integration testing"
    exit 0
else
    echo -e "${RED}✗ ${FAIL_COUNT} check(s) failed${NC}"
    echo ""
    echo "Review failures above and fix before proceeding to Phase 6."
    echo ""
    echo "Common issues:"
    echo "  - Code not deployed (git pull + rebuild needed)"
    echo "  - Migration not applied (run migration 046)"
    echo "  - Service not restarted (systemctl restart gururmm-server)"
    echo "  - Build not triggered (no .channel files yet)"
    exit 1
fi