dotfiles/scripts/system-health.sh

#!/usr/bin/env bash
# System health monitoring and observability
# Provides insights into system performance and potential issues

set -euo pipefail

DOTFILES_DIR="$HOME/dotfiles"
HEALTH_LOG_DIR="$HOME/.health-logs"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
HEALTH_LOG="$HEALTH_LOG_DIR/health-$TIMESTAMP.json"

# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m'

# Create log directory
mkdir -p "$HEALTH_LOG_DIR"

# Function to check system resources
check_system_resources() {
    local cpu_usage memory_usage disk_usage
    
    # CPU usage (1-minute load average normalized by CPU count)
    local load_avg=$(uptime | awk -F'load averages: ' '{print $2}' | cut -d' ' -f1)
    local cpu_count=$(sysctl -n hw.ncpu)
    cpu_usage=$(echo "scale=2; $load_avg / $cpu_count * 100" | bc -l 2>/dev/null || echo "0")
    
    # Memory usage
    local mem_stats=$(vm_stat | grep -E "(free|inactive|wired|compressed)")
    local page_size=$(vm_stat | head -1 | grep -o '[0-9]*')
    local free_pages=$(echo "$mem_stats" | grep "free" | awk '{print $3}' | tr -d '.')
    local inactive_pages=$(echo "$mem_stats" | grep "inactive" | awk '{print $3}' | tr -d '.')
    local wired_pages=$(echo "$mem_stats" | grep "wired" | awk '{print $4}' | tr -d '.')
    local compressed_pages=$(echo "$mem_stats" | grep "compressed" | awk '{print $4}' | tr -d '.')
    
    local total_mem=$(echo "($free_pages + $inactive_pages + $wired_pages + $compressed_pages) * $page_size / 1024 / 1024" | bc -l)
    local used_mem=$(echo "($wired_pages + $compressed_pages) * $page_size / 1024 / 1024" | bc -l)
    memory_usage=$(echo "scale=2; $used_mem / $total_mem * 100" | bc -l)
    
    # Disk usage for root volume
    disk_usage=$(df -h / | awk 'NR==2 {print $5}' | tr -d '%')
    
    echo "{\"cpu_usage\": $cpu_usage, \"memory_usage\": $memory_usage, \"disk_usage\": $disk_usage}"
}

# Function to check critical services
check_critical_services() {
    local services=("Homebrew" "Git" "Zsh" "SSH")
    local service_status=()
    
    # Homebrew
    if command -v brew >/dev/null 2>&1 && brew --version >/dev/null 2>&1; then
        service_status+=('"Homebrew": "healthy"')
    else
        service_status+=('"Homebrew": "unhealthy"')
    fi
    
    # Git
    if command -v git >/dev/null 2>&1 && git --version >/dev/null 2>&1; then
        service_status+=('"Git": "healthy"')
    else
        service_status+=('"Git": "unhealthy"')
    fi
    
    # Zsh
    if [[ -f "$HOME/.zshrc" ]] && zsh -c "source $HOME/.zshrc" >/dev/null 2>&1; then
        service_status+=('"Zsh": "healthy"')
    else
        service_status+=('"Zsh": "unhealthy"')
    fi
    
    # SSH
    if [[ -d "$HOME/.ssh" ]] && [[ -f "$HOME/.ssh/config" ]]; then
        service_status+=('"SSH": "healthy"')
    else
        service_status+=('"SSH": "degraded"')
    fi
    
    echo "{$(IFS=', '; echo "${service_status[*]}")}"
}

# Function to check package health
check_package_health() {
    local total_packages outdated_packages broken_packages
    
    # Count total packages
    total_packages=$(( $(brew list --formula | wc -l) + $(brew list --cask | wc -l) ))
    
    # Count outdated packages
    outdated_packages=$(brew outdated | wc -l | tr -d ' ')
    
    # Check for broken packages
    broken_packages=$(brew doctor 2>&1 | grep -c "Warning\|Error" || echo "0")
    
    echo "{\"total_packages\": $total_packages, \"outdated_packages\": $outdated_packages, \"broken_packages\": $broken_packages}"
}

# Function to check shell performance
check_shell_performance() {
    local shell_load_time plugin_count
    
    # Measure shell load time (rough approximation)
    shell_load_time=$(time (zsh -i -c exit) 2>&1 | grep real | awk '{print $2}' | sed 's/[ms]//g' || echo "0.0")
    
    # Count loaded plugins (if using antidote)
    if [[ -f "$HOME/dotfiles/shell/.zsh-plugins.txt" ]]; then
        plugin_count=$(grep -v '^#' "$HOME/dotfiles/shell/.zsh-plugins.txt" | grep -v '^$' | wc -l | tr -d ' ')
    else
        plugin_count=0
    fi
    
    echo "{\"shell_load_time\": \"$shell_load_time\", \"plugin_count\": $plugin_count}"
}

# Function to check network connectivity
check_network_health() {
    local internet_status dns_status github_status
    
    # Internet connectivity
    if ping -c 1 8.8.8.8 >/dev/null 2>&1; then
        internet_status="connected"
    else
        internet_status="disconnected"
    fi
    
    # DNS resolution
    if nslookup google.com >/dev/null 2>&1; then
        dns_status="working"
    else
        dns_status="failing"
    fi
    
    # GitHub connectivity (important for development)
    if curl -s --connect-timeout 5 https://github.com >/dev/null 2>&1; then
        github_status="accessible"
    else
        github_status="inaccessible"
    fi
    
    echo "{\"internet\": \"$internet_status\", \"dns\": \"$dns_status\", \"github\": \"$github_status\"}"
}

# Function to check security posture
check_security_health() {
    local ssh_key_count gpg_key_count firewall_status
    
    # Count SSH keys
    ssh_key_count=$(find "$HOME/.ssh" -name "id_*" -not -name "*.pub" 2>/dev/null | wc -l | tr -d ' ')
    
    # Count GPG keys
    gpg_key_count=$(gpg --list-secret-keys 2>/dev/null | grep -c "^sec" || echo "0")
    
    # Check firewall status
    if sudo /usr/libexec/ApplicationFirewall/socketfilterfw --getglobalstate 2>/dev/null | grep -q "enabled"; then
        firewall_status="enabled"
    else
        firewall_status="disabled"
    fi
    
    echo "{\"ssh_keys\": $ssh_key_count, \"gpg_keys\": $gpg_key_count, \"firewall\": \"$firewall_status\"}"
}

# Function to assess overall health
assess_overall_health() {
    local resources services packages shell network security
    local cpu_ok mem_ok disk_ok services_ok packages_ok overall_status
    
    # Parse component health
    resources=$(check_system_resources)
    services=$(check_critical_services)
    packages=$(check_package_health)
    shell=$(check_shell_performance)
    network=$(check_network_health)
    security=$(check_security_health)
    
    # Assess individual components
    cpu_usage=$(echo "$resources" | jq -r '.cpu_usage' 2>/dev/null || echo "0")
    mem_usage=$(echo "$resources" | jq -r '.memory_usage' 2>/dev/null || echo "0")
    disk_usage=$(echo "$resources" | jq -r '.disk_usage' 2>/dev/null || echo "0")
    
    # Simple health rules
    cpu_ok=$(echo "$cpu_usage < 80" | bc -l)
    mem_ok=$(echo "$mem_usage < 85" | bc -l)
    disk_ok=$(echo "$disk_usage < 90" | bc -l)
    
    services_ok=1
    if echo "$services" | grep -q "unhealthy"; then
        services_ok=0
    fi
    
    packages_ok=1
    broken_count=$(echo "$packages" | jq -r '.broken_packages' 2>/dev/null || echo "0")
    if [[ "$broken_count" -gt 0 ]]; then
        packages_ok=0
    fi
    
    # Overall assessment
    if [[ "$cpu_ok" == "1" && "$mem_ok" == "1" && "$disk_ok" == "1" && "$services_ok" == "1" && "$packages_ok" == "1" ]]; then
        overall_status="healthy"
    elif [[ "$services_ok" == "0" || "$packages_ok" == "0" ]]; then
        overall_status="unhealthy"
    else
        overall_status="degraded"
    fi
    
    # Create comprehensive health report
    cat > "$HEALTH_LOG" << EOF
{
    "timestamp": "$(date -Iseconds)",
    "system": "$(sw_vers -productVersion)",
    "hostname": "$(hostname)",
    "overall_status": "$overall_status",
    "components": {
        "resources": $resources,
        "services": $services,
        "packages": $packages,
        "shell": $shell,
        "network": $network,
        "security": $security
    },
    "recommendations": []
}
EOF
    
    # Add recommendations based on findings
    if [[ "$cpu_ok" == "0" ]]; then
        echo "$(jq '.recommendations += ["High CPU usage detected - consider closing applications"]' "$HEALTH_LOG")" > "$HEALTH_LOG"
    fi
    
    if [[ "$mem_ok" == "0" ]]; then
        echo "$(jq '.recommendations += ["High memory usage detected - consider restarting applications"]' "$HEALTH_LOG")" > "$HEALTH_LOG"
    fi
    
    if [[ "$disk_ok" == "0" ]]; then
        echo "$(jq '.recommendations += ["Low disk space - consider cleanup with: brew cleanup"]' "$HEALTH_LOG")" > "$HEALTH_LOG"
    fi
    
    echo "$overall_status"
}

# Function to display health dashboard
display_dashboard() {
    local overall_status
    
    echo -e "${BLUE}System Health Dashboard${NC}"
    echo "========================"
    
    overall_status=$(assess_overall_health)
    
    case "$overall_status" in
        "healthy")
            echo -e "${GREEN}✅ System Status: HEALTHY${NC}"
            ;;
        "degraded")
            echo -e "${YELLOW}⚠️  System Status: DEGRADED${NC}"
            ;;
        "unhealthy")
            echo -e "${RED}❌ System Status: UNHEALTHY${NC}"
            ;;
    esac
    
    echo ""
    echo "Component Details:"
    
    # Resources
    local resources=$(jq -r '.components.resources | "CPU: \(.cpu_usage)% | Memory: \(.memory_usage)% | Disk: \(.disk_usage)%"' "$HEALTH_LOG" 2>/dev/null || echo "Resources: N/A")
    echo "  Resources: $resources"
    
    # Services
    local unhealthy_services=$(jq -r '.components.services | to_entries[] | select(.value == "unhealthy") | .key' "$HEALTH_LOG" 2>/dev/null)
    if [[ -n "$unhealthy_services" ]]; then
        echo -e "  ${RED}Unhealthy Services: $unhealthy_services${NC}"
    else
        echo -e "  ${GREEN}Services: All healthy${NC}"
    fi
    
    # Packages
    local outdated=$(jq -r '.components.packages.outdated_packages' "$HEALTH_LOG" 2>/dev/null || echo "0")
    local broken=$(jq -r '.components.packages.broken_packages' "$HEALTH_LOG" 2>/dev/null || echo "0")
    echo "  Packages: $outdated outdated, $broken broken"
    
    # Network
    local network_status=$(jq -r '.components.network | "Internet: \(.internet) | DNS: \(.dns) | GitHub: \(.github)"' "$HEALTH_LOG" 2>/dev/null || echo "Network: N/A")
    echo "  Network: $network_status"
    
    echo ""
    echo "Recommendations:"
    local recommendations=$(jq -r '.recommendations[]' "$HEALTH_LOG" 2>/dev/null)
    if [[ -n "$recommendations" ]]; then
        echo "$recommendations" | sed 's/^/  - /'
    else
        echo "  - No immediate actions required"
    fi
    
    echo ""
    echo "Full report: $HEALTH_LOG"
    
    # Clean old logs (keep last 20)
    find "$HEALTH_LOG_DIR" -name "health-*.json" | sort -r | tail -n +21 | xargs rm -f 2>/dev/null || true
}

# Function to run continuous monitoring
run_monitoring() {
    echo -e "${BLUE}Starting continuous health monitoring...${NC}"
    echo "Press Ctrl+C to stop"
    
    while true; do
        clear
        display_dashboard
        echo ""
        echo "$(date) - Next check in 30 seconds..."
        sleep 30
    done
}

# Main execution
case "${1:-dashboard}" in
    "dashboard"|"-d"|"--dashboard")
        display_dashboard
        ;;
    "monitor"|"-m"|"--monitor")
        run_monitoring
        ;;
    "json"|"-j"|"--json")
        assess_overall_health >/dev/null
        cat "$HEALTH_LOG"
        ;;
    "quiet"|"-q"|"--quiet")
        assess_overall_health
        ;;
    *)
        echo "Usage: $0 [dashboard|monitor|json|quiet]"
        echo "  dashboard (default): Show health dashboard"
        echo "  monitor: Continuous monitoring mode"
        echo "  json: Output raw JSON report"
        echo "  quiet: Just return overall status"
        exit 1
        ;;
esac