#!/usr/bin/env bash # System health monitoring and observability # Provides insights into system performance and potential issues set -euo pipefail DOTFILES_DIR="$HOME/dotfiles" HEALTH_LOG_DIR="$HOME/.health-logs" TIMESTAMP=$(date +%Y%m%d_%H%M%S) HEALTH_LOG="$HEALTH_LOG_DIR/health-$TIMESTAMP.json" # Colors GREEN='\033[0;32m' YELLOW='\033[1;33m' RED='\033[0;31m' BLUE='\033[0;34m' NC='\033[0m' # Create log directory mkdir -p "$HEALTH_LOG_DIR" # Function to check system resources check_system_resources() { local cpu_usage memory_usage disk_usage # CPU usage (1-minute load average normalized by CPU count) local load_avg=$(uptime | awk -F'load averages: ' '{print $2}' | cut -d' ' -f1) local cpu_count=$(sysctl -n hw.ncpu) cpu_usage=$(echo "scale=2; $load_avg / $cpu_count * 100" | bc -l 2>/dev/null || echo "0") # Memory usage local mem_stats=$(vm_stat | grep -E "(free|inactive|wired|compressed)") local page_size=$(vm_stat | head -1 | grep -o '[0-9]*') local free_pages=$(echo "$mem_stats" | grep "free" | awk '{print $3}' | tr -d '.') local inactive_pages=$(echo "$mem_stats" | grep "inactive" | awk '{print $3}' | tr -d '.') local wired_pages=$(echo "$mem_stats" | grep "wired" | awk '{print $4}' | tr -d '.') local compressed_pages=$(echo "$mem_stats" | grep "compressed" | awk '{print $4}' | tr -d '.') local total_mem=$(echo "($free_pages + $inactive_pages + $wired_pages + $compressed_pages) * $page_size / 1024 / 1024" | bc -l) local used_mem=$(echo "($wired_pages + $compressed_pages) * $page_size / 1024 / 1024" | bc -l) memory_usage=$(echo "scale=2; $used_mem / $total_mem * 100" | bc -l) # Disk usage for root volume disk_usage=$(df -h / | awk 'NR==2 {print $5}' | tr -d '%') echo "{\"cpu_usage\": $cpu_usage, \"memory_usage\": $memory_usage, \"disk_usage\": $disk_usage}" } # Function to check critical services check_critical_services() { local services=("Homebrew" "Git" "Zsh" "SSH") local service_status=() # Homebrew if command -v brew >/dev/null 2>&1 && brew --version >/dev/null 2>&1; then service_status+=('"Homebrew": "healthy"') else service_status+=('"Homebrew": "unhealthy"') fi # Git if command -v git >/dev/null 2>&1 && git --version >/dev/null 2>&1; then service_status+=('"Git": "healthy"') else service_status+=('"Git": "unhealthy"') fi # Zsh if [[ -f "$HOME/.zshrc" ]] && zsh -c "source $HOME/.zshrc" >/dev/null 2>&1; then service_status+=('"Zsh": "healthy"') else service_status+=('"Zsh": "unhealthy"') fi # SSH if [[ -d "$HOME/.ssh" ]] && [[ -f "$HOME/.ssh/config" ]]; then service_status+=('"SSH": "healthy"') else service_status+=('"SSH": "degraded"') fi echo "{$(IFS=', '; echo "${service_status[*]}")}" } # Function to check package health check_package_health() { local total_packages outdated_packages broken_packages # Count total packages total_packages=$(( $(brew list --formula | wc -l) + $(brew list --cask | wc -l) )) # Count outdated packages outdated_packages=$(brew outdated | wc -l | tr -d ' ') # Check for broken packages broken_packages=$(brew doctor 2>&1 | grep -c "Warning\|Error" || echo "0") echo "{\"total_packages\": $total_packages, \"outdated_packages\": $outdated_packages, \"broken_packages\": $broken_packages}" } # Function to check shell performance check_shell_performance() { local shell_load_time plugin_count # Measure shell load time (rough approximation) shell_load_time=$(time (zsh -i -c exit) 2>&1 | grep real | awk '{print $2}' | sed 's/[ms]//g' || echo "0.0") # Count loaded plugins (if using antidote) if [[ -f "$HOME/dotfiles/shell/.zsh-plugins.txt" ]]; then plugin_count=$(grep -v '^#' "$HOME/dotfiles/shell/.zsh-plugins.txt" | grep -v '^$' | wc -l | tr -d ' ') else plugin_count=0 fi echo "{\"shell_load_time\": \"$shell_load_time\", \"plugin_count\": $plugin_count}" } # Function to check network connectivity check_network_health() { local internet_status dns_status github_status # Internet connectivity if ping -c 1 8.8.8.8 >/dev/null 2>&1; then internet_status="connected" else internet_status="disconnected" fi # DNS resolution if nslookup google.com >/dev/null 2>&1; then dns_status="working" else dns_status="failing" fi # GitHub connectivity (important for development) if curl -s --connect-timeout 5 https://github.com >/dev/null 2>&1; then github_status="accessible" else github_status="inaccessible" fi echo "{\"internet\": \"$internet_status\", \"dns\": \"$dns_status\", \"github\": \"$github_status\"}" } # Function to check security posture check_security_health() { local ssh_key_count gpg_key_count firewall_status # Count SSH keys ssh_key_count=$(find "$HOME/.ssh" -name "id_*" -not -name "*.pub" 2>/dev/null | wc -l | tr -d ' ') # Count GPG keys gpg_key_count=$(gpg --list-secret-keys 2>/dev/null | grep -c "^sec" || echo "0") # Check firewall status if sudo /usr/libexec/ApplicationFirewall/socketfilterfw --getglobalstate 2>/dev/null | grep -q "enabled"; then firewall_status="enabled" else firewall_status="disabled" fi echo "{\"ssh_keys\": $ssh_key_count, \"gpg_keys\": $gpg_key_count, \"firewall\": \"$firewall_status\"}" } # Function to assess overall health assess_overall_health() { local resources services packages shell network security local cpu_ok mem_ok disk_ok services_ok packages_ok overall_status # Parse component health resources=$(check_system_resources) services=$(check_critical_services) packages=$(check_package_health) shell=$(check_shell_performance) network=$(check_network_health) security=$(check_security_health) # Assess individual components cpu_usage=$(echo "$resources" | jq -r '.cpu_usage' 2>/dev/null || echo "0") mem_usage=$(echo "$resources" | jq -r '.memory_usage' 2>/dev/null || echo "0") disk_usage=$(echo "$resources" | jq -r '.disk_usage' 2>/dev/null || echo "0") # Simple health rules cpu_ok=$(echo "$cpu_usage < 80" | bc -l) mem_ok=$(echo "$mem_usage < 85" | bc -l) disk_ok=$(echo "$disk_usage < 90" | bc -l) services_ok=1 if echo "$services" | grep -q "unhealthy"; then services_ok=0 fi packages_ok=1 broken_count=$(echo "$packages" | jq -r '.broken_packages' 2>/dev/null || echo "0") if [[ "$broken_count" -gt 0 ]]; then packages_ok=0 fi # Overall assessment if [[ "$cpu_ok" == "1" && "$mem_ok" == "1" && "$disk_ok" == "1" && "$services_ok" == "1" && "$packages_ok" == "1" ]]; then overall_status="healthy" elif [[ "$services_ok" == "0" || "$packages_ok" == "0" ]]; then overall_status="unhealthy" else overall_status="degraded" fi # Create comprehensive health report cat > "$HEALTH_LOG" << EOF { "timestamp": "$(date -Iseconds)", "system": "$(sw_vers -productVersion)", "hostname": "$(hostname)", "overall_status": "$overall_status", "components": { "resources": $resources, "services": $services, "packages": $packages, "shell": $shell, "network": $network, "security": $security }, "recommendations": [] } EOF # Add recommendations based on findings if [[ "$cpu_ok" == "0" ]]; then echo "$(jq '.recommendations += ["High CPU usage detected - consider closing applications"]' "$HEALTH_LOG")" > "$HEALTH_LOG" fi if [[ "$mem_ok" == "0" ]]; then echo "$(jq '.recommendations += ["High memory usage detected - consider restarting applications"]' "$HEALTH_LOG")" > "$HEALTH_LOG" fi if [[ "$disk_ok" == "0" ]]; then echo "$(jq '.recommendations += ["Low disk space - consider cleanup with: brew cleanup"]' "$HEALTH_LOG")" > "$HEALTH_LOG" fi echo "$overall_status" } # Function to display health dashboard display_dashboard() { local overall_status echo -e "${BLUE}System Health Dashboard${NC}" echo "========================" overall_status=$(assess_overall_health) case "$overall_status" in "healthy") echo -e "${GREEN}✅ System Status: HEALTHY${NC}" ;; "degraded") echo -e "${YELLOW}⚠️ System Status: DEGRADED${NC}" ;; "unhealthy") echo -e "${RED}❌ System Status: UNHEALTHY${NC}" ;; esac echo "" echo "Component Details:" # Resources local resources=$(jq -r '.components.resources | "CPU: \(.cpu_usage)% | Memory: \(.memory_usage)% | Disk: \(.disk_usage)%"' "$HEALTH_LOG" 2>/dev/null || echo "Resources: N/A") echo " Resources: $resources" # Services local unhealthy_services=$(jq -r '.components.services | to_entries[] | select(.value == "unhealthy") | .key' "$HEALTH_LOG" 2>/dev/null) if [[ -n "$unhealthy_services" ]]; then echo -e " ${RED}Unhealthy Services: $unhealthy_services${NC}" else echo -e " ${GREEN}Services: All healthy${NC}" fi # Packages local outdated=$(jq -r '.components.packages.outdated_packages' "$HEALTH_LOG" 2>/dev/null || echo "0") local broken=$(jq -r '.components.packages.broken_packages' "$HEALTH_LOG" 2>/dev/null || echo "0") echo " Packages: $outdated outdated, $broken broken" # Network local network_status=$(jq -r '.components.network | "Internet: \(.internet) | DNS: \(.dns) | GitHub: \(.github)"' "$HEALTH_LOG" 2>/dev/null || echo "Network: N/A") echo " Network: $network_status" echo "" echo "Recommendations:" local recommendations=$(jq -r '.recommendations[]' "$HEALTH_LOG" 2>/dev/null) if [[ -n "$recommendations" ]]; then echo "$recommendations" | sed 's/^/ - /' else echo " - No immediate actions required" fi echo "" echo "Full report: $HEALTH_LOG" # Clean old logs (keep last 20) find "$HEALTH_LOG_DIR" -name "health-*.json" | sort -r | tail -n +21 | xargs rm -f 2>/dev/null || true } # Function to run continuous monitoring run_monitoring() { echo -e "${BLUE}Starting continuous health monitoring...${NC}" echo "Press Ctrl+C to stop" while true; do clear display_dashboard echo "" echo "$(date) - Next check in 30 seconds..." sleep 30 done } # Main execution case "${1:-dashboard}" in "dashboard"|"-d"|"--dashboard") display_dashboard ;; "monitor"|"-m"|"--monitor") run_monitoring ;; "json"|"-j"|"--json") assess_overall_health >/dev/null cat "$HEALTH_LOG" ;; "quiet"|"-q"|"--quiet") assess_overall_health ;; *) echo "Usage: $0 [dashboard|monitor|json|quiet]" echo " dashboard (default): Show health dashboard" echo " monitor: Continuous monitoring mode" echo " json: Output raw JSON report" echo " quiet: Just return overall status" exit 1 ;; esac