#!/usr/bin/env python3 """ DevServer System Monitor Comprehensive health check and status monitoring for all services """ import requests import json import time from datetime import datetime from typing import Dict, List, Tuple import sys # Service endpoints SERVICES = { "FastAPI Backend": "http://localhost:8000/health", "Prometheus": "http://localhost:9090/-/healthy", "Grafana": "http://localhost:3000/api/health", "Jaeger": "http://localhost:16686", "Loki": "http://localhost:3100/ready", "Redis": "http://localhost:6379", "RabbitMQ": "http://localhost:15672", "cAdvisor": "http://localhost:8080/healthz", "Node Exporter": "http://localhost:9100/metrics", } COLORS = { "green": "\033[92m", "red": "\033[91m", "yellow": "\033[93m", "blue": "\033[94m", "reset": "\033[0m", "bold": "\033[1m", } def colored(text: str, color: str) -> str: """Return colored text for terminal output""" return f"{COLORS.get(color, '')}{text}{COLORS['reset']}" def check_service(name: str, url: str, timeout: int = 5) -> Tuple[bool, str, float]: """Check if a service is healthy""" start = time.time() try: response = requests.get(url, timeout=timeout) duration = (time.time() - start) * 1000 # Convert to ms if response.status_code in [200, 204]: return True, "OK", duration else: return False, f"HTTP {response.status_code}", duration except requests.exceptions.ConnectionError: duration = (time.time() - start) * 1000 return False, "Connection Refused", duration except requests.exceptions.Timeout: return False, "Timeout", timeout * 1000 except Exception as e: duration = (time.time() - start) * 1000 return False, str(e)[:50], duration def get_prometheus_metrics() -> Dict: """Fetch key metrics from Prometheus""" try: queries = { "up_services": 'count(up == 1)', "total_services": 'count(up)', "api_requests_rate": 'rate(http_requests_total{job="fastapi-app"}[5m])', "error_rate": 'rate(http_requests_total{job="fastapi-app",status=~"5.."}[5m])', } metrics = {} for name, query in queries.items(): response = requests.get( f"http://localhost:9090/api/v1/query", params={"query": query}, timeout=5 ) if response.status_code == 200: data = response.json() if data.get("data", {}).get("result"): value = data["data"]["result"][0]["value"][1] metrics[name] = float(value) return metrics except Exception as e: return {"error": str(e)} def get_redis_stats() -> Dict: """Fetch Redis statistics""" try: response = requests.get("http://localhost:8000/api/cache/stats", timeout=5) if response.status_code == 200: return response.json() return {} except Exception: return {} def print_header(): """Print system status header""" print("\n" + "=" * 80) print(colored("DevServer System Status Monitor", "bold")) print(colored(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", "blue")) print("=" * 80 + "\n") def print_service_status(services_status: Dict): """Print service health status table""" print(colored("Service Health Status:", "bold")) print("-" * 80) print(f"{'Service':<25} {'Status':<15} {'Details':<25} {'Response Time (ms)':<15}") print("-" * 80) healthy_count = 0 total_count = len(services_status) for service, (is_healthy, details, response_time) in services_status.items(): if is_healthy: status = colored("✓ HEALTHY", "green") healthy_count += 1 else: status = colored("✗ UNHEALTHY", "red") print(f"{service:<25} {status:<24} {details:<25} {response_time:>10.2f}") print("-" * 80) health_percentage = (healthy_count / total_count) * 100 color = "green" if health_percentage == 100 else "yellow" if health_percentage >= 80 else "red" print(colored(f"Overall Health: {healthy_count}/{total_count} services ({health_percentage:.1f}%)", color)) print() def print_prometheus_metrics(metrics: Dict): """Print Prometheus metrics summary""" if "error" in metrics: print(colored(f"Failed to fetch Prometheus metrics: {metrics['error']}", "red")) return print(colored("Prometheus Metrics Summary:", "bold")) print("-" * 80) if "up_services" in metrics and "total_services" in metrics: print(f"Active Targets: {int(metrics['up_services'])}/{int(metrics['total_services'])}") if "api_requests_rate" in metrics: print(f"API Request Rate: {metrics['api_requests_rate']:.2f} req/s") if "error_rate" in metrics: print(f"API Error Rate: {metrics['error_rate']:.2f} errors/s") print() def print_redis_stats(stats: Dict): """Print Redis cache statistics""" if not stats: print(colored("Redis statistics unavailable", "yellow")) return print(colored("Redis Cache Statistics:", "bold")) print("-" * 80) if "total_keys" in stats: print(f"Total Keys: {stats['total_keys']}") if "memory_used" in stats: print(f"Memory Used: {stats['memory_used']}") if "connected_clients" in stats: print(f"Connected Clients: {stats['connected_clients']}") if "hits" in stats and "misses" in stats: total = stats["hits"] + stats["misses"] if total > 0: hit_rate = (stats["hits"] / total) * 100 print(f"Cache Hit Rate: {hit_rate:.2f}% ({stats['hits']} hits / {stats['misses']} misses)") print() def print_recommendations(services_status: Dict): """Print recommendations based on service status""" unhealthy = [name for name, (healthy, _, _) in services_status.items() if not healthy] if unhealthy: print(colored("Recommendations:", "bold")) print("-" * 80) for service in unhealthy: print(colored(f"✗ {service} is down", "red")) if "Redis" in service: print(" → Check Redis container: docker-compose logs redis") elif "RabbitMQ" in service: print(" → Check RabbitMQ container: docker-compose logs rabbitmq") elif "FastAPI" in service: print(" → Check FastAPI logs: docker-compose logs devserver") elif "Prometheus" in service: print(" → Check Prometheus config: docker-compose logs prometheus") elif "Grafana" in service: print(" → Check Grafana logs: docker-compose logs grafana") else: print(f" → Check container logs: docker-compose logs {service.lower().replace(' ', '-')}") print() def main(): """Main monitoring function""" print_header() # Check all services print("Checking all services...\n") services_status = {} for service, url in SERVICES.items(): services_status[service] = check_service(service, url) # Print results print_service_status(services_status) # Get and print Prometheus metrics metrics = get_prometheus_metrics() print_prometheus_metrics(metrics) # Get and print Redis stats redis_stats = get_redis_stats() print_redis_stats(redis_stats) # Print recommendations print_recommendations(services_status) # Print monitoring URLs print(colored("Monitoring Dashboards:", "bold")) print("-" * 80) print(f"Grafana Dashboard: {colored('http://localhost:3000', 'blue')} (admin/admin123)") print(f"Prometheus: {colored('http://localhost:9090', 'blue')}") print(f"Jaeger Tracing: {colored('http://localhost:16686', 'blue')}") print(f"RabbitMQ Management: {colored('http://localhost:15672', 'blue')} (guest/guest)") print(f"cAdvisor: {colored('http://localhost:8080', 'blue')}") print("=" * 80 + "\n") # Return exit code based on health unhealthy_count = len([s for s, (h, _, _) in services_status.items() if not h]) return 0 if unhealthy_count == 0 else 1 if __name__ == "__main__": try: exit_code = main() sys.exit(exit_code) except KeyboardInterrupt: print("\n\nMonitoring interrupted by user") sys.exit(130) except Exception as e: print(colored(f"\nError: {e}", "red")) sys.exit(1)