2304c3ed12
- FastAPI application with OpenTelemetry instrumentation - Redis caching and Celery task processing - Docker Compose setup with monitoring stack - Grafana dashboards and Prometheus metrics - Jaeger distributed tracing - Comprehensive documentation in docs/ folder - Monitoring configuration and troubleshooting guides
257 lines
8.5 KiB
Python
257 lines
8.5 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
DevServer System Monitor
|
|
Comprehensive health check and status monitoring for all services
|
|
"""
|
|
|
|
import requests
|
|
import json
|
|
import time
|
|
from datetime import datetime
|
|
from typing import Dict, List, Tuple
|
|
import sys
|
|
|
|
# Service endpoints
|
|
SERVICES = {
|
|
"FastAPI Backend": "http://localhost:8000/health",
|
|
"Prometheus": "http://localhost:9090/-/healthy",
|
|
"Grafana": "http://localhost:3000/api/health",
|
|
"Jaeger": "http://localhost:16686",
|
|
"Loki": "http://localhost:3100/ready",
|
|
"Redis": "http://localhost:6379",
|
|
"RabbitMQ": "http://localhost:15672",
|
|
"cAdvisor": "http://localhost:8080/healthz",
|
|
"Node Exporter": "http://localhost:9100/metrics",
|
|
}
|
|
|
|
COLORS = {
|
|
"green": "\033[92m",
|
|
"red": "\033[91m",
|
|
"yellow": "\033[93m",
|
|
"blue": "\033[94m",
|
|
"reset": "\033[0m",
|
|
"bold": "\033[1m",
|
|
}
|
|
|
|
|
|
def colored(text: str, color: str) -> str:
|
|
"""Return colored text for terminal output"""
|
|
return f"{COLORS.get(color, '')}{text}{COLORS['reset']}"
|
|
|
|
|
|
def check_service(name: str, url: str, timeout: int = 5) -> Tuple[bool, str, float]:
|
|
"""Check if a service is healthy"""
|
|
start = time.time()
|
|
try:
|
|
response = requests.get(url, timeout=timeout)
|
|
duration = (time.time() - start) * 1000 # Convert to ms
|
|
|
|
if response.status_code in [200, 204]:
|
|
return True, "OK", duration
|
|
else:
|
|
return False, f"HTTP {response.status_code}", duration
|
|
except requests.exceptions.ConnectionError:
|
|
duration = (time.time() - start) * 1000
|
|
return False, "Connection Refused", duration
|
|
except requests.exceptions.Timeout:
|
|
return False, "Timeout", timeout * 1000
|
|
except Exception as e:
|
|
duration = (time.time() - start) * 1000
|
|
return False, str(e)[:50], duration
|
|
|
|
|
|
def get_prometheus_metrics() -> Dict:
|
|
"""Fetch key metrics from Prometheus"""
|
|
try:
|
|
queries = {
|
|
"up_services": 'count(up == 1)',
|
|
"total_services": 'count(up)',
|
|
"api_requests_rate": 'rate(http_requests_total{job="fastapi-app"}[5m])',
|
|
"error_rate": 'rate(http_requests_total{job="fastapi-app",status=~"5.."}[5m])',
|
|
}
|
|
|
|
metrics = {}
|
|
for name, query in queries.items():
|
|
response = requests.get(
|
|
f"http://localhost:9090/api/v1/query",
|
|
params={"query": query},
|
|
timeout=5
|
|
)
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
if data.get("data", {}).get("result"):
|
|
value = data["data"]["result"][0]["value"][1]
|
|
metrics[name] = float(value)
|
|
|
|
return metrics
|
|
except Exception as e:
|
|
return {"error": str(e)}
|
|
|
|
|
|
def get_redis_stats() -> Dict:
|
|
"""Fetch Redis statistics"""
|
|
try:
|
|
response = requests.get("http://localhost:8000/api/cache/stats", timeout=5)
|
|
if response.status_code == 200:
|
|
return response.json()
|
|
return {}
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def print_header():
|
|
"""Print system status header"""
|
|
print("\n" + "=" * 80)
|
|
print(colored("DevServer System Status Monitor", "bold"))
|
|
print(colored(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", "blue"))
|
|
print("=" * 80 + "\n")
|
|
|
|
|
|
def print_service_status(services_status: Dict):
|
|
"""Print service health status table"""
|
|
print(colored("Service Health Status:", "bold"))
|
|
print("-" * 80)
|
|
print(f"{'Service':<25} {'Status':<15} {'Details':<25} {'Response Time (ms)':<15}")
|
|
print("-" * 80)
|
|
|
|
healthy_count = 0
|
|
total_count = len(services_status)
|
|
|
|
for service, (is_healthy, details, response_time) in services_status.items():
|
|
if is_healthy:
|
|
status = colored("✓ HEALTHY", "green")
|
|
healthy_count += 1
|
|
else:
|
|
status = colored("✗ UNHEALTHY", "red")
|
|
|
|
print(f"{service:<25} {status:<24} {details:<25} {response_time:>10.2f}")
|
|
|
|
print("-" * 80)
|
|
health_percentage = (healthy_count / total_count) * 100
|
|
color = "green" if health_percentage == 100 else "yellow" if health_percentage >= 80 else "red"
|
|
print(colored(f"Overall Health: {healthy_count}/{total_count} services ({health_percentage:.1f}%)", color))
|
|
print()
|
|
|
|
|
|
def print_prometheus_metrics(metrics: Dict):
|
|
"""Print Prometheus metrics summary"""
|
|
if "error" in metrics:
|
|
print(colored(f"Failed to fetch Prometheus metrics: {metrics['error']}", "red"))
|
|
return
|
|
|
|
print(colored("Prometheus Metrics Summary:", "bold"))
|
|
print("-" * 80)
|
|
|
|
if "up_services" in metrics and "total_services" in metrics:
|
|
print(f"Active Targets: {int(metrics['up_services'])}/{int(metrics['total_services'])}")
|
|
|
|
if "api_requests_rate" in metrics:
|
|
print(f"API Request Rate: {metrics['api_requests_rate']:.2f} req/s")
|
|
|
|
if "error_rate" in metrics:
|
|
print(f"API Error Rate: {metrics['error_rate']:.2f} errors/s")
|
|
|
|
print()
|
|
|
|
|
|
def print_redis_stats(stats: Dict):
|
|
"""Print Redis cache statistics"""
|
|
if not stats:
|
|
print(colored("Redis statistics unavailable", "yellow"))
|
|
return
|
|
|
|
print(colored("Redis Cache Statistics:", "bold"))
|
|
print("-" * 80)
|
|
|
|
if "total_keys" in stats:
|
|
print(f"Total Keys: {stats['total_keys']}")
|
|
|
|
if "memory_used" in stats:
|
|
print(f"Memory Used: {stats['memory_used']}")
|
|
|
|
if "connected_clients" in stats:
|
|
print(f"Connected Clients: {stats['connected_clients']}")
|
|
|
|
if "hits" in stats and "misses" in stats:
|
|
total = stats["hits"] + stats["misses"]
|
|
if total > 0:
|
|
hit_rate = (stats["hits"] / total) * 100
|
|
print(f"Cache Hit Rate: {hit_rate:.2f}% ({stats['hits']} hits / {stats['misses']} misses)")
|
|
|
|
print()
|
|
|
|
|
|
def print_recommendations(services_status: Dict):
|
|
"""Print recommendations based on service status"""
|
|
unhealthy = [name for name, (healthy, _, _) in services_status.items() if not healthy]
|
|
|
|
if unhealthy:
|
|
print(colored("Recommendations:", "bold"))
|
|
print("-" * 80)
|
|
for service in unhealthy:
|
|
print(colored(f"✗ {service} is down", "red"))
|
|
if "Redis" in service:
|
|
print(" → Check Redis container: docker-compose logs redis")
|
|
elif "RabbitMQ" in service:
|
|
print(" → Check RabbitMQ container: docker-compose logs rabbitmq")
|
|
elif "FastAPI" in service:
|
|
print(" → Check FastAPI logs: docker-compose logs devserver")
|
|
elif "Prometheus" in service:
|
|
print(" → Check Prometheus config: docker-compose logs prometheus")
|
|
elif "Grafana" in service:
|
|
print(" → Check Grafana logs: docker-compose logs grafana")
|
|
else:
|
|
print(f" → Check container logs: docker-compose logs {service.lower().replace(' ', '-')}")
|
|
print()
|
|
|
|
|
|
def main():
|
|
"""Main monitoring function"""
|
|
print_header()
|
|
|
|
# Check all services
|
|
print("Checking all services...\n")
|
|
services_status = {}
|
|
for service, url in SERVICES.items():
|
|
services_status[service] = check_service(service, url)
|
|
|
|
# Print results
|
|
print_service_status(services_status)
|
|
|
|
# Get and print Prometheus metrics
|
|
metrics = get_prometheus_metrics()
|
|
print_prometheus_metrics(metrics)
|
|
|
|
# Get and print Redis stats
|
|
redis_stats = get_redis_stats()
|
|
print_redis_stats(redis_stats)
|
|
|
|
# Print recommendations
|
|
print_recommendations(services_status)
|
|
|
|
# Print monitoring URLs
|
|
print(colored("Monitoring Dashboards:", "bold"))
|
|
print("-" * 80)
|
|
print(f"Grafana Dashboard: {colored('http://localhost:3000', 'blue')} (admin/admin123)")
|
|
print(f"Prometheus: {colored('http://localhost:9090', 'blue')}")
|
|
print(f"Jaeger Tracing: {colored('http://localhost:16686', 'blue')}")
|
|
print(f"RabbitMQ Management: {colored('http://localhost:15672', 'blue')} (guest/guest)")
|
|
print(f"cAdvisor: {colored('http://localhost:8080', 'blue')}")
|
|
print("=" * 80 + "\n")
|
|
|
|
# Return exit code based on health
|
|
unhealthy_count = len([s for s, (h, _, _) in services_status.items() if not h])
|
|
return 0 if unhealthy_count == 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
exit_code = main()
|
|
sys.exit(exit_code)
|
|
except KeyboardInterrupt:
|
|
print("\n\nMonitoring interrupted by user")
|
|
sys.exit(130)
|
|
except Exception as e:
|
|
print(colored(f"\nError: {e}", "red"))
|
|
sys.exit(1)
|