Initial commit: Complete DevServer monitoring framework
- FastAPI application with OpenTelemetry instrumentation - Redis caching and Celery task processing - Docker Compose setup with monitoring stack - Grafana dashboards and Prometheus metrics - Jaeger distributed tracing - Comprehensive documentation in docs/ folder - Monitoring configuration and troubleshooting guides
This commit is contained in:
Hardik Sankhla
256
system_monitor.py
Normal file
256
system_monitor.py
Normal file
@@ -0,0 +1,256 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
DevServer System Monitor
|
||||
Comprehensive health check and status monitoring for all services
|
||||
"""
|
||||
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Tuple
|
||||
import sys
|
||||
|
||||
# Service endpoints
|
||||
SERVICES = {
|
||||
"FastAPI Backend": "http://localhost:8000/health",
|
||||
"Prometheus": "http://localhost:9090/-/healthy",
|
||||
"Grafana": "http://localhost:3000/api/health",
|
||||
"Jaeger": "http://localhost:16686",
|
||||
"Loki": "http://localhost:3100/ready",
|
||||
"Redis": "http://localhost:6379",
|
||||
"RabbitMQ": "http://localhost:15672",
|
||||
"cAdvisor": "http://localhost:8080/healthz",
|
||||
"Node Exporter": "http://localhost:9100/metrics",
|
||||
}
|
||||
|
||||
COLORS = {
|
||||
"green": "\033[92m",
|
||||
"red": "\033[91m",
|
||||
"yellow": "\033[93m",
|
||||
"blue": "\033[94m",
|
||||
"reset": "\033[0m",
|
||||
"bold": "\033[1m",
|
||||
}
|
||||
|
||||
|
||||
def colored(text: str, color: str) -> str:
|
||||
"""Return colored text for terminal output"""
|
||||
return f"{COLORS.get(color, '')}{text}{COLORS['reset']}"
|
||||
|
||||
|
||||
def check_service(name: str, url: str, timeout: int = 5) -> Tuple[bool, str, float]:
|
||||
"""Check if a service is healthy"""
|
||||
start = time.time()
|
||||
try:
|
||||
response = requests.get(url, timeout=timeout)
|
||||
duration = (time.time() - start) * 1000 # Convert to ms
|
||||
|
||||
if response.status_code in [200, 204]:
|
||||
return True, "OK", duration
|
||||
else:
|
||||
return False, f"HTTP {response.status_code}", duration
|
||||
except requests.exceptions.ConnectionError:
|
||||
duration = (time.time() - start) * 1000
|
||||
return False, "Connection Refused", duration
|
||||
except requests.exceptions.Timeout:
|
||||
return False, "Timeout", timeout * 1000
|
||||
except Exception as e:
|
||||
duration = (time.time() - start) * 1000
|
||||
return False, str(e)[:50], duration
|
||||
|
||||
|
||||
def get_prometheus_metrics() -> Dict:
|
||||
"""Fetch key metrics from Prometheus"""
|
||||
try:
|
||||
queries = {
|
||||
"up_services": 'count(up == 1)',
|
||||
"total_services": 'count(up)',
|
||||
"api_requests_rate": 'rate(http_requests_total{job="fastapi-app"}[5m])',
|
||||
"error_rate": 'rate(http_requests_total{job="fastapi-app",status=~"5.."}[5m])',
|
||||
}
|
||||
|
||||
metrics = {}
|
||||
for name, query in queries.items():
|
||||
response = requests.get(
|
||||
f"http://localhost:9090/api/v1/query",
|
||||
params={"query": query},
|
||||
timeout=5
|
||||
)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get("data", {}).get("result"):
|
||||
value = data["data"]["result"][0]["value"][1]
|
||||
metrics[name] = float(value)
|
||||
|
||||
return metrics
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
def get_redis_stats() -> Dict:
|
||||
"""Fetch Redis statistics"""
|
||||
try:
|
||||
response = requests.get("http://localhost:8000/api/cache/stats", timeout=5)
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
return {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def print_header():
|
||||
"""Print system status header"""
|
||||
print("\n" + "=" * 80)
|
||||
print(colored("DevServer System Status Monitor", "bold"))
|
||||
print(colored(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", "blue"))
|
||||
print("=" * 80 + "\n")
|
||||
|
||||
|
||||
def print_service_status(services_status: Dict):
|
||||
"""Print service health status table"""
|
||||
print(colored("Service Health Status:", "bold"))
|
||||
print("-" * 80)
|
||||
print(f"{'Service':<25} {'Status':<15} {'Details':<25} {'Response Time (ms)':<15}")
|
||||
print("-" * 80)
|
||||
|
||||
healthy_count = 0
|
||||
total_count = len(services_status)
|
||||
|
||||
for service, (is_healthy, details, response_time) in services_status.items():
|
||||
if is_healthy:
|
||||
status = colored("✓ HEALTHY", "green")
|
||||
healthy_count += 1
|
||||
else:
|
||||
status = colored("✗ UNHEALTHY", "red")
|
||||
|
||||
print(f"{service:<25} {status:<24} {details:<25} {response_time:>10.2f}")
|
||||
|
||||
print("-" * 80)
|
||||
health_percentage = (healthy_count / total_count) * 100
|
||||
color = "green" if health_percentage == 100 else "yellow" if health_percentage >= 80 else "red"
|
||||
print(colored(f"Overall Health: {healthy_count}/{total_count} services ({health_percentage:.1f}%)", color))
|
||||
print()
|
||||
|
||||
|
||||
def print_prometheus_metrics(metrics: Dict):
|
||||
"""Print Prometheus metrics summary"""
|
||||
if "error" in metrics:
|
||||
print(colored(f"Failed to fetch Prometheus metrics: {metrics['error']}", "red"))
|
||||
return
|
||||
|
||||
print(colored("Prometheus Metrics Summary:", "bold"))
|
||||
print("-" * 80)
|
||||
|
||||
if "up_services" in metrics and "total_services" in metrics:
|
||||
print(f"Active Targets: {int(metrics['up_services'])}/{int(metrics['total_services'])}")
|
||||
|
||||
if "api_requests_rate" in metrics:
|
||||
print(f"API Request Rate: {metrics['api_requests_rate']:.2f} req/s")
|
||||
|
||||
if "error_rate" in metrics:
|
||||
print(f"API Error Rate: {metrics['error_rate']:.2f} errors/s")
|
||||
|
||||
print()
|
||||
|
||||
|
||||
def print_redis_stats(stats: Dict):
|
||||
"""Print Redis cache statistics"""
|
||||
if not stats:
|
||||
print(colored("Redis statistics unavailable", "yellow"))
|
||||
return
|
||||
|
||||
print(colored("Redis Cache Statistics:", "bold"))
|
||||
print("-" * 80)
|
||||
|
||||
if "total_keys" in stats:
|
||||
print(f"Total Keys: {stats['total_keys']}")
|
||||
|
||||
if "memory_used" in stats:
|
||||
print(f"Memory Used: {stats['memory_used']}")
|
||||
|
||||
if "connected_clients" in stats:
|
||||
print(f"Connected Clients: {stats['connected_clients']}")
|
||||
|
||||
if "hits" in stats and "misses" in stats:
|
||||
total = stats["hits"] + stats["misses"]
|
||||
if total > 0:
|
||||
hit_rate = (stats["hits"] / total) * 100
|
||||
print(f"Cache Hit Rate: {hit_rate:.2f}% ({stats['hits']} hits / {stats['misses']} misses)")
|
||||
|
||||
print()
|
||||
|
||||
|
||||
def print_recommendations(services_status: Dict):
|
||||
"""Print recommendations based on service status"""
|
||||
unhealthy = [name for name, (healthy, _, _) in services_status.items() if not healthy]
|
||||
|
||||
if unhealthy:
|
||||
print(colored("Recommendations:", "bold"))
|
||||
print("-" * 80)
|
||||
for service in unhealthy:
|
||||
print(colored(f"✗ {service} is down", "red"))
|
||||
if "Redis" in service:
|
||||
print(" → Check Redis container: docker-compose logs redis")
|
||||
elif "RabbitMQ" in service:
|
||||
print(" → Check RabbitMQ container: docker-compose logs rabbitmq")
|
||||
elif "FastAPI" in service:
|
||||
print(" → Check FastAPI logs: docker-compose logs devserver")
|
||||
elif "Prometheus" in service:
|
||||
print(" → Check Prometheus config: docker-compose logs prometheus")
|
||||
elif "Grafana" in service:
|
||||
print(" → Check Grafana logs: docker-compose logs grafana")
|
||||
else:
|
||||
print(f" → Check container logs: docker-compose logs {service.lower().replace(' ', '-')}")
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
"""Main monitoring function"""
|
||||
print_header()
|
||||
|
||||
# Check all services
|
||||
print("Checking all services...\n")
|
||||
services_status = {}
|
||||
for service, url in SERVICES.items():
|
||||
services_status[service] = check_service(service, url)
|
||||
|
||||
# Print results
|
||||
print_service_status(services_status)
|
||||
|
||||
# Get and print Prometheus metrics
|
||||
metrics = get_prometheus_metrics()
|
||||
print_prometheus_metrics(metrics)
|
||||
|
||||
# Get and print Redis stats
|
||||
redis_stats = get_redis_stats()
|
||||
print_redis_stats(redis_stats)
|
||||
|
||||
# Print recommendations
|
||||
print_recommendations(services_status)
|
||||
|
||||
# Print monitoring URLs
|
||||
print(colored("Monitoring Dashboards:", "bold"))
|
||||
print("-" * 80)
|
||||
print(f"Grafana Dashboard: {colored('http://localhost:3000', 'blue')} (admin/admin123)")
|
||||
print(f"Prometheus: {colored('http://localhost:9090', 'blue')}")
|
||||
print(f"Jaeger Tracing: {colored('http://localhost:16686', 'blue')}")
|
||||
print(f"RabbitMQ Management: {colored('http://localhost:15672', 'blue')} (guest/guest)")
|
||||
print(f"cAdvisor: {colored('http://localhost:8080', 'blue')}")
|
||||
print("=" * 80 + "\n")
|
||||
|
||||
# Return exit code based on health
|
||||
unhealthy_count = len([s for s, (h, _, _) in services_status.items() if not h])
|
||||
return 0 if unhealthy_count == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
exit_code = main()
|
||||
sys.exit(exit_code)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\nMonitoring interrupted by user")
|
||||
sys.exit(130)
|
||||
except Exception as e:
|
||||
print(colored(f"\nError: {e}", "red"))
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user