Initial commit: Complete DevServer monitoring framework
- FastAPI application with OpenTelemetry instrumentation - Redis caching and Celery task processing - Docker Compose setup with monitoring stack - Grafana dashboards and Prometheus metrics - Jaeger distributed tracing - Comprehensive documentation in docs/ folder - Monitoring configuration and troubleshooting guides
This commit is contained in:
Hardik Sankhla
309
monitoring/grafana/dashboards/devserver-dashboard.json
Normal file
309
monitoring/grafana/dashboards/devserver-dashboard.json
Normal file
@@ -0,0 +1,309 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "devserver-main",
|
||||
"title": "DevServer Monitoring Dashboard",
|
||||
"tags": ["devserver", "monitoring", "fastapi"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 27,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API Request Rate",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(http_requests_total[5m])",
|
||||
"legendFormat": "{{method}} {{endpoint}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API Response Time (p95 & p50)",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))",
|
||||
"legendFormat": "95th percentile",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, rate(http_request_duration_seconds_bucket[5m]))",
|
||||
"legendFormat": "50th percentile",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "HTTP Status Codes Distribution",
|
||||
"type": "piechart",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(http_requests_total[5m])) by (status)",
|
||||
"legendFormat": "{{status}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 8}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Active Requests",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "http_requests_in_progress",
|
||||
"legendFormat": "In Progress",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 4, "w": 8, "x": 8, "y": 8},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"value": null, "color": "green"},
|
||||
{"value": 50, "color": "yellow"},
|
||||
{"value": 100, "color": "red"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Total Requests (5m)",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(http_requests_total[5m]))",
|
||||
"legendFormat": "Total",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 4, "w": 8, "x": 16, "y": 8},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Error Rate (5xx)",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) / sum(rate(http_requests_total[5m])) * 100",
|
||||
"legendFormat": "Error Rate",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 4, "w": 8, "x": 8, "y": 12},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"value": null, "color": "green"},
|
||||
{"value": 1, "color": "yellow"},
|
||||
{"value": 5, "color": "red"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Average Response Time",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(http_request_duration_seconds_sum[5m]) / rate(http_request_duration_seconds_count[5m])",
|
||||
"legendFormat": "Avg Response Time",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 4, "w": 8, "x": 16, "y": 12},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"decimals": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Container CPU Usage",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(container_cpu_usage_seconds_total{name=~\"devserver.*\"}[5m]) * 100",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 16},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Container Memory Usage",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "container_memory_usage_bytes{name=~\"devserver.*\"} / 1024 / 1024",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 16},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "mbytes"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "RabbitMQ Queue Size",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rabbitmq_queue_messages",
|
||||
"legendFormat": "{{queue}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 6, "w": 12, "x": 0, "y": 24},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "RabbitMQ Message Rate",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(rabbitmq_queue_messages_published_total[5m])",
|
||||
"legendFormat": "Published",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "rate(rabbitmq_queue_messages_delivered_total[5m])",
|
||||
"legendFormat": "Delivered",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 6, "w": 12, "x": 12, "y": 24},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "msgs/s"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "OpenTelemetry Collector Metrics",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(otelcol_receiver_accepted_spans[5m])",
|
||||
"legendFormat": "Spans Received",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "rate(otelcol_exporter_sent_spans[5m])",
|
||||
"legendFormat": "Spans Exported",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 6, "w": 12, "x": 0, "y": 30},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "Process Memory Usage",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "process_resident_memory_bytes / 1024 / 1024",
|
||||
"legendFormat": "{{job}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 6, "w": 12, "x": 12, "y": 30},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "mbytes"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"enable": true,
|
||||
"expr": "ALERTS{alertstate=\"firing\"}",
|
||||
"iconColor": "red",
|
||||
"name": "Alerts",
|
||||
"step": "60s",
|
||||
"tagKeys": "alertname",
|
||||
"textFormat": "{{alertname}}",
|
||||
"titleFormat": "Alert"
|
||||
}
|
||||
]
|
||||
},
|
||||
"links": [],
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"style": "dark"
|
||||
}
|
||||
10
monitoring/grafana/provisioning/dashboards/dashboards.yml
Normal file
10
monitoring/grafana/provisioning/dashboards/dashboards.yml
Normal file
@@ -0,0 +1,10 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: 'DevServer Dashboards'
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 10
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
27
monitoring/grafana/provisioning/datasources/datasources.yaml
Normal file
27
monitoring/grafana/provisioning/datasources/datasources.yaml
Normal file
@@ -0,0 +1,27 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: true
|
||||
jsonData:
|
||||
timeInterval: "5s"
|
||||
|
||||
- name: Loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
isDefault: false
|
||||
editable: true
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
|
||||
- name: Jaeger
|
||||
type: jaeger
|
||||
access: proxy
|
||||
url: http://jaeger:16686
|
||||
isDefault: false
|
||||
editable: true
|
||||
Reference in New Issue
Block a user