Initial commit: Complete DevServer monitoring framework
- FastAPI application with OpenTelemetry instrumentation - Redis caching and Celery task processing - Docker Compose setup with monitoring stack - Grafana dashboards and Prometheus metrics - Jaeger distributed tracing - Comprehensive documentation in docs/ folder - Monitoring configuration and troubleshooting guides
This commit is contained in:
Hardik Sankhla
309
monitoring/grafana/dashboards/devserver-dashboard.json
Normal file
309
monitoring/grafana/dashboards/devserver-dashboard.json
Normal file
@@ -0,0 +1,309 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "devserver-main",
|
||||
"title": "DevServer Monitoring Dashboard",
|
||||
"tags": ["devserver", "monitoring", "fastapi"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 27,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API Request Rate",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(http_requests_total[5m])",
|
||||
"legendFormat": "{{method}} {{endpoint}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API Response Time (p95 & p50)",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))",
|
||||
"legendFormat": "95th percentile",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, rate(http_request_duration_seconds_bucket[5m]))",
|
||||
"legendFormat": "50th percentile",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "HTTP Status Codes Distribution",
|
||||
"type": "piechart",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(http_requests_total[5m])) by (status)",
|
||||
"legendFormat": "{{status}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 8}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Active Requests",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "http_requests_in_progress",
|
||||
"legendFormat": "In Progress",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 4, "w": 8, "x": 8, "y": 8},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"value": null, "color": "green"},
|
||||
{"value": 50, "color": "yellow"},
|
||||
{"value": 100, "color": "red"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Total Requests (5m)",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(http_requests_total[5m]))",
|
||||
"legendFormat": "Total",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 4, "w": 8, "x": 16, "y": 8},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Error Rate (5xx)",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) / sum(rate(http_requests_total[5m])) * 100",
|
||||
"legendFormat": "Error Rate",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 4, "w": 8, "x": 8, "y": 12},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"value": null, "color": "green"},
|
||||
{"value": 1, "color": "yellow"},
|
||||
{"value": 5, "color": "red"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Average Response Time",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(http_request_duration_seconds_sum[5m]) / rate(http_request_duration_seconds_count[5m])",
|
||||
"legendFormat": "Avg Response Time",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 4, "w": 8, "x": 16, "y": 12},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"decimals": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Container CPU Usage",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(container_cpu_usage_seconds_total{name=~\"devserver.*\"}[5m]) * 100",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 16},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Container Memory Usage",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "container_memory_usage_bytes{name=~\"devserver.*\"} / 1024 / 1024",
|
||||
"legendFormat": "{{name}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 16},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "mbytes"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "RabbitMQ Queue Size",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rabbitmq_queue_messages",
|
||||
"legendFormat": "{{queue}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 6, "w": 12, "x": 0, "y": 24},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "RabbitMQ Message Rate",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(rabbitmq_queue_messages_published_total[5m])",
|
||||
"legendFormat": "Published",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "rate(rabbitmq_queue_messages_delivered_total[5m])",
|
||||
"legendFormat": "Delivered",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 6, "w": 12, "x": 12, "y": 24},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "msgs/s"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "OpenTelemetry Collector Metrics",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(otelcol_receiver_accepted_spans[5m])",
|
||||
"legendFormat": "Spans Received",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "rate(otelcol_exporter_sent_spans[5m])",
|
||||
"legendFormat": "Spans Exported",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 6, "w": 12, "x": 0, "y": 30},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "Process Memory Usage",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "process_resident_memory_bytes / 1024 / 1024",
|
||||
"legendFormat": "{{job}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 6, "w": 12, "x": 12, "y": 30},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "mbytes"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"enable": true,
|
||||
"expr": "ALERTS{alertstate=\"firing\"}",
|
||||
"iconColor": "red",
|
||||
"name": "Alerts",
|
||||
"step": "60s",
|
||||
"tagKeys": "alertname",
|
||||
"textFormat": "{{alertname}}",
|
||||
"titleFormat": "Alert"
|
||||
}
|
||||
]
|
||||
},
|
||||
"links": [],
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"style": "dark"
|
||||
}
|
||||
10
monitoring/grafana/provisioning/dashboards/dashboards.yml
Normal file
10
monitoring/grafana/provisioning/dashboards/dashboards.yml
Normal file
@@ -0,0 +1,10 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: 'DevServer Dashboards'
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 10
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
27
monitoring/grafana/provisioning/datasources/datasources.yaml
Normal file
27
monitoring/grafana/provisioning/datasources/datasources.yaml
Normal file
@@ -0,0 +1,27 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: true
|
||||
jsonData:
|
||||
timeInterval: "5s"
|
||||
|
||||
- name: Loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
isDefault: false
|
||||
editable: true
|
||||
jsonData:
|
||||
maxLines: 1000
|
||||
|
||||
- name: Jaeger
|
||||
type: jaeger
|
||||
access: proxy
|
||||
url: http://jaeger:16686
|
||||
isDefault: false
|
||||
editable: true
|
||||
51
monitoring/loki-config.yaml
Normal file
51
monitoring/loki-config.yaml
Normal file
@@ -0,0 +1,51 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /tmp/loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /tmp/loki/chunks
|
||||
rules_directory: /tmp/loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: false
|
||||
|
||||
query_range:
|
||||
results_cache:
|
||||
cache:
|
||||
embedded_cache:
|
||||
enabled: true
|
||||
max_size_mb: 100
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2020-10-24
|
||||
store: boltdb-shipper
|
||||
object_store: filesystem
|
||||
schema: v11
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
ruler:
|
||||
alertmanager_url: http://localhost:9093
|
||||
|
||||
# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
|
||||
# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
|
||||
#
|
||||
# Statistics help us better understand how Loki is used and guide our investments.
|
||||
# This helps us prioritize features and documentation.
|
||||
# For more information on the stats we collect check out
|
||||
# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go
|
||||
# Refer to the documentation for more info on configuring the usage_stats config.
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
79
monitoring/otel-collector-config.yaml
Normal file
79
monitoring/otel-collector-config.yaml
Normal file
@@ -0,0 +1,79 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
prometheus:
|
||||
config:
|
||||
scrape_configs:
|
||||
- job_name: 'otel-collector'
|
||||
static_configs:
|
||||
- targets: ['localhost:8888']
|
||||
|
||||
processors:
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 1024
|
||||
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 512
|
||||
spike_limit_mib: 128
|
||||
|
||||
exporters:
|
||||
debug:
|
||||
verbosity: detailed
|
||||
|
||||
prometheus:
|
||||
endpoint: "0.0.0.0:8889"
|
||||
namespace: otelcol
|
||||
|
||||
prometheusremotewrite:
|
||||
endpoint: "http://prometheus:9090/api/v1/write"
|
||||
|
||||
# Jaeger exporter - using OTLP/gRPC protocol
|
||||
otlp/jaeger:
|
||||
endpoint: "jaeger:4317"
|
||||
tls:
|
||||
insecure: true
|
||||
|
||||
file/logs:
|
||||
path: "/var/log/otel-collector/logs.json"
|
||||
rotation:
|
||||
max_megabytes: 100
|
||||
max_days: 3
|
||||
max_backups: 3
|
||||
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
pprof:
|
||||
endpoint: 0.0.0.0:1888
|
||||
|
||||
zpages:
|
||||
endpoint: 0.0.0.0:55679
|
||||
|
||||
service:
|
||||
extensions: [health_check, pprof, zpages]
|
||||
telemetry:
|
||||
logs:
|
||||
level: info
|
||||
metrics:
|
||||
level: detailed
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [debug, otlp/jaeger]
|
||||
metrics:
|
||||
receivers: [otlp, prometheus]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus, prometheusremotewrite, debug]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [file/logs, debug]
|
||||
52
monitoring/prometheus.yml
Normal file
52
monitoring/prometheus.yml
Normal file
@@ -0,0 +1,52 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
# Prometheus itself
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
# FastAPI Application
|
||||
- job_name: 'fastapi-app'
|
||||
static_configs:
|
||||
- targets: ['devserver:8000']
|
||||
scrape_interval: 5s
|
||||
scrape_timeout: 4s
|
||||
metrics_path: '/metrics'
|
||||
|
||||
# RabbitMQ Metrics
|
||||
- job_name: 'rabbitmq'
|
||||
static_configs:
|
||||
- targets: ['rabbitmq:15692']
|
||||
scrape_interval: 15s
|
||||
|
||||
# Loki Metrics
|
||||
- job_name: 'loki'
|
||||
static_configs:
|
||||
- targets: ['loki:3100']
|
||||
scrape_interval: 15s
|
||||
metrics_path: '/metrics'
|
||||
|
||||
# cAdvisor Container Metrics
|
||||
- job_name: 'cadvisor'
|
||||
static_configs:
|
||||
- targets: ['cadvisor:8080']
|
||||
scrape_interval: 15s
|
||||
metrics_path: '/metrics'
|
||||
|
||||
# OTEL Collector Metrics
|
||||
- job_name: 'otel-collector'
|
||||
static_configs:
|
||||
- targets: ['otel-collector:8889']
|
||||
scrape_interval: 15s
|
||||
scrape_timeout: 10s
|
||||
metrics_path: '/metrics'
|
||||
|
||||
# Jaeger Metrics
|
||||
- job_name: 'jaeger'
|
||||
static_configs:
|
||||
- targets: ['jaeger:14269']
|
||||
scrape_interval: 15s
|
||||
metrics_path: '/metrics'
|
||||
44
monitoring/promtail-config.yaml
Normal file
44
monitoring/promtail-config.yaml
Normal file
@@ -0,0 +1,44 @@
|
||||
server:
|
||||
http_listen_port: 9080
|
||||
grpc_listen_port: 0
|
||||
|
||||
positions:
|
||||
filename: /tmp/positions.yaml
|
||||
|
||||
clients:
|
||||
- url: http://loki:3100/loki/api/v1/push
|
||||
|
||||
scrape_configs:
|
||||
- job_name: docker
|
||||
docker_sd_configs:
|
||||
- host: unix:///var/run/docker.sock
|
||||
refresh_interval: 5s
|
||||
relabel_configs:
|
||||
- source_labels: ['__meta_docker_container_name']
|
||||
regex: '/(.*)'
|
||||
target_label: 'container'
|
||||
- source_labels: ['__meta_docker_container_name']
|
||||
regex: '/(.*)'
|
||||
target_label: 'container_name'
|
||||
- source_labels: ['__meta_docker_container_name']
|
||||
regex: '/(.*)'
|
||||
target_label: 'job'
|
||||
- source_labels: ['__meta_docker_container_name']
|
||||
regex: '/(.*)'
|
||||
replacement: '$1'
|
||||
target_label: 'instance'
|
||||
pipeline_stages:
|
||||
- json:
|
||||
expressions:
|
||||
level: level
|
||||
timestamp: timestamp
|
||||
message: message
|
||||
service: service
|
||||
- labels:
|
||||
level:
|
||||
service:
|
||||
- timestamp:
|
||||
source: timestamp
|
||||
format: '2006-01-02T15:04:05.000Z07:00'
|
||||
- output:
|
||||
source: message
|
||||
Reference in New Issue
Block a user