Initial commit: Complete DevServer monitoring framework

- FastAPI application with OpenTelemetry instrumentation
- Redis caching and Celery task processing
- Docker Compose setup with monitoring stack
- Grafana dashboards and Prometheus metrics
- Jaeger distributed tracing
- Comprehensive documentation in docs/ folder
- Monitoring configuration and troubleshooting guides

app/opentelemetry_config.py

@@ -0,0 +1,140 @@
+"""
+OpenTelemetry Configuration for DevServer
+
+This module provides explicit OpenTelemetry initialization to ensure
+proper trace generation and export to the OTEL Collector.
+"""
+
+import os
+import logging
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.resources import Resource, SERVICE_NAME, SERVICE_VERSION
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.redis import RedisInstrumentor
+
+logger = logging.getLogger(__name__)
+
+
+def init_opentelemetry():
+    """
+    Initialize OpenTelemetry with OTLP exporter.
+    
+    This function:
+    1. Creates a TracerProvider with service metadata
+    2. Configures OTLP gRPC exporter to send traces to the OTEL Collector
+    3. Sets up batch span processing for efficient trace export
+    4. Registers the provider globally
+    
+    Environment Variables:
+        OTEL_SERVICE_NAME: Name of the service (default: devserver.api.fastapi)
+        OTEL_EXPORTER_OTLP_ENDPOINT: OTLP endpoint (default: http://otel-collector:4317)
+        OTEL_PYTHON_LOG_LEVEL: OpenTelemetry log level (optional)
+    """
+    
+    # Get configuration from environment
+    service_name = os.getenv("OTEL_SERVICE_NAME", "devserver.api.fastapi")
+    otlp_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4317")
+    
+    logger.info(f"🔧 Initializing OpenTelemetry for service: {service_name}")
+    logger.info(f"📡 OTLP Endpoint: {otlp_endpoint}")
+    
+    # Create resource with service metadata
+    resource = Resource(attributes={
+        SERVICE_NAME: service_name,
+        SERVICE_VERSION: "1.0.0",
+        "deployment.environment": os.getenv("ENVIRONMENT", "development"),
+    })
+    
+    # Create tracer provider with resource
+    provider = TracerProvider(resource=resource)
+    
+    try:
+        # Create OTLP exporter for traces
+        otlp_exporter = OTLPSpanExporter(
+            endpoint=otlp_endpoint,
+            insecure=True  # Use insecure channel (no TLS) for internal network
+        )
+        
+        # Add batch span processor for efficient export
+        span_processor = BatchSpanProcessor(otlp_exporter)
+        provider.add_span_processor(span_processor)
+        
+        # Set as global tracer provider
+        trace.set_tracer_provider(provider)
+        
+        logger.info("✅ OpenTelemetry TracerProvider initialized successfully")
+        logger.info("✅ OTLP exporter configured and ready")
+        
+    except Exception as e:
+        logger.error(f"❌ Failed to initialize OpenTelemetry: {e}")
+        logger.error("Traces will not be exported!")
+        raise
+    
+    return provider
+
+
+def instrument_fastapi_app(app):
+    """
+    Instrument a FastAPI application with OpenTelemetry.
+    
+    This function:
+    1. Initializes the global OpenTelemetry tracer provider
+    2. Instruments the FastAPI app to generate spans for HTTP requests
+    3. Instruments Redis client to trace cache operations
+    
+    Args:
+        app: FastAPI application instance to instrument
+        
+    Returns:
+        TracerProvider: The initialized tracer provider
+    """
+    
+    logger.info("🚀 Starting OpenTelemetry instrumentation...")
+    
+    # Initialize OpenTelemetry first
+    provider = init_opentelemetry()
+    
+    # Instrument FastAPI
+    try:
+        FastAPIInstrumentor.instrument_app(app)
+        logger.info("✅ FastAPI instrumented - HTTP requests will generate spans")
+    except Exception as e:
+        logger.error(f"❌ Failed to instrument FastAPI: {e}")
+    
+    # Instrument Redis
+    try:
+        RedisInstrumentor().instrument()
+        logger.info("✅ Redis instrumented - Cache operations will generate spans")
+    except Exception as e:
+        logger.error(f"❌ Failed to instrument Redis: {e}")
+    
+    logger.info("🎉 OpenTelemetry instrumentation complete!")
+    
+    return provider
+
+
+def get_tracer(name: str = __name__):
+    """
+    Get a tracer instance for manual span creation.
+    
+    Args:
+        name: Name of the tracer (usually __name__ of the calling module)
+        
+    Returns:
+        Tracer: OpenTelemetry tracer instance
+        
+    Example:
+        ```python
+        from app.opentelemetry_config import get_tracer
+        
+        tracer = get_tracer(__name__)
+        
+        with tracer.start_as_current_span("my-operation"):
+            # Your code here
+            pass
+        ```
+    """
+    return trace.get_tracer(name)