feat: implement prometheus exporter (#406)

Co-authored-by: mickael <contact@mickael-caudrelier.fr>

.gitignore

@@ -10,6 +10,7 @@ Thumbs.db
 
 # Python virtual-envs & tooling
 .venv*/
+venv/
 .python-version
 __pycache__/
 *.egg-info/
@@ -38,3 +39,6 @@ tmp/
 # Project-specific files
 history.txt
 digest.txt
+
+# Environment variables
+.env

.pre-commit-config.yaml

@@ -117,6 +117,7 @@ repos:
             'fastapi[standard]>=0.109.1',
             httpx,
             pathspec>=0.12.1,
+            prometheus-client,
             pydantic,
             pytest-asyncio,
             pytest-mock,
@@ -138,6 +139,7 @@ repos:
             'fastapi[standard]>=0.109.1',
             httpx,
             pathspec>=0.12.1,
+            prometheus-client,
             pydantic,
             pytest-asyncio,
             pytest-mock,

Dockerfile

@@ -43,4 +43,5 @@ RUN set -eux; \
 USER appuser
 
 EXPOSE 8000
+EXPOSE 9090
 CMD ["python", "-m", "uvicorn", "server.main:app", "--host", "0.0.0.0", "--port", "8000"]

pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
     "tiktoken>=0.7.0",  # Support for o200k_base encoding
     "typing_extensions>= 4.0.0; python_version < '3.10'",
     "uvicorn>=0.11.7",  # Minimum safe release (https://osv.dev/vulnerability/PYSEC-2020-150)
+    "prometheus-client",
 ]
 
 license = {file = "LICENSE"}

requirements.txt

@@ -2,6 +2,7 @@ click>=8.0.0
 fastapi[standard]>=0.109.1  # Vulnerable to https://osv.dev/vulnerability/PYSEC-2024-38
 httpx
 pathspec>=0.12.1
+prometheus-client
 pydantic
 python-dotenv
 slowapi

src/server/main.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import os
+import threading
 from pathlib import Path
 
 from dotenv import load_dotenv
@@ -12,6 +13,7 @@ from fastapi.staticfiles import StaticFiles
 from slowapi.errors import RateLimitExceeded
 from starlette.middleware.trustedhost import TrustedHostMiddleware
 
+from server.metrics_server import start_metrics_server
 from server.routers import dynamic, index, ingest
 from server.server_config import templates
 from server.server_utils import lifespan, limiter, rate_limit_exception_handler
@@ -26,6 +28,17 @@ app.state.limiter = limiter
 # Register the custom exception handler for rate limits
 app.add_exception_handler(RateLimitExceeded, rate_limit_exception_handler)
 
+# Start metrics server in a separate thread if enabled
+if os.getenv("GITINGEST_METRICS_ENABLED", "false").lower() == "true":
+    metrics_host = os.getenv("GITINGEST_METRICS_HOST", "127.0.0.1")
+    metrics_port = int(os.getenv("GITINGEST_METRICS_PORT", "9090"))
+    metrics_thread = threading.Thread(
+        target=start_metrics_server,
+        args=(metrics_host, metrics_port),
+        daemon=True,
+    )
+    metrics_thread.start()
+
 
 # Mount static files dynamically to serve CSS, JS, and other static assets
 static_dir = Path(__file__).parent.parent / "static"

src/server/metrics_server.py

@@ -0,0 +1,57 @@
+"""Prometheus metrics server running on a separate port."""
+
+import logging
+
+import uvicorn
+from fastapi import FastAPI
+from fastapi.responses import HTMLResponse
+from prometheus_client import REGISTRY, generate_latest
+
+# Create a logger for this module
+logger = logging.getLogger(__name__)
+
+# Create a separate FastAPI app for metrics
+metrics_app = FastAPI(
+    title="Gitingest Metrics",
+    description="Prometheus metrics for Gitingest",
+    docs_url=None,
+    redoc_url=None,
+)
+
+
+@metrics_app.get("/metrics")
+async def metrics() -> HTMLResponse:
+    """Serve Prometheus metrics without authentication.
+
+    This endpoint is only accessible from the local network.
+
+    Returns
+    -------
+    HTMLResponse
+        Prometheus metrics in text format
+
+    """
+    return HTMLResponse(
+        content=generate_latest(REGISTRY),
+        status_code=200,
+        media_type="text/plain",
+    )
+
+
+def start_metrics_server(host: str = "127.0.0.1", port: int = 9090) -> None:
+    """Start the metrics server on a separate port.
+
+    Parameters
+    ----------
+    host : str
+        The host to bind to (default: 127.0.0.1 for local network only)
+    port : int
+        The port to bind to (default: 9090)
+
+    Returns
+    -------
+    None
+
+    """
+    logger.info("Starting metrics server on %s:%s", host, port)
+    uvicorn.run(metrics_app, host=host, port=port)

src/server/routers/ingest.py

@@ -2,6 +2,7 @@
 
 from fastapi import APIRouter, HTTPException, Request, status
 from fastapi.responses import FileResponse, JSONResponse
+from prometheus_client import Counter
 
 from gitingest.config import TMP_BASE_PATH
 from server.models import IngestRequest
@@ -9,6 +10,8 @@ from server.routers_utils import COMMON_INGEST_RESPONSES, _perform_ingestion
 from server.server_config import MAX_DISPLAY_SIZE
 from server.server_utils import limiter
 
+ingest_counter = Counter("gitingest_ingest_total", "Number of ingests", ["status", "url"])
+
 router = APIRouter()
 
 
@@ -33,13 +36,16 @@ async def api_ingest(
     - **JSONResponse**: Success response with ingestion results or error response with appropriate HTTP status code
 
     """
-    return await _perform_ingestion(
+    response = await _perform_ingestion(
         input_text=ingest_request.input_text,
         max_file_size=ingest_request.max_file_size,
         pattern_type=ingest_request.pattern_type,
         pattern=ingest_request.pattern,
         token=ingest_request.token,
     )
+    # limit URL to 255 characters
+    ingest_counter.labels(status=response.status_code, url=ingest_request.input_text[:255]).inc()
+    return response
 
 
 @router.get("/api/{user}/{repository}", responses=COMMON_INGEST_RESPONSES)
@@ -72,13 +78,16 @@ async def api_ingest_get(
     **Returns**
     - **JSONResponse**: Success response with ingestion results or error response with appropriate HTTP status code
     """
-    return await _perform_ingestion(
+    response = await _perform_ingestion(
         input_text=f"{user}/{repository}",
         max_file_size=max_file_size,
         pattern_type=pattern_type,
         pattern=pattern,
         token=token or None,
     )
+    # limit URL to 255 characters
+    ingest_counter.labels(status=response.status_code, url=f"{user}/{repository}"[:255]).inc()
+    return response
 
 
 @router.get("/api/download/file/{ingest_id}", response_class=FileResponse)

src/server/server_config.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+from pathlib import Path
+
 from fastapi.templating import Jinja2Templates
 
 MAX_DISPLAY_SIZE: int = 300_000
@@ -19,4 +21,7 @@ EXAMPLE_REPOS: list[dict[str, str]] = [
     {"name": "ApiAnalytics", "url": "https://github.com/tom-draper/api-analytics"},
 ]
 
-templates = Jinja2Templates(directory="server/templates")
+
+# Use absolute path to templates directory
+templates_dir = Path(__file__).parent / "templates"
+templates = Jinja2Templates(directory=templates_dir)