diff --git a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json index 9d1aaf0b4..2ba18149b 100644 --- a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json +++ b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json @@ -41,8 +41,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "max(user_total{service_name=~\"$service\", deployment_environment=~\"$env\"})", - "legendFormat": "total users" + "legendFormat": "total users", + "range": true }, "version": "v0" }, @@ -60,8 +62,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(increase(user_registered_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[24h]))", - "legendFormat": "new today" + "legendFormat": "new today", + "range": true }, "version": "v0" }, @@ -119,7 +123,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -141,8 +145,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "avg(user_active_sessions{service_name=~\"$service\", deployment_environment=~\"$env\"})", - "legendFormat": "sessions" + "legendFormat": "sessions", + "range": true }, "version": "v0" }, @@ -200,7 +206,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -222,8 +228,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[5m]))", - "legendFormat": "req/s" + "legendFormat": "req/s", + "range": true }, "version": "v0" }, @@ -288,7 +296,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -310,8 +318,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "100 * sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=~\"5..\"}[5m])) / clamp_min(sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[5m])), 1)", - "legendFormat": "fail %" + "legendFormat": "fail %", + "range": true }, "version": "v0" }, @@ -376,7 +386,7 @@ "sizing": "auto" } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -398,8 +408,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))", - "legendFormat": "req/s" + "legendFormat": "req/s", + "range": true }, "version": "v0" }, @@ -456,7 +468,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -478,8 +490,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "max(user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"24h\"})", - "legendFormat": "DAU" + "legendFormat": "DAU", + "range": true }, "version": "v0" }, @@ -538,7 +552,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -560,8 +574,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "max(user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"7d\"})", - "legendFormat": "WAU" + "legendFormat": "WAU", + "range": true }, "version": "v0" }, @@ -620,7 +636,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -642,8 +658,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "max(user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"30d\"})", - "legendFormat": "MAU" + "legendFormat": "MAU", + "range": true }, "version": "v0" }, @@ -702,7 +720,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -724,8 +742,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(ws_connections_active{service_name=~\"$service\", deployment_environment=~\"$env\"})", - "legendFormat": "online" + "legendFormat": "online", + "range": true }, "version": "v0" }, @@ -784,7 +804,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -806,8 +826,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(ws_connections_active{service_name=~\"$service\", deployment_environment=~\"$env\"})", - "legendFormat": "connections" + "legendFormat": "connections", + "range": true }, "version": "v0" }, @@ -890,6 +912,8 @@ "max" ], "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", "placement": "right", "showLegend": true }, @@ -900,7 +924,7 @@ } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -922,8 +946,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\"}[$__range]))", - "legendFormat": "events" + "legendFormat": "events", + "range": true }, "version": "v0" }, @@ -982,7 +1008,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -1004,8 +1030,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "100 * sum(increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", status=\"failed\"}[$__range])) / clamp_min(sum(increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\"}[$__range])), 1)", - "legendFormat": "failed %" + "legendFormat": "failed %", + "range": true }, "version": "v0" }, @@ -1071,7 +1099,7 @@ "sizing": "auto" } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -1093,6 +1121,7 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "topk(12, sum by (feature, action, status) (increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\"}[$__range])))", "legendFormat": "{{feature}} · {{action}} · {{status}}", "instant": true, @@ -1154,7 +1183,7 @@ "valueMode": "color" } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -1176,8 +1205,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum by (feature, action, status) (rate(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\"}[$__rate_interval]))", - "legendFormat": "{{feature}} · {{action}} · {{status}}" + "legendFormat": "{{feature}} · {{action}} · {{status}}", + "range": true }, "version": "v0" }, @@ -1260,6 +1291,8 @@ "max" ], "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", "placement": "right", "showLegend": true }, @@ -1270,7 +1303,7 @@ } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -1292,8 +1325,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { - "expr": "100 * sum(increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", feature=\"tts\", action=\"speech_succeeded\", status=\"succeeded\"}[$__range])) / clamp_min(sum(increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", feature=\"tts\", action=\"speech_requested\", status=\"started\"}[$__range])), 1)", - "legendFormat": "success %" + "editorMode": "code", + "expr": "max (user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"24h\"})", + "legendFormat": "__auto", + "range": true }, "version": "v0" }, @@ -1305,182 +1340,10 @@ "transformations": [] } }, - "description": "Server-side TTS successes divided by TTS requests over the dashboard range. Includes REST and WS TTS product events. Drops here mean users are asking for speech but not receiving audio; inspect failed/blocked panels next.", + "description": "", "id": 99, "links": [], - "title": "TTS Success %", - "vizConfig": { - "group": "gauge", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": 0 - }, - { - "color": "yellow", - "value": 90 - }, - { - "color": "green", - "value": 98 - } - ] - }, - "unit": "percent", - "decimals": 2, - "noValue": "0", - "min": 0, - "max": 100 - }, - "overrides": [] - }, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - } - }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-100": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "topk(12, sum by (action, status, source) (increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", feature=\"tts\", action=~\"speech_failed|speech_blocked\"}[$__range])))", - "legendFormat": "{{action}} · {{status}} · {{source}}", - "instant": true, - "range": false - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "TTS user-impacting failures over the dashboard range, split by action/status/source. `speech_failed` usually means upstream/runtime failure; `speech_blocked` usually means balance/preflight blocked. Keep voice/model drilldown in Postgres metadata, not Prometheus labels.", - "id": 100, - "links": [], - "title": "TTS Failed / Blocked (range)", - "vizConfig": { - "group": "bargauge", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 12, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - } - }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-101": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "sum by (source, action, status) (rate(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", feature=\"tts\"}[$__rate_interval]))", - "legendFormat": "{{source}} · {{action}} · {{status}}" - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "TTS product event rate by source and action. Use this to distinguish chat auto-TTS, manual previews/settings tests, and API audio.speech traffic when speech health changes.", - "id": 101, - "links": [], - "title": "TTS Event Rate by Source", + "title": "日活", "vizConfig": { "group": "timeseries", "kind": "VizConfig", @@ -1488,7 +1351,8 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "continuous-GrYlRd", + "seriesBy": "last" }, "custom": { "axisBorderShow": false, @@ -1499,17 +1363,20 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", + "fillOpacity": 17, + "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 3, "scaleDistribution": { "type": "linear" }, @@ -1530,10 +1397,13 @@ { "color": "green", "value": 0 + }, + { + "color": "red", + "value": 80 } ] - }, - "unit": "eps" + } }, "overrides": [] }, @@ -1543,188 +1413,21 @@ "multiLane": false }, "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "right", + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "multi", - "sort": "desc" + "mode": "single", + "sort": "none" } } }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-102": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "topk(12, sum by (reason, source) (increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", feature=\"tts\", action=\"speech_blocked\", status=\"blocked\"}[$__range])))", - "legendFormat": "{{reason}} · {{source}}", - "instant": true, - "range": false - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Blocked TTS events over the dashboard range, grouped by bounded product reason and source. Today this mostly shows insufficient balance; new policy/provider/preflight buckets can be added without exposing user, voice, or model labels.", - "id": 102, - "links": [], - "title": "TTS Blocked by Reason", - "vizConfig": { - "group": "bargauge", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 12, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - } - }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-103": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "topk(8, sum by (flux_balance_bucket, source) (increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", feature=\"tts\", action=\"speech_blocked\", status=\"blocked\", flux_balance_bucket!=\"\"}[$__range])))", - "legendFormat": "{{flux_balance_bucket}} · {{source}}", - "instant": true, - "range": false - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Blocked TTS events over the dashboard range, grouped by coarse Flux balance bucket. This helps separate truly empty accounts from low-balance accounts without exposing exact user balances.", - "id": 103, - "links": [], - "title": "TTS Blocked by Flux Bucket", - "vizConfig": { - "group": "bargauge", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 12, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - } - }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -1746,7 +1449,8 @@ "group": "prometheus", "kind": "DataQuery", "spec": { - "expr": "topk(10, sum by (http_route) (increase(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_route!=\"\"}[$__range])))", + "editorMode": "code", + "expr": "topk(50, sum by (http_route) (increase(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_route!=\"\"}[$__range])))", "legendFormat": "{{http_route}}", "instant": true, "range": false @@ -1806,7 +1510,7 @@ "valueMode": "color" } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -1828,8 +1532,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[$__rate_interval]))", - "legendFormat": "{{http_response_status_code}}" + "legendFormat": "{{http_response_status_code}}", + "range": true }, "version": "v0" }, @@ -1844,7 +1550,7 @@ "description": "HTTP status-code mix over time, one row per status code, colour = request rate in each time bucket. The 200 row dominates in steady state; a 5xx / 4xx row suddenly lighting up flags an incident at a glance. Non-OPTIONS traffic only.", "id": 40, "links": [], - "title": "Error Rate %", + "title": "Status Distribution", "vizConfig": { "group": "heatmap", "kind": "VizConfig", @@ -1904,7 +1610,7 @@ } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -1926,8 +1632,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { - "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_route!~\"/api/v1/openai/.*\", http_response_status_code!=\"404\"}[$__rate_interval])))", - "legendFormat": "{{http_route}}" + "editorMode": "code", + "expr": "sum by (http_route) (\n rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_route!~\"/api/v1/openai/.*\", http_response_status_code!=\"404\"}[$__rate_interval])\n)", + "legendFormat": "{{http_route}}", + "range": true }, "version": "v0" }, @@ -1939,10 +1647,10 @@ "transformations": [] } }, - "description": "P95 latency per Hono-matched route, excluding /api/v1/openai/* (LLM gateway latency lives in the LLM Gateway row). 404s excluded so missing-route noise does not skew the curve.", + "description": "", "id": 20, "links": [], - "title": "HTTP P95 by Route", + "title": "Request Latency (by Route)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", @@ -2010,7 +1718,9 @@ "max" ], "displayMode": "table", - "placement": "right", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -2020,7 +1730,7 @@ } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -2042,8 +1752,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum by (http_route, http_response_status_code) (increase(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code!~\"2..|3..|401|402|404\"}[$__rate_interval]))", - "legendFormat": "{{http_response_status_code}} {{http_route}}" + "legendFormat": "{{http_response_status_code}} {{http_route}}", + "range": true }, "version": "v0" }, @@ -2126,6 +1838,8 @@ "max" ], "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", "placement": "right", "showLegend": true }, @@ -2136,7 +1850,7 @@ } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -2158,8 +1872,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", - "legendFormat": "{{gen_ai_request_model}}" + "legendFormat": "{{gen_ai_request_model}}", + "range": true }, "version": "v0" }, @@ -2176,83 +1892,53 @@ "links": [], "title": "LLM Request Rate by Model", "vizConfig": { - "group": "timeseries", + "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { + "fixedColor": "#73BF69", "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, "unit": "reqps" }, "overrides": [] }, "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, + "displayLabels": [ + "percent" + ], "legend": { - "calcs": [ - "lastNotNull", - "max" - ], "displayMode": "table", - "placement": "right", + "overflow": "ellipsis", + "placement": "bottom", "showLegend": true }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "sort": "desc", "tooltip": { "hideZeros": false, - "mode": "multi", - "sort": "desc" + "mode": "single", + "sort": "none" } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -2274,8 +1960,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { - "expr": "histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_first_token_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])))", - "legendFormat": "TTFB p95" + "editorMode": "code", + "expr": "sum by () (rate(gen_ai_client_first_token_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "TTFB p95", + "range": true }, "version": "v0" }, @@ -2293,8 +1981,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])))", - "legendFormat": "end-to-end p95" + "legendFormat": "end-to-end p95", + "range": true }, "version": "v0" }, @@ -2377,6 +2067,8 @@ "max" ], "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", "placement": "right", "showLegend": true }, @@ -2387,7 +2079,7 @@ } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -2409,8 +2101,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\"}[$__rate_interval]))", - "legendFormat": "{{provider}}" + "legendFormat": "{{provider}}", + "range": true }, "version": "v0" }, @@ -2493,6 +2187,8 @@ "max" ], "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", "placement": "right", "showLegend": true }, @@ -2503,7 +2199,7 @@ } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -2525,8 +2221,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, provider) (rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\"}[$__rate_interval])))", - "legendFormat": "{{provider}}" + "legendFormat": "{{provider}}", + "range": true }, "version": "v0" }, @@ -2609,6 +2307,8 @@ "max" ], "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", "placement": "right", "showLegend": true }, @@ -2619,7 +2319,7 @@ } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -2641,8 +2341,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "100 * sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\", http_response_status_code=~\"4..|5..\"}[$__rate_interval])) / clamp_min(sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\"}[$__rate_interval])), 1)", - "legendFormat": "{{provider}}" + "legendFormat": "{{provider}}", + "range": true }, "version": "v0" }, @@ -2725,6 +2427,8 @@ "max" ], "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", "placement": "right", "showLegend": true }, @@ -2735,7 +2439,7 @@ } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -2757,8 +2461,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum by (model) (rate(airi_billing_tts_chars_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "{{model}}" + "legendFormat": "{{model}}", + "range": true }, "version": "v0" }, @@ -2841,6 +2547,8 @@ "max" ], "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", "placement": "right", "showLegend": true }, @@ -2851,7 +2559,7 @@ } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -2873,8 +2581,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(increase(gen_ai_client_token_usage_input_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range]))", - "legendFormat": "input" + "legendFormat": "input", + "range": true }, "version": "v0" }, @@ -2892,8 +2602,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(increase(gen_ai_client_token_usage_output_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range]))", - "legendFormat": "output" + "legendFormat": "output", + "range": true }, "version": "v0" }, @@ -2952,7 +2664,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -2974,8 +2686,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(rate(gen_ai_client_token_usage_input_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "input tokens/s" + "legendFormat": "input tokens/s", + "range": true }, "version": "v0" }, @@ -2993,8 +2707,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(rate(gen_ai_client_token_usage_output_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "output tokens/s" + "legendFormat": "output tokens/s", + "range": true }, "version": "v0" }, @@ -3077,6 +2793,8 @@ "max" ], "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", "placement": "right", "showLegend": true }, @@ -3087,7 +2805,7 @@ } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -3109,8 +2827,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(increase(airi_billing_flux_unbilled_total{service_name=~\"$service\", deployment_environment=~\"$env\", reason!=\"partial_debit_drained\"}[$__range]))", - "legendFormat": "flux" + "legendFormat": "flux", + "range": true }, "version": "v0" }, @@ -3171,7 +2891,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -3193,8 +2913,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(increase(airi_gen_ai_stream_interrupted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range]))", - "legendFormat": "interruptions" + "legendFormat": "interruptions", + "range": true }, "version": "v0" }, @@ -3259,7 +2981,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -3281,8 +3003,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(increase(airi_gen_ai_gateway_key_exhausted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))", - "legendFormat": "events" + "legendFormat": "events", + "range": true }, "version": "v0" }, @@ -3343,7 +3067,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -3365,8 +3089,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum(increase(airi_gen_ai_gateway_decrypt_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))", - "legendFormat": "events" + "legendFormat": "events", + "range": true }, "version": "v0" }, @@ -3427,7 +3153,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -3449,8 +3175,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])) / clamp_min(sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])), 1)", - "legendFormat": "fallback %" + "legendFormat": "fallback %", + "range": true }, "version": "v0" }, @@ -3516,7 +3244,7 @@ "sizing": "auto" } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -3538,8 +3266,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum by (provider, status_code) (rate(airi_gen_ai_gateway_upstream_errors_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "{{provider}} · {{status_code}}" + "legendFormat": "{{provider}} · {{status_code}}", + "range": true }, "version": "v0" }, @@ -3622,6 +3352,8 @@ "max" ], "displayMode": "table", + "enableFacetedFilter": false, + "overflow": "ellipsis", "placement": "right", "showLegend": true }, @@ -3632,7 +3364,7 @@ } } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -3654,8 +3386,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum by (currency) (increase(airi_stripe_revenue_minor_unit_total{service_name=~\"$service\", deployment_environment=~\"$env\", currency!=\"\"}[$__range])) / 100", - "legendFormat": "{{currency}}" + "legendFormat": "{{currency}}", + "range": true }, "version": "v0" }, @@ -3715,7 +3449,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -3737,8 +3471,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "100 * sum(increase(stripe_checkout_completed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range])) / clamp_min(sum(increase(stripe_checkout_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range])), 1)", - "legendFormat": "completed %" + "legendFormat": "completed %", + "range": true }, "version": "v0" }, @@ -3804,7 +3540,7 @@ "sizing": "auto" } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -3826,8 +3562,10 @@ "group": "prometheus", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "sum by (event_type) (increase(stripe_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", event_type!=\"\"}[$__range]))", - "legendFormat": "{{event_type}}" + "legendFormat": "{{event_type}}", + "range": true }, "version": "v0" }, @@ -3886,443 +3624,7 @@ "wideLayout": true } }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-50": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "histogram_quantile(0.95, sum by (le) (rate(db_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])))", - "legendFormat": "p95" - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "PostgreSQL query duration P95 from PgInstrumentation. Fixed 5m window. Spikes correlate with index misses, connection exhaustion, or backend lock contention.", - "id": 50, - "links": [], - "title": "DB Query P95 (5m)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 0.05 - }, - { - "color": "red", - "value": 0.5 - } - ] - }, - "unit": "s", - "decimals": 3 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-51": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "sum by (service_instance_id) (db_client_connection_count{service_name=~\"$service\", deployment_environment=~\"$env\"})", - "legendFormat": "{{service_instance_id}}" - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Open PostgreSQL connections, per replica (`service_instance_id`). Each instance has its own pool sized by env `DB_POOL_MAX`. One instance with a permanently-high count = pool leak on that pod.", - "id": 51, - "links": [], - "title": "DB Pool Connections by Instance", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-52": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "100 * sum by (service_instance_id) (v8js_memory_heap_used_bytes{service_name=~\"$service\", deployment_environment=~\"$env\"}) / clamp_min(sum by (service_instance_id) (v8js_memory_heap_limit_bytes{service_name=~\"$service\", deployment_environment=~\"$env\"}), 1)", - "legendFormat": "{{service_instance_id}}" - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "V8 heap used ÷ heap limit, per replica (`service_instance_id`). A single replica trending up while others stay flat = leak on that pod.", - "id": 52, - "links": [], - "title": "Heap Used % by Instance", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-53": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "max by (service_instance_id) (nodejs_eventloop_delay_p99_seconds{service_name=~\"$service\", deployment_environment=~\"$env\"})", - "legendFormat": "{{service_instance_id}}" - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "P99 event-loop delay per replica. One replica climbing while others stay flat = CPU-bound work pinning that pod. >50ms sustained is bad anywhere.", - "id": 53, - "links": [], - "title": "Event Loop Delay P99 by Instance", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -4344,8 +3646,10 @@ "group": "loki", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "{service_name=~\"$service\", deployment_environment=~\"$env\"} | json | level=~\"warn|error\"", - "legendFormat": "" + "legendFormat": "", + "range": true }, "version": "v0" }, @@ -4387,7 +3691,7 @@ "wrapLogMessage": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } }, @@ -4409,8 +3713,10 @@ "group": "loki", "kind": "DataQuery", "spec": { + "editorMode": "code", "expr": "{service_name=~\"$service\", deployment_environment=~\"$env\"} |= ``", - "legendFormat": "" + "legendFormat": "", + "range": true }, "version": "v0" }, @@ -4452,7 +3758,7 @@ "wrapLogMessage": true } }, - "version": "13.0.0-23630096546" + "version": "13.2.0-28666480772" } } } @@ -4474,10 +3780,10 @@ "spec": { "element": { "kind": "ElementReference", - "name": "panel-1" + "name": "panel-3" }, - "height": 4, - "width": 6, + "height": 5, + "width": 4, "x": 0, "y": 0 } @@ -4487,24 +3793,11 @@ "spec": { "element": { "kind": "ElementReference", - "name": "panel-3" + "name": "panel-5" }, - "height": 4, - "width": 6, - "x": 6, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-4" - }, - "height": 4, - "width": 6, - "x": 12, + "height": 5, + "width": 4, + "x": 4, "y": 0 } }, @@ -4515,38 +3808,25 @@ "kind": "ElementReference", "name": "panel-40" }, - "height": 8, + "height": 6, + "width": 10, + "x": 8, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-90" + }, + "height": 38, "width": 6, "x": 18, "y": 0 } }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-15" - }, - "height": 4, - "width": 6, - "x": 0, - "y": 4 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-5" - }, - "height": 4, - "width": 6, - "x": 6, - "y": 4 - } - }, { "kind": "GridLayoutItem", "spec": { @@ -4554,10 +3834,36 @@ "kind": "ElementReference", "name": "panel-93" }, - "height": 4, - "width": 6, - "x": 12, - "y": 4 + "height": 5, + "width": 4, + "x": 0, + "y": 5 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-4" + }, + "height": 5, + "width": 4, + "x": 4, + "y": 5 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-20" + }, + "height": 15, + "width": 10, + "x": 8, + "y": 6 } }, { @@ -4567,10 +3873,127 @@ "kind": "ElementReference", "name": "panel-92" }, - "height": 5, - "width": 24, + "height": 10, + "width": 4, "x": 0, - "y": 8 + "y": 10 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-73" + }, + "height": 10, + "width": 2, + "x": 4, + "y": 10 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-11" + }, + "height": 10, + "width": 2, + "x": 6, + "y": 10 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-16" + }, + "height": 18, + "width": 8, + "x": 0, + "y": 20 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-71" + }, + "height": 5, + "width": 5, + "x": 8, + "y": 21 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-21" + }, + "height": 5, + "width": 5, + "x": 13, + "y": 21 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-69" + }, + "height": 6, + "width": 5, + "x": 8, + "y": 26 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-67" + }, + "height": 6, + "width": 5, + "x": 13, + "y": 26 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-66" + }, + "height": 6, + "width": 5, + "x": 8, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-68" + }, + "height": 6, + "width": 5, + "x": 13, + "y": 32 } } ] @@ -4594,8 +4017,8 @@ "kind": "ElementReference", "name": "panel-80" }, - "height": 4, - "width": 8, + "height": 5, + "width": 3, "x": 0, "y": 0 } @@ -4605,11 +4028,37 @@ "spec": { "element": { "kind": "ElementReference", - "name": "panel-81" + "name": "panel-1" }, - "height": 4, - "width": 8, - "x": 8, + "height": 5, + "width": 3, + "x": 3, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-99" + }, + "height": 11, + "width": 12, + "x": 6, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-98" + }, + "height": 11, + "width": 6, + "x": 18, "y": 0 } }, @@ -4620,10 +4069,75 @@ "kind": "ElementReference", "name": "panel-82" }, - "height": 4, - "width": 8, - "x": 16, - "y": 0 + "height": 3, + "width": 3, + "x": 0, + "y": 5 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-15" + }, + "height": 6, + "width": 3, + "x": 3, + "y": 5 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-81" + }, + "height": 3, + "width": 3, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-95" + }, + "height": 9, + "width": 6, + "x": 0, + "y": 11 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-96" + }, + "height": 9, + "width": 6, + "x": 6, + "y": 11 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-97" + }, + "height": 9, + "width": 12, + "x": 12, + "y": 11 } } ] @@ -4639,125 +4153,7 @@ "layout": { "kind": "GridLayout", "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-95" - }, - "height": 5, - "width": 6, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-96" - }, - "height": 5, - "width": 6, - "x": 6, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-99" - }, - "height": 5, - "width": 6, - "x": 12, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-100" - }, - "height": 5, - "width": 6, - "x": 18, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-97" - }, - "height": 8, - "width": 12, - "x": 0, - "y": 5 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-98" - }, - "height": 4, - "width": 12, - "x": 12, - "y": 5 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-101" - }, - "height": 4, - "width": 12, - "x": 12, - "y": 9 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-102" - }, - "height": 5, - "width": 12, - "x": 0, - "y": 13 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-103" - }, - "height": 5, - "width": 12, - "x": 12, - "y": 13 - } - } - ] + "items": [] } }, "title": "Product Analytics" @@ -4783,32 +4179,6 @@ "x": 0, "y": 0 } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-16" - }, - "height": 11, - "width": 7, - "x": 0, - "y": 8 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-20" - }, - "height": 11, - "width": 17, - "x": 7, - "y": 8 - } } ] } @@ -4824,138 +4194,6 @@ "kind": "GridLayout", "spec": { "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-11" - }, - "height": 8, - "width": 12, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-21" - }, - "height": 8, - "width": 12, - "x": 12, - "y": 0 - } - } - ] - } - }, - "title": "LLM Gateway" - } - }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-66" - }, - "height": 7, - "width": 6, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-67" - }, - "height": 7, - "width": 6, - "x": 6, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-68" - }, - "height": 7, - "width": 6, - "x": 12, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-69" - }, - "height": 7, - "width": 6, - "x": 18, - "y": 0 - } - } - ] - } - }, - "title": "Provider Upstreams" - } - }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-73" - }, - "height": 7, - "width": 6, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-71" - }, - "height": 7, - "width": 10, - "x": 6, - "y": 0 - } - }, { "kind": "GridLayoutItem", "spec": { @@ -4964,8 +4202,8 @@ "name": "panel-43" }, "height": 7, - "width": 4, - "x": 16, + "width": 6, + "x": 0, "y": 0 } }, @@ -4977,8 +4215,8 @@ "name": "panel-41" }, "height": 7, - "width": 4, - "x": 20, + "width": 6, + "x": 6, "y": 0 } } @@ -5107,72 +4345,6 @@ "title": "Business" } }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": true, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-50" - }, - "height": 6, - "width": 6, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-51" - }, - "height": 6, - "width": 6, - "x": 6, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-52" - }, - "height": 6, - "width": 6, - "x": 12, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-53" - }, - "height": 6, - "width": 6, - "x": 18, - "y": 0 - } - } - ] - } - }, - "title": "Infrastructure" - } - }, { "kind": "RowsLayoutRow", "spec": { @@ -5188,24 +4360,11 @@ "kind": "ElementReference", "name": "panel-91" }, - "height": 10, + "height": 8, "width": 24, "x": 0, "y": 0 } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-90" - }, - "height": 10, - "width": 24, - "x": 0, - "y": 10 - } } ] } @@ -5225,7 +4384,7 @@ "grafana-cloud" ], "timeSettings": { - "autoRefresh": "30s", + "autoRefresh": "", "autoRefreshIntervals": [ "5s", "10s", @@ -5239,7 +4398,7 @@ "1d" ], "fiscalYearStartMonth": 0, - "from": "now-1h", + "from": "now-6h", "hideTimepicker": false, "timezone": "browser", "to": "now" diff --git a/apps/server/otel/grafana/dashboards/build.test.ts b/apps/server/otel/grafana/dashboards/build.test.ts index 0b82e47ee..f9dcd328e 100644 --- a/apps/server/otel/grafana/dashboards/build.test.ts +++ b/apps/server/otel/grafana/dashboards/build.test.ts @@ -58,14 +58,12 @@ describe('grafana dashboard builder', () => { * @example * expect(panelTitle('panel-99')).toBe('TTS Success %') */ - it('keeps the product analytics row focused on server-side TTS health', () => { + it('keeps the product analytics row focused on Prometheus-safe engagement signals', () => { expect(panelTitle('panel-95')).toBe('Product Events (range)') expect(panelTitle('panel-96')).toBe('Product Failure %') - expect(panelTitle('panel-99')).toBe('TTS Success %') - expect(panelTitle('panel-100')).toBe('TTS Failed / Blocked (range)') - expect(panelTitle('panel-101')).toBe('TTS Event Rate by Source') - expect(panelTitle('panel-102')).toBe('TTS Blocked by Reason') - expect(panelTitle('panel-103')).toBe('TTS Blocked by Flux Bucket') + expect(panelTitle('panel-97')).toBe('Top Product Actions (range)') + expect(panelTitle('panel-98')).toBe('Product Event Rate') + expect(panelTitle('panel-99')).toBe('日活') }) /** @@ -80,10 +78,6 @@ describe('grafana dashboard builder', () => { dashboard.elements['panel-97'], dashboard.elements['panel-98'], dashboard.elements['panel-99'], - dashboard.elements['panel-100'], - dashboard.elements['panel-101'], - dashboard.elements['panel-102'], - dashboard.elements['panel-103'], ]).join('\n') expect(productPanelExpressions).not.toContain('voice_id') diff --git a/apps/server/otel/grafana/dashboards/build.ts b/apps/server/otel/grafana/dashboards/build.ts index 0b7fc543c..bb438ce12 100644 --- a/apps/server/otel/grafana/dashboards/build.ts +++ b/apps/server/otel/grafana/dashboards/build.ts @@ -34,7 +34,7 @@ import { fileURLToPath, pathToFileURL } from 'node:url' const PROM = { name: 'grafanacloud-projairi-prom' } const LOKI = { name: 'grafanacloud-projairi-logs' } -const SCHEMA_VERSION = '13.0.0-23630096546' +const SCHEMA_VERSION = '13.2.0-28666480772' // Service / env filter applied to every Prom query. Pulled into a helper so // the variable name only appears once. @@ -62,9 +62,10 @@ function query(expr: string, legend: string, refId = 'A', datasource: DataSource group: datasource === LOKI ? 'loki' : 'prometheus', kind: 'DataQuery', spec: { + editorMode: 'code', expr, legendFormat: legend, - ...(opts.instant && { instant: true, range: false }), + ...(opts.instant ? { instant: true, range: false } : { range: true }), }, version: 'v0', }, @@ -128,6 +129,8 @@ interface TimeseriesPanelOpts { stack?: boolean fillOpacity?: number legendCalcs?: LegendCalc[] + legendPlacement?: 'bottom' | 'right' + legendDisplayMode?: 'list' | 'table' } // `noValue` shows a friendly placeholder instead of "No data" red text when @@ -269,7 +272,14 @@ function barGaugePanel(id: number, title: string, description: string, queries: } function timeseriesPanel(id: number, title: string, description: string, queries: PanelQuery[], opts: TimeseriesPanelOpts = {}) { - const { unit = 'short', stack = false, fillOpacity = 20, legendCalcs = ['lastNotNull', 'max'] } = opts + const { + unit = 'short', + stack = false, + fillOpacity = 20, + legendCalcs = ['lastNotNull', 'max'], + legendPlacement = 'right', + legendDisplayMode = 'table', + } = opts return { kind: 'Panel', spec: { @@ -318,7 +328,14 @@ function timeseriesPanel(id: number, title: string, description: string, queries // Show last + max in the legend table so viewers don't have to // click each line to see numbers — same trick as Keycloak's // "Login Errors" panel. - legend: { calcs: legendCalcs, displayMode: 'table', placement: 'right', showLegend: true }, + legend: { + calcs: legendCalcs, + displayMode: legendDisplayMode, + enableFacetedFilter: false, + overflow: 'ellipsis', + placement: legendPlacement, + showLegend: true, + }, tooltip: { hideZeros: false, mode: 'multi', sort: 'desc' }, }, }, @@ -328,6 +345,119 @@ function timeseriesPanel(id: number, title: string, description: string, queries } } +function pieChartPanel(id: number, title: string, description: string, queries: PanelQuery[], unit = 'short') { + return { + kind: 'Panel', + spec: { + data: { kind: 'QueryGroup', spec: { queries, queryOptions: {}, transformations: [] } }, + description, + id, + links: [], + title, + vizConfig: { + group: 'piechart', + kind: 'VizConfig', + spec: { + fieldConfig: { + defaults: { + color: { fixedColor: '#73BF69', mode: 'palette-classic' }, + custom: { hideFrom: { legend: false, tooltip: false, viz: false } }, + unit, + }, + overrides: [], + }, + options: { + displayLabels: ['percent'], + legend: { displayMode: 'table', overflow: 'ellipsis', placement: 'bottom', showLegend: true }, + pieType: 'pie', + reduceOptions: { calcs: ['sum'], fields: '', values: false }, + sort: 'desc', + tooltip: { hideZeros: false, mode: 'single', sort: 'none' }, + }, + }, + version: SCHEMA_VERSION, + }, + }, + } +} + +// Custom Grafana UI panel retained as code so the generated dashboard matches +// the latest hand-tuned cloud version without checking in cloud metadata. +function dailyActiveUsersTrendPanel() { + return { + kind: 'Panel', + spec: { + data: { + kind: 'QueryGroup', + spec: { + queries: [ + query( + `max (user_active_rolling{${SERVICE_FILTER}, window="24h"})`, + '__auto', + ), + ], + queryOptions: {}, + transformations: [], + }, + }, + description: '', + id: 99, + links: [], + title: '日活', + vizConfig: { + group: 'timeseries', + kind: 'VizConfig', + spec: { + fieldConfig: { + defaults: { + color: { mode: 'continuous-GrYlRd', seriesBy: 'last' }, + custom: { + axisBorderShow: false, + axisCenteredZero: false, + axisColorMode: 'text', + axisLabel: '', + axisPlacement: 'auto', + barAlignment: 0, + barWidthFactor: 0.6, + drawStyle: 'line', + fillOpacity: 17, + gradientMode: 'scheme', + hideFrom: { legend: false, tooltip: false, viz: false }, + insertNulls: false, + lineInterpolation: 'linear', + lineStyle: { fill: 'solid' }, + lineWidth: 2, + pointSize: 3, + scaleDistribution: { type: 'linear' }, + showPoints: 'auto', + showValues: false, + spanNulls: false, + stacking: { group: 'A', mode: 'none' }, + thresholdsStyle: { mode: 'off' }, + }, + thresholds: thresholds([{ color: 'green', value: 0 }, { color: 'red', value: 80 }]), + }, + overrides: [], + }, + options: { + annotations: { clustering: -1, multiLane: false }, + legend: { + calcs: [], + displayMode: 'list', + enableFacetedFilter: false, + overflow: 'ellipsis', + placement: 'bottom', + showLegend: true, + }, + tooltip: { hideZeros: false, mode: 'single', sort: 'none' }, + }, + }, + version: SCHEMA_VERSION, + }, + }, + } +} + interface HeatmapPanelOpts { unit?: string } @@ -577,69 +707,7 @@ elements['panel-98'] = timeseriesPanel( { unit: 'eps', fillOpacity: 15 }, ) -elements['panel-99'] = gaugePanel( - 99, - 'TTS Success %', - 'Server-side TTS successes divided by TTS requests over the dashboard range. Includes REST and WS TTS product events. Drops here mean users are asking for speech but not receiving audio; inspect failed/blocked panels next.', - [query( - `100 * sum(increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}, feature="tts", action="speech_succeeded", status="succeeded"}[$__range])) / clamp_min(sum(increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}, feature="tts", action="speech_requested", status="started"}[$__range])), 1)`, - 'success %', - )], - { steps: [{ color: 'red', value: 0 }, { color: 'yellow', value: 90 }, { color: 'green', value: 98 }], max: 100, decimals: 2, noValue: '0' }, -) - -elements['panel-100'] = barGaugePanel( - 100, - 'TTS Failed / Blocked (range)', - 'TTS user-impacting failures over the dashboard range, split by action/status/source. `speech_failed` usually means upstream/runtime failure; `speech_blocked` usually means balance/preflight blocked. Keep voice/model drilldown in Postgres metadata, not Prometheus labels.', - [query( - `topk(12, sum by (action, status, source) (increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}, feature="tts", action=~"speech_failed|speech_blocked"}[$__range])))`, - '{{action}} · {{status}} · {{source}}', - 'A', - PROM, - { instant: true }, - )], - { unit: 'short', noValue: '0' }, -) - -elements['panel-101'] = timeseriesPanel( - 101, - 'TTS Event Rate by Source', - 'TTS product event rate by source and action. Use this to distinguish chat auto-TTS, manual previews/settings tests, and API audio.speech traffic when speech health changes.', - [query( - `sum by (source, action, status) (rate(airi_product_events_total{${PRODUCT_EVENT_FILTER}, feature="tts"}[$__rate_interval]))`, - '{{source}} · {{action}} · {{status}}', - )], - { unit: 'eps', fillOpacity: 15 }, -) - -elements['panel-102'] = barGaugePanel( - 102, - 'TTS Blocked by Reason', - 'Blocked TTS events over the dashboard range, grouped by bounded product reason and source. Today this mostly shows insufficient balance; new policy/provider/preflight buckets can be added without exposing user, voice, or model labels.', - [query( - `topk(12, sum by (reason, source) (increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}, feature="tts", action="speech_blocked", status="blocked"}[$__range])))`, - '{{reason}} · {{source}}', - 'A', - PROM, - { instant: true }, - )], - { unit: 'short', noValue: '0' }, -) - -elements['panel-103'] = barGaugePanel( - 103, - 'TTS Blocked by Flux Bucket', - 'Blocked TTS events over the dashboard range, grouped by coarse Flux balance bucket. This helps separate truly empty accounts from low-balance accounts without exposing exact user balances.', - [query( - `topk(8, sum by (flux_balance_bucket, source) (increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}, feature="tts", action="speech_blocked", status="blocked", flux_balance_bucket!=""}[$__range])))`, - '{{flux_balance_bucket}} · {{source}}', - 'A', - PROM, - { instant: true }, - )], - { unit: 'short', noValue: '0' }, -) +elements['panel-99'] = dailyActiveUsersTrendPanel() // --- Row 2: HTTP — traffic ranking, error trend, latency trend ------------- elements['panel-16'] = barGaugePanel( @@ -647,7 +715,7 @@ elements['panel-16'] = barGaugePanel( 'Top Routes by Requests (range)', 'Top Hono-matched routes by request count over the dashboard range. The main traffic list: which API surfaces are hottest. Wildcard patterns like `/api/v1/openai/*` are requests that did not reach a concrete handler (404 / auth-rejected); concrete paths are successful routes.', [query( - `topk(10, sum by (http_route) (increase(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_route!=""}[$__range])))`, + `topk(50, sum by (http_route) (increase(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_route!=""}[$__range])))`, '{{http_route}}', 'A', PROM, @@ -658,7 +726,7 @@ elements['panel-16'] = barGaugePanel( elements['panel-40'] = heatmapPanel( 40, - 'Error Rate %', + 'Status Distribution', 'HTTP status-code mix over time, one row per status code, colour = request rate in each time bucket. The 200 row dominates in steady state; a 5xx / 4xx row suddenly lighting up flags an incident at a glance. Non-OPTIONS traffic only.', [query( `sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS"}[$__rate_interval]))`, @@ -669,13 +737,15 @@ elements['panel-40'] = heatmapPanel( elements['panel-20'] = timeseriesPanel( 20, - 'HTTP P95 by Route', - 'P95 latency per Hono-matched route, excluding /api/v1/openai/* (LLM gateway latency lives in the LLM Gateway row). 404s excluded so missing-route noise does not skew the curve.', + 'Request Latency (by Route)', + '', [query( - `histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_route!~"/api/v1/openai/.*", http_response_status_code!="404"}[$__rate_interval])))`, + `sum by (http_route) ( + rate(http_server_request_duration_seconds_bucket{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_route!~"/api/v1/openai/.*", http_response_status_code!="404"}[$__rate_interval]) +)`, '{{http_route}}', )], - { unit: 's' }, + { unit: 's', legendPlacement: 'bottom' }, ) elements['panel-94'] = timeseriesPanel( @@ -690,7 +760,7 @@ elements['panel-94'] = timeseriesPanel( ) // --- Row 3: LLM Gateway — request mix + latency ---------------------------- -elements['panel-11'] = timeseriesPanel( +elements['panel-11'] = pieChartPanel( 11, 'LLM Request Rate by Model', 'Per-model request rate (chat + tts). Useful for capacity planning and spotting model-routing regressions.', @@ -698,7 +768,7 @@ elements['panel-11'] = timeseriesPanel( `sum by (gen_ai_request_model) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, gen_ai_request_model!=""}[$__rate_interval]))`, '{{gen_ai_request_model}}', )], - { unit: 'reqps' }, + 'reqps', ) elements['panel-21'] = timeseriesPanel( @@ -706,7 +776,7 @@ elements['panel-21'] = timeseriesPanel( 'LLM Latency P95', 'Two P95 latency signals for the LLM gateway, aggregated across models. TTFB = time to first streamed token (streaming chat UX). End-to-end = full operation duration — the only latency signal for non-streaming chat and TTS, which have no first-token event.', [ - query(`histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_first_token_duration_seconds_bucket{${SERVICE_FILTER}}[$__rate_interval])))`, 'TTFB p95', 'A'), + query(`sum by () (rate(gen_ai_client_first_token_duration_seconds_bucket{${SERVICE_FILTER}}[$__rate_interval]))`, 'TTFB p95', 'A'), query(`histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_operation_duration_seconds_bucket{${SERVICE_FILTER}}[$__rate_interval])))`, 'end-to-end p95', 'B'), ], { unit: 's' }, @@ -885,51 +955,6 @@ elements['panel-32'] = statPanel( { unit: 'short', variant: 'count', noValue: '—', graphMode: 'none' }, ) -// --- Row 7: Infrastructure (collapsed) — process / DB health --------------- -elements['panel-50'] = statPanel( - 50, - 'DB Query P95 (5m)', - 'PostgreSQL query duration P95 from PgInstrumentation. Fixed 5m window. Spikes correlate with index misses, connection exhaustion, or backend lock contention.', - [query( - `histogram_quantile(0.95, sum by (le) (rate(db_client_operation_duration_seconds_bucket{${SERVICE_FILTER}}[5m])))`, - 'p95', - )], - { unit: 's', steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 0.05 }, { color: 'red', value: 0.5 }], decimals: 3 }, -) - -elements['panel-51'] = timeseriesPanel( - 51, - 'DB Pool Connections by Instance', - 'Open PostgreSQL connections, per replica (`service_instance_id`). Each instance has its own pool sized by env `DB_POOL_MAX`. One instance with a permanently-high count = pool leak on that pod.', - [query( - `sum by (service_instance_id) (db_client_connection_count{${SERVICE_FILTER}})`, - '{{service_instance_id}}', - )], - { unit: 'short' }, -) - -elements['panel-52'] = timeseriesPanel( - 52, - 'Heap Used % by Instance', - 'V8 heap used ÷ heap limit, per replica (`service_instance_id`). A single replica trending up while others stay flat = leak on that pod.', - [query( - `100 * sum by (service_instance_id) (v8js_memory_heap_used_bytes{${SERVICE_FILTER}}) / clamp_min(sum by (service_instance_id) (v8js_memory_heap_limit_bytes{${SERVICE_FILTER}}), 1)`, - '{{service_instance_id}}', - )], - { unit: 'percent' }, -) - -elements['panel-53'] = timeseriesPanel( - 53, - 'Event Loop Delay P99 by Instance', - 'P99 event-loop delay per replica. One replica climbing while others stay flat = CPU-bound work pinning that pod. >50ms sustained is bad anywhere.', - [query( - `max by (service_instance_id) (nodejs_eventloop_delay_p99_seconds{${SERVICE_FILTER}})`, - '{{service_instance_id}}', - )], - { unit: 's' }, -) - // --- Row 8: Logs ------------------------------------------------------------ elements['panel-91'] = logsPanel( 91, @@ -950,68 +975,53 @@ elements['panel-90'] = logsPanel( // --------------------------------------------------------------------------- const rows = [ - // Row 1: Service Health — two rows of glance stats + the status-code heatmap - // standing tall on the right, with the live WS-connections trend full-width - // underneath. counts (New Users / Active Sessions / WS Online) read blue with - // a trend delta; req-rate + 5xx stay traffic-light. + // Row 1: Service Health — dense single-screen operations layout from the + // latest Grafana Cloud edit. Logs stay docked on the right while traffic, + // latency, LLM, token, and provider panels fill the left. row('Service Health', [ - item('panel-1', 0, 0, 6, 4), - item('panel-3', 6, 0, 6, 4), - item('panel-4', 12, 0, 6, 4), - item('panel-40', 18, 0, 6, 8), - item('panel-15', 0, 4, 6, 4), - item('panel-5', 6, 4, 6, 4), - item('panel-93', 12, 4, 6, 4), - item('panel-92', 0, 8, 24, 5), + item('panel-3', 0, 0, 4, 5), + item('panel-5', 4, 0, 4, 5), + item('panel-40', 8, 0, 10, 6), + item('panel-90', 18, 0, 6, 38), + item('panel-93', 0, 5, 4, 5), + item('panel-4', 4, 5, 4, 5), + item('panel-20', 8, 6, 10, 15), + item('panel-92', 0, 10, 4, 10), + item('panel-73', 4, 10, 2, 10), + item('panel-11', 6, 10, 2, 10), + item('panel-16', 0, 20, 8, 18), + item('panel-71', 8, 21, 5, 5), + item('panel-21', 13, 21, 5, 5), + item('panel-69', 8, 26, 5, 6), + item('panel-67', 13, 26, 5, 6), + item('panel-66', 8, 32, 5, 6), + item('panel-68', 13, 32, 5, 6), ]), - // Row 2: User Engagement — rolling-window active users (DAU/WAU/MAU) from - // user.last_seen_at. Kept its own row so it can grow (retention, cohorts) - // without crowding the health glance above. + // Row 2: User Engagement — rolling-window active users and Prom-safe product + // analytics. The hand-tuned "日活" trend gives the row a visual engagement + // anchor while compact stats keep user/session totals nearby. row('User Engagement', [ - item('panel-80', 0, 0, 8, 4), - item('panel-81', 8, 0, 8, 4), - item('panel-82', 16, 0, 8, 4), + item('panel-80', 0, 0, 3, 5), + item('panel-1', 3, 0, 3, 5), + item('panel-99', 6, 0, 12, 11), + item('panel-98', 18, 0, 6, 11), + item('panel-82', 0, 5, 3, 3), + item('panel-15', 3, 5, 3, 6), + item('panel-81', 0, 8, 3, 3), + item('panel-95', 0, 11, 6, 9), + item('panel-96', 6, 11, 6, 9), + item('panel-97', 12, 11, 12, 9), ]), - // Row 3: Product Analytics — Prom-safe event volume and server TTS health. - // Distinct-user analytics stay in Postgres `product_events`; this row - // intentionally never uses user_id/session/request labels. - row('Product Analytics', [ - item('panel-95', 0, 0, 6, 5), - item('panel-96', 6, 0, 6, 5), - item('panel-99', 12, 0, 6, 5), - item('panel-100', 18, 0, 6, 5), - item('panel-97', 0, 5, 12, 8), - item('panel-98', 12, 5, 12, 4), - item('panel-101', 12, 9, 12, 4), - item('panel-102', 0, 13, 12, 5), - item('panel-103', 12, 13, 12, 5), - ]), - // Row 3: HTTP — full-width error breakdown on top, then traffic ranking + - // latency trend side by side. + row('Product Analytics', []), + // Row 3: HTTP — full-width error breakdown; traffic ranking and latency moved + // into the dense Service Health screen above. row('HTTP', [ item('panel-94', 0, 0, 24, 8), - item('panel-16', 0, 8, 7, 11), - item('panel-20', 7, 8, 17, 11), - ]), - // Row 4: LLM gateway — request mix + latency side by side. - row('LLM Gateway', [ - item('panel-11', 0, 0, 12, 8), - item('panel-21', 12, 0, 12, 8), - ]), - // Row 5: Provider Upstreams — per-provider rollup so the vendor consoles - // don't have to be opened one by one. Four wide, one screen line. - row('Provider Upstreams', [ - item('panel-66', 0, 0, 6, 7), - item('panel-67', 6, 0, 6, 7), - item('panel-68', 12, 0, 6, 7), - item('panel-69', 18, 0, 6, 7), ]), // Row 4: token totals + throughput + the two revenue/quality alert stats. row('LLM Tokens & Quality', [ - item('panel-73', 0, 0, 6, 7), - item('panel-71', 6, 0, 10, 7), - item('panel-43', 16, 0, 4, 7), - item('panel-41', 20, 0, 4, 7), + item('panel-43', 0, 0, 6, 7), + item('panel-41', 6, 0, 6, 7), ]), // Row 5: router health — three "wake someone up" stats/gauge + upstream errors. row('LLM Router Health', [ @@ -1026,17 +1036,10 @@ const rows = [ item('panel-31', 8, 0, 8, 7), item('panel-32', 16, 0, 8, 7), ]), - // Row 7: infra (collapsed) — by-instance breakdowns catch single-replica issues. - row('Infrastructure', [ - item('panel-50', 0, 0, 6, 6), - item('panel-51', 6, 0, 6, 6), - item('panel-52', 12, 0, 6, 6), - item('panel-53', 18, 0, 6, 6), - ], { collapse: true }), - // Row 8: full-width logs — errors on top (triage focus), firehose below. + // Row 8: focused error logs; the live application firehose is docked in + // Service Health for the latest cloud layout. row('Logs', [ - item('panel-91', 0, 0, 24, 10), - item('panel-90', 0, 10, 24, 10), + item('panel-91', 0, 0, 24, 8), ]), ] @@ -1105,18 +1108,15 @@ const variables = [ * AIRI Server Overview dashboard. * * Reading order: - * 1. Service Health — signup/sessions/WS counts, req-rate, 5xx, status-code - * heatmap, live WS trend: "is anything broken right now?" - * 2. User Engagement — rolling DAU/WAU/MAU from user.last_seen_at - * 3. Product Analytics — Prom-safe product event volume + server TTS health - * 4. HTTP — error breakdown by route, request ranking, latency by route - * 5. LLM Gateway — per-model request rate + latency (TTFB + end-to-end) - * 6. Provider Upstreams — per-provider rate/latency/failure + TTS chars - * 7. LLM Tokens & Quality — token totals/throughput, revenue-leak alerts - * 8. LLM Router Health — key/decrypt/fallback "wake someone up" signals - * 9. Business — Stripe / Flux money flow - * 10. Infrastructure (collapsed) — DB / runtime health for triage - * 11. Logs — Loki for live debugging + * 1. Service Health — dense operations screen with request, LLM, provider, + * token, WebSocket, status, and live application-log signals. + * 2. User Engagement — rolling DAU/WAU/MAU, total users, sessions, product + * event health, and the hand-tuned daily-active-user trend. + * 3. HTTP — full-width route error breakdown. + * 4. LLM Tokens & Quality — revenue-leak and stream-interruption alerts. + * 5. LLM Router Health — key/decrypt/fallback "wake someone up" signals. + * 6. Business — Stripe / Flux money flow. + * 7. Logs — Loki warning/error logs for live debugging. * * One metric, one panel: we deliberately do not duplicate a metric across * stat/trend/bar/pie forms. Counter conventions: rate() for "now" trends, @@ -1154,10 +1154,10 @@ export const dashboard = { preload: false, tags: ['airi', 'observability', 'grafana-cloud'], timeSettings: { - autoRefresh: '30s', + autoRefresh: '', autoRefreshIntervals: ['5s', '10s', '30s', '1m', '5m', '15m', '30m', '1h', '2h', '1d'], fiscalYearStartMonth: 0, - from: 'now-1h', + from: 'now-6h', hideTimepicker: false, timezone: 'browser', to: 'now',