{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#33FF00", "value": null }, { "color": "#FFCC00", "value": 100 }, { "color": "#FF4444", "value": 500 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 }, "id": 1, "options": { "colorMode": "background_gradient", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.0.0", "targets": [ { "expr": "histogram_quantile(0.5, rate(http_request_duration_seconds_bucket[$rate_interval])) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "P50", "refId": "A" }, { "expr": "histogram_quantile(0.5, rate(kubelet_http_requests_duration_seconds_bucket[$rate_interval])) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "P50 (kubelet)", "refId": "B" } ], "title": "P50 Latency", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#33FF00", "value": null }, { "color": "#FFCC00", "value": 100 }, { "color": "#FF4444", "value": 500 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 }, "id": 2, "options": { "colorMode": "background_gradient", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.0.0", "targets": [ { "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[$rate_interval])) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "P95", "refId": "A" }, { "expr": "histogram_quantile(0.95, rate(kubelet_http_requests_duration_seconds_bucket[$rate_interval])) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "P95 (kubelet)", "refId": "B" } ], "title": "P95 Latency", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#33FF00", "value": null }, { "color": "#FFCC00", "value": 100 }, { "color": "#FF4444", "value": 500 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 12, "y": 0 }, "id": 3, "options": { "colorMode": "background_gradient", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.0.0", "targets": [ { "expr": "histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[$rate_interval])) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "P99", "refId": "A" }, { "expr": "histogram_quantile(0.99, rate(kubelet_http_requests_duration_seconds_bucket[$rate_interval])) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "P99 (kubelet)", "refId": "B" } ], "title": "P99 Latency", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#33FF00", "value": null }, { "color": "#FFCC00", "value": 100 }, { "color": "#FF4444", "value": 500 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 18, "y": 0 }, "id": 4, "options": { "colorMode": "background_gradient", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.0.0", "targets": [ { "expr": "rate(http_request_duration_seconds_sum[$rate_interval]) / rate(http_request_duration_seconds_count[$rate_interval]) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "Avg", "refId": "A" }, { "expr": "rate(kubelet_http_requests_duration_seconds_sum[$rate_interval]) / rate(kubelet_http_requests_duration_seconds_count[$rate_interval]) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "Avg (kubelet)", "refId": "B" } ], "title": "Avg Duration", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "#33FF00", "mode": "fixed" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Latency (ms)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [ { "matcher": { "id": "byName", "options": "P50" }, "properties": [ { "id": "color", "value": { "fixedColor": "#33FF00", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "P90" }, "properties": [ { "id": "color", "value": { "fixedColor": "#00A3FF", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "P95" }, "properties": [ { "id": "color", "value": { "fixedColor": "#00A3FF", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "P99" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFCC00", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 5 }, "id": 5, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "expr": "histogram_quantile(0.5, rate(http_request_duration_seconds_bucket[$rate_interval])) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "P50", "refId": "A" }, { "expr": "histogram_quantile(0.90, rate(http_request_duration_seconds_bucket[$rate_interval])) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "P90", "refId": "B" }, { "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[$rate_interval])) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "P95", "refId": "C" }, { "expr": "histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[$rate_interval])) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "P99", "refId": "D" } ], "title": "Latency Distribution Over Time", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-reds" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Count", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 100, "gradientMode": "scheme", "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 5 }, "id": 6, "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "expr": "increase(http_request_duration_seconds_bucket{le!=\"+Inf\"}[$rate_interval])", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{ le }}", "refId": "A" } ], "title": "Latency Heatmap", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "left", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#33FF00", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "P99 (ms)" }, "properties": [ { "id": "unit", "value": "ms" }, { "id": "custom.displayMode", "value": "color-background" }, { "id": "color", "value": { "mode": "thresholds" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "#33FF00", "value": null }, { "color": "#FFCC00", "value": 100 }, { "color": "#FF4444", "value": 500 } ] } } ] } ] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 15 }, "id": 7, "options": { "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "displayName": "P99 (ms)", "desc": true } ] }, "pluginVersion": "10.0.0", "targets": [ { "expr": "topk(10, histogram_quantile(0.99, rate(http_request_duration_seconds_bucket{job!=\"\"}[$rate_interval])) * 1000)", "format": "table", "instant": true, "intervalFactor": 2, "legendFormat": "{{ handler }}", "refId": "A" } ], "title": "Latency by Endpoint (Top 10)", "transparent": true, "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "job": false, "instance": false }, "indexByName": {}, "renameByName": { "Value": "P99 (ms)", "handler": "Endpoint", "job": "Service", "instance": "Instance" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "#FF4444", "mode": "fixed" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Latency (ms)", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#33FF00", "value": null }, { "color": "#FFCC00", "value": 200 }, { "color": "#FF4444", "value": 500 } ] }, "unit": "ms" }, "overrides": [ { "matcher": { "id": "byName", "options": "SLO Threshold" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FF4444", "mode": "fixed" } }, { "id": "custom.lineStyle", "value": { "dash": [ 10, 10 ] } }, { "id": "custom.lineWidth", "value": 2 } ] }, { "matcher": { "id": "byName", "options": "P99" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FFCC00", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 15 }, "id": 8, "options": { "legend": { "calcs": [ "mean", "max", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "expr": "histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[$rate_interval])) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "P99", "refId": "A" }, { "expr": "500", "format": "time_series", "intervalFactor": 2, "legendFormat": "SLO Threshold", "refId": "B" } ], "title": "SLO Alerting (P99 > 500ms)", "transparent": true, "type": "timeseries" } ], "refresh": "10s", "schemaVersion": 39, "style": "dark", "tags": [ "retrodash", "golden-signals", "latency" ], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus", "value": "prometheus" }, "hide": 0, "includeAll": false, "label": "datasource", "multi": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "datasource" }, { "name": "job_node", "type": "custom", "label": "Node Exporter Job", "current": { "text": "node-exporter", "value": "node-exporter" }, "options": [ { "text": "node-exporter", "value": "node-exporter", "selected": true }, { "text": "node", "value": "node", "selected": false } ], "query": "node-exporter,node" }, { "name": "rate_interval", "type": "custom", "label": "Rate Window", "current": { "text": "5m", "value": "5m" }, "options": [ { "text": "1m", "value": "1m", "selected": false }, { "text": "5m", "value": "5m", "selected": true }, { "text": "15m", "value": "15m", "selected": false }, { "text": "30m", "value": "30m", "selected": false } ], "query": "1m,5m,15m,30m" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "RetroDash // LATENCY", "uid": "retrodash-latency", "version": 1, "weekStart": "", "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "Prometheus datasource", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ] }