{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "Prometheus data source scraping codex-pooler", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "11.0.0" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } ], "timezone": "browser", "editable": true, "graphTooltip": 1, "fiscalYearStartMonth": 0, "schemaVersion": 41, "templating": { "list": [ { "type": "query", "name": "cluster", "skipUrlSync": false, "multi": false, "allowCustomValue": true, "includeAll": false, "auto": false, "auto_min": "10s", "auto_count": 30, "label": "Cluster", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "query": "label_values(kube_node_info, cluster)", "refresh": 1 }, { "type": "query", "name": "namespace", "skipUrlSync": false, "multi": false, "allowCustomValue": true, "includeAll": false, "auto": false, "auto_min": "10s", "auto_count": 30, "label": "Namespace", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "query": "label_values(up{job=\"codex-pooler-app\"}, namespace)", "refresh": 1 }, { "type": "query", "name": "pod", "skipUrlSync": false, "multi": true, "allowCustomValue": true, "includeAll": true, "auto": false, "auto_min": "10s", "auto_count": 30, "label": "Pod", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "query": "label_values(up{job=\"codex-pooler-app\", namespace=\"$namespace\"}, pod)", "refresh": 1 } ] }, "annotations": {}, "title": "Codex Pooler / Runtime Triage", "description": "Starter triage view for Codex Pooler runtime health: memory pressure, gateway admission, HTTP traffic, database query pressure, and gateway outcomes.", "tags": [ "codex-pooler", "kubernetes" ], "refresh": "1m", "time": { "from": "now-6h", "to": "now" }, "panels": [ { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Websocket Control Path Failure Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Observed websocket callback failures and deferred cleanup, including initialization before request reservation. These are server observations, not proof of a TCP reset or a delivered close frame. Investigate any sustained database_error or process_exit increase; cleanup_deferred means supervised cleanup outlived the socket's close budget.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (phase, reason) (rate(codex_pooler_gateway_websocket_control_failure_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{phase}} {{reason}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 } }, { "type": "row", "collapsed": false, "id": 0, "panels": [], "title": "Triage Signals", "gridPos": { "x": 0, "y": 8, "h": 1, "w": 24 } }, { "type": "stat", "transparent": false, "repeatDirection": "h", "title": "Scraped App Pods", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, "targets": [ { "expr": "sum(up{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"})", "refId": "", "legendFormat": "scraped pods", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 4, "w": 4, "x": 0, "y": 9 } }, { "type": "stat", "transparent": false, "repeatDirection": "h", "title": "Max Pod Memory / Limit", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] }, "targets": [ { "expr": "max((max by (pod) (container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"}) / max by (pod) (kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"${pod:regex}\", resource=\"memory\", exported_container=\"app\"})))", "refId": "", "legendFormat": "max pod usage / limit", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 4, "w": 5, "x": 4, "y": 9 } }, { "type": "stat", "transparent": false, "repeatDirection": "h", "title": "Restarts In 30m", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, "targets": [ { "expr": "sum(increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"${pod:regex}\", exported_container=\"app\"}[30m]))", "refId": "", "legendFormat": "restarts", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 4, "w": 5, "x": 9, "y": 9 } }, { "type": "stat", "transparent": false, "repeatDirection": "h", "title": "OOM Events In 30m", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, "targets": [ { "expr": "sum by () (max by (pod, name) (increase(container_oom_events_total{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\"}[30m])))", "refId": "", "legendFormat": "oom events", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 4, "w": 5, "x": 14, "y": 9 } }, { "type": "stat", "transparent": false, "repeatDirection": "h", "title": "BEAM Binary Share", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] }, "targets": [ { "expr": "max(vm_memory_binary_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"} / vm_memory_total_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"})", "refId": "", "legendFormat": "binary / total", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 4, "w": 5, "x": 19, "y": 9 } }, { "type": "row", "collapsed": false, "id": 0, "panels": [], "title": "Memory", "gridPos": { "x": 0, "y": 13, "h": 1, "w": 24 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Container Working Set vs Limit", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"})", "refId": "", "legendFormat": "{{pod}} working set", "range": true, "instant": false, "editorMode": "code" }, { "expr": "max by (pod) (kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"${pod:regex}\", resource=\"memory\", exported_container=\"app\"})", "refId": "", "legendFormat": "{{pod}} limit", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "BEAM Memory Breakdown", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "vm_memory_total_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} total", "range": true, "instant": false, "editorMode": "code" }, { "expr": "vm_memory_processes_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} processes", "range": true, "instant": false, "editorMode": "code" }, { "expr": "vm_memory_binary_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} binary", "range": true, "instant": false, "editorMode": "code" }, { "expr": "vm_memory_ets_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} ets", "range": true, "instant": false, "editorMode": "code" }, { "expr": "vm_memory_code_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} code", "range": true, "instant": false, "editorMode": "code" }, { "expr": "vm_memory_atom_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} atom", "range": true, "instant": false, "editorMode": "code" }, { "expr": "vm_memory_atom_used_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} atom used", "range": true, "instant": false, "editorMode": "code" }, { "expr": "vm_memory_system_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} system", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "BEAM Process and Port Counts", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "short", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "vm_system_counts_process_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} processes", "range": true, "instant": false, "editorMode": "code" }, { "expr": "vm_system_counts_port_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} ports", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 12, "x": 0, "y": 22 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "BEAM Run Queue", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total, CPU, and IO scheduler queues. Sudden growth usually means the VM is saturated or blocked on external work.", "fieldConfig": { "defaults": { "unit": "none", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "vm_total_run_queue_lengths_total{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} total", "range": true, "instant": false, "editorMode": "code" }, { "expr": "vm_total_run_queue_lengths_cpu{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} cpu", "range": true, "instant": false, "editorMode": "code" }, { "expr": "vm_total_run_queue_lengths_io{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} io", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 12, "x": 12, "y": 22 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Cgroup Minus BEAM", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Container working set not accounted for by BEAM total memory. Use this to separate BEAM heap/binary growth from native allocator, libc, NIF, or cgroup accounting pressure.", "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "clamp_min(max by (pod) (container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"}) - on (pod) max by (pod) (vm_memory_total_bytes{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}), 0)", "refId": "", "legendFormat": "{{pod}} cgroup - beam", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 0, "y": 29 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Container RSS / Cache", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Cgroup memory categories from cAdvisor. RSS rising with flat BEAM memory points outside normal BEAM heap attribution.", "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "max by (pod) (container_memory_rss{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"})", "refId": "", "legendFormat": "{{pod}} rss", "range": true, "instant": false, "editorMode": "code" }, { "expr": "max by (pod) (container_memory_cache{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"})", "refId": "", "legendFormat": "{{pod}} cache", "range": true, "instant": false, "editorMode": "code" }, { "expr": "max by (pod) (container_memory_mapped_file{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"})", "refId": "", "legendFormat": "{{pod}} mapped file", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 8, "y": 29 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "CPU Throttling Ratio", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "CPU throttling can stretch request lifetimes and queues, increasing memory retained by in-flight requests.", "fieldConfig": { "defaults": { "unit": "percentunit", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (pod) (rate(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"}[$__rate_interval])) / clamp_min(sum by (pod) (rate(container_cpu_cfs_periods_total{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"}[$__rate_interval])), 0.001)", "refId": "", "legendFormat": "{{pod}} throttled", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 16, "y": 29 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Workload Memory by Component", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "App, worker, and scheduler cgroup memory in one view. App Prometheus metrics are scraped from the app service; worker/scheduler visibility here comes from Kubernetes cgroup metrics.", "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "max by (pod, container) (container_memory_working_set_bytes{namespace=\"$namespace\", pod=~\"codex-pooler-(app|oban-worker|oban-scheduler).*\", container=~\"app|oban-worker|oban-scheduler\", image!=\"\"})", "refId": "", "legendFormat": "{{pod}} {{container}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 12, "x": 0, "y": 36 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "CPU Usage", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "App container CPU usage in cores. Correlate CPU spikes with request rate, DB query pressure, run queue, and admission queueing.", "fieldConfig": { "defaults": { "unit": "cores", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\", image!=\"\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{pod}} cpu", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 12, "x": 12, "y": 36 } }, { "type": "row", "collapsed": false, "id": 0, "panels": [], "title": "Gateway Pressure", "gridPos": { "x": 0, "y": 43, "h": 1, "w": 24 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Request Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (pod) (rate(phoenix_endpoint_stop_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{pod}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 0, "y": 44 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Endpoint Duration p95", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "s", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le, pod) (rate(phoenix_endpoint_stop_duration_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "refId": "", "legendFormat": "{{pod}} p95", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 8, "y": 44 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Slowest Routes p95", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Top active routes by p95 latency. The active-route filter avoids stale NaN/empty legend rows when a route has no observations in the current interval.", "fieldConfig": { "defaults": { "unit": "s", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "topk(8, histogram_quantile(0.95, sum by (le, route) (rate(phoenix_router_dispatch_stop_duration_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))) and on (route) (sum by (route) (rate(phoenix_router_dispatch_stop_duration_seconds_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])) > 0))", "refId": "", "legendFormat": "{{route}} p95", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 16, "y": 44 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Stream Buffer Release Events", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (buffer, transport, route_class, endpoint) (rate(codex_pooler_gateway_stream_buffer_oversized_count{namespace=\"$namespace\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "oversized {{transport}} {{route_class}} {{endpoint}}", "range": true, "instant": false, "editorMode": "code" }, { "expr": "sum by (buffer, transport, route_class, endpoint) (rate(codex_pooler_gateway_stream_buffer_truncated_count{namespace=\"$namespace\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "truncated {{transport}} {{route_class}} {{endpoint}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 12, "x": 0, "y": 51 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Stream Buffer Size p95", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "p95 observed oversized and truncated stream-buffer payload size. Oversized observations saturate at 128 MiB for histogram export, so a value at that ceiling means 128 MiB or larger. The saturation changes telemetry only, not stream-buffer handling or the raw telemetry event.", "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le, buffer, transport, route_class, endpoint) (rate(codex_pooler_gateway_stream_buffer_oversized_bytes_bucket{namespace=\"$namespace\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "refId": "", "legendFormat": "oversized p95 {{transport}} {{route_class}} {{endpoint}}", "range": true, "instant": false, "editorMode": "code" }, { "expr": "histogram_quantile(0.95, sum by (le, buffer, transport, route_class, endpoint) (rate(codex_pooler_gateway_stream_buffer_truncated_bytes_bucket{namespace=\"$namespace\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "refId": "", "legendFormat": "truncated p95 {{transport}} {{route_class}} {{endpoint}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 12, "x": 12, "y": 51 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "HTTP Route Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Low-cardinality request rate by route, method, and status class. Use this to separate route-specific traffic growth from error storms.", "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (route, method, status_class) (rate(codex_pooler_http_route_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{route}} {{method}} {{status_class}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 0, "y": 58 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Admission Events", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Local gateway bulkhead pressure by route class and transport. Queue/reject/timeout spikes show admission backpressure before memory climbs.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (route_class, transport) (rate(codex_pooler_gateway_admission_accepted_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "accepted {{route_class}} {{transport}}", "range": true, "instant": false, "editorMode": "code" }, { "expr": "sum by (route_class, transport) (rate(codex_pooler_gateway_admission_enqueued_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "enqueued {{route_class}} {{transport}}", "range": true, "instant": false, "editorMode": "code" }, { "expr": "sum by (route_class, transport) (rate(codex_pooler_gateway_admission_dequeued_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "dequeued {{route_class}} {{transport}}", "range": true, "instant": false, "editorMode": "code" }, { "expr": "sum by (route_class, transport) (rate(codex_pooler_gateway_admission_rejected_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "rejected {{route_class}} {{transport}}", "range": true, "instant": false, "editorMode": "code" }, { "expr": "sum by (route_class, transport) (rate(codex_pooler_gateway_admission_timeout_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "timeout {{route_class}} {{transport}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 8, "y": 58 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Admission Queue Time p95", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Time requests spent waiting in local admission queues before execution or timeout.", "fieldConfig": { "defaults": { "unit": "s", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le, route_class, transport) (rate(codex_pooler_gateway_admission_dequeued_time_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "refId": "", "legendFormat": "dequeued {{route_class}} {{transport}} p95", "range": true, "instant": false, "editorMode": "code" }, { "expr": "histogram_quantile(0.95, sum by (le, route_class, transport) (rate(codex_pooler_gateway_admission_timeout_time_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "refId": "", "legendFormat": "timeout {{route_class}} {{transport}} p95", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 16, "y": 58 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Admission Saturation", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Current per-pod local admission state. Sum only after selecting the pods you mean to inspect: each app pod reports its own running and queued work by route class.", "fieldConfig": { "defaults": { "unit": "short", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "codex_pooler_gateway_admission_running{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} {{route_class}} running", "range": true, "instant": false, "editorMode": "code" }, { "expr": "codex_pooler_gateway_admission_queued{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}", "refId": "", "legendFormat": "{{pod}} {{route_class}} queued", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 0, "y": 65 } }, { "type": "row", "collapsed": false, "id": 0, "panels": [], "title": "Database / Ecto Query Pressure", "gridPos": { "x": 0, "y": 72, "h": 1, "w": 24 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Repo Query Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (pod) (rate(codex_pooler_repo_query_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{pod}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 0, "y": 73 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Queries / Request", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "short", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (pod) (rate(codex_pooler_repo_query_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])) / clamp_min(sum by (pod) (rate(phoenix_endpoint_stop_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])), 0.001)", "refId": "", "legendFormat": "{{pod}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 8, "y": 73 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Hot Query Sources", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Top Ecto query sources by safe source and SQL command. Source falls back to a sanitized table identifier for raw SQL when available.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "topk(10, sum by (source, command) (rate(codex_pooler_repo_query_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "refId": "", "legendFormat": "{{source}} {{command}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 16, "y": 73 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Repo Total Time p95", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "s", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le, source, command) (rate(codex_pooler_repo_query_total_time_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "refId": "", "legendFormat": "{{source}} {{command}} p95", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 0, "y": 80 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "DB Execution Time p95", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "s", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le, source, command) (rate(codex_pooler_repo_query_query_time_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "refId": "", "legendFormat": "{{source}} {{command}} p95", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 8, "y": 80 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "DB Queue Time p95", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "s", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le, pod) (rate(codex_pooler_repo_query_queue_time_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "refId": "", "legendFormat": "{{pod}} p95", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 16, "y": 80 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Unknown Query Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Source-less queries that could not be mapped to a safe Ecto source or table identifier. Use this to spot unclassified raw SQL or telemetry source-attribution gaps.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (pod, command) (rate(codex_pooler_repo_query_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\", source=\"unknown\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{pod}} {{command}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 0, "y": 87 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Query Command Mix", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "SQL command mix across repository queries. Useful for spotting unexpected write or transaction churn during incidents.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (command) (rate(codex_pooler_repo_query_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{command}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 8, "y": 87 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Repo Queue Time by Source p95", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Database connection checkout pressure by safe source and command.", "fieldConfig": { "defaults": { "unit": "s", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le, source, command) (rate(codex_pooler_repo_query_queue_time_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "refId": "", "legendFormat": "{{source}} {{command}} p95", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 16, "y": 87 } }, { "type": "row", "collapsed": false, "id": 0, "panels": [], "title": "Restart Evidence", "gridPos": { "x": 0, "y": 94, "h": 1, "w": 24 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Restarts and OOM Events", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "short", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"${pod:regex}\", exported_container=\"app\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{pod}} restarts", "range": true, "instant": false, "editorMode": "code" }, { "expr": "sum by (pod) (max by (pod, name) (increase(container_oom_events_total{namespace=\"$namespace\", pod=~\"${pod:regex}\", container=\"app\"}[$__rate_interval])))", "refId": "", "legendFormat": "{{pod}} oom events", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 12, "x": 0, "y": 95 } }, { "type": "table", "transparent": false, "repeatDirection": "h", "title": "Current Pods Last Terminated Reason", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Current kube-state-metrics lastState.terminated reason for live pods only. Deleted pod history is covered by restart and OOM panels, not this table.", "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, "targets": [ { "expr": "kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"${pod:regex}\", exported_container=\"app\"} == 1", "refId": "", "legendFormat": "{{pod}} {{reason}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 12, "x": 12, "y": 95 } }, { "type": "row", "collapsed": false, "id": 0, "panels": [], "title": "Admin Stats Dashboard", "gridPos": { "x": 0, "y": 102, "h": 1, "w": 24 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Admin Stats Reload Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Admin stats LiveView reload scheduling, coalescing, cancellation, and execution rate by window and scope. Coalesced spikes mean a page is already waiting on one debounced reload.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (stage, window, scope) (rate(codex_pooler_admin_stats_reload_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{stage}} {{window}} {{scope}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 0, "y": 103 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Admin Stats Dashboard Build Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Full admin stats dashboard builds by outcome, selected time window, and scope. This shows the reload work that can drive read-model DB pressure from open admin pages.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (outcome, window, scope) (rate(codex_pooler_admin_stats_dashboard_build_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{outcome}} {{window}} {{scope}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 8, "y": 103 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Admin Stats Dashboard Build p95", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "p95 duration for full admin stats dashboard builds. Pair spikes here with hot query source and DB queue panels to identify expensive stats reload paths.", "fieldConfig": { "defaults": { "unit": "s", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le, outcome, window, scope) (rate(codex_pooler_admin_stats_dashboard_build_duration_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "refId": "", "legendFormat": "{{outcome}} {{window}} {{scope}} p95", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 8, "x": 16, "y": 103 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Admin Request Logs Reload Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Admin request-log LiveView reload rate by bounded stage and scope. Use this to separate first load, filter patch, and event-refresh pressure without tagging pool ids, models, request ids, users, paths, params, or API key prefixes.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (stage, scope) (rate(codex_pooler_admin_request_logs_reload_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{stage}} {{scope}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 12, "x": 0, "y": 110 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Admin Request Logs Reload p95", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "p95 admin request-log reload duration by bounded stage and scope. Pair spikes with DB queue, hot safe query source, and pg_stat_statements evidence when validating request-log read-path remediation.", "fieldConfig": { "defaults": { "unit": "s", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le, stage, scope) (rate(codex_pooler_admin_request_logs_reload_duration_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval])))", "refId": "", "legendFormat": "{{stage}} {{scope}} p95", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 7, "w": 12, "x": 12, "y": 110 } }, { "type": "row", "collapsed": false, "id": 0, "panels": [], "title": "Gateway Outcomes", "gridPos": { "x": 0, "y": 117, "h": 1, "w": 24 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Stream Usage Completeness Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Successful stream finalization rate. Compare usage_unknown against usage_known without request, account, model, or payload labels.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (usage_status, usage_source, downstream_transport, upstream_transport) (rate(codex_pooler_gateway_stream_finalization_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{usage_status}} {{usage_source}} {{downstream_transport}} -> {{upstream_transport}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 0, "y": 118 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Stream Outcome Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "First-settlement stream outcomes by bounded outcome and downstream or upstream transport. The metric has at most 45 series per app pod. This panel is the native in-process share (via=in_process) while the job relay remains shadowed. Only web-role pods run the Prometheus reporter (it is disabled for OBAN_MODE=worker and scheduler), so the turns that expired-owner recovery settles from the runtime cleanup job are never exported here. A low or empty interrupted series means interruptions settled by that job are not measured, not that they did not happen; read the accounting request and attempt rows instead.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (outcome, downstream_transport, upstream_transport) (rate(codex_pooler_gateway_stream_outcome_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\", via=\"in_process\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{outcome}} {{downstream_transport}} -> {{upstream_transport}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 12, "y": 118 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Pre-attempt Reservation Releases", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Reservations released with no attempt row, by bounded pre-attempt phase. This panel is the native in-process share (via=in_process) while the job relay remains shadowed. routing_rejected is a deliberate refusal before dispatch and is expected traffic. turn_interrupted is a live turn interrupted before any attempt existed (client disconnect, owner drain or crash, expired lease); the release ledger entry's release_reason says which. task_exception is the pre-dispatch path raising before it created an attempt, so a sustained rate there is a defect to chase rather than traffic to explain. unrecorded means the releasing path declared no boundary; no release path in the application writes it any more, so a rising unrecorded count means a new release site reached the ledger without declaring one. stale_sweep is the six-hour backstop closing a reservation nothing live ever reached, and is the series to act on: a sustained rate means pre-attempt turns are being abandoned and only the backstop is noticing, six hours late. The stale_sweep series is only emitted where the sweeper runs beside the Prometheus reporter (OBAN_MODE=all); on split web/worker deployments read it from the release ledger's pre_attempt_phase detail instead.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (phase, transport) (rate(codex_pooler_accounting_reservation_pre_attempt_release_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\", via=\"in_process\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{phase}} {{transport}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 0, "y": 127 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Websocket Bridge Fallback Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Pre-content bridge fallbacks by bounded reason. There are at most 20 reason series per app pod; precommit buffer overflows remain a separate untagged counter.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (reason) (rate(codex_pooler_gateway_websocket_bridge_fallback_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{reason}}", "range": true, "instant": false, "editorMode": "code" }, { "expr": "sum(rate(codex_pooler_gateway_websocket_bridge_precommit_overflow_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "precommit overflow", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 12, "y": 127 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Quota Cycle Decision Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Quota cycle refresh, confirmation, candidate, rejection, and superseded-primary rejection rate. This panel is the native in-process share (via=in_process) while the job relay remains shadowed. Repeated candidate decisions indicate a convergence loop. Only web-role pods run the Prometheus reporter (it is disabled for OBAN_MODE=worker and scheduler), so the decisions taken by account reconciliation, saved-reset redemption and alert evaluation are never exported here. An empty or low graph means those roles are not measured, not that no cycle decision was taken; read the account quota window rows and their evidence history instead.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (scope, decision, source) (rate(codex_pooler_quota_cycle_decision_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\", via=\"in_process\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{scope}} {{decision}} {{source}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 0, "y": 136 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Duplicate Turn Refusals", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Client-visible 409 duplicate_turn refusals, by bounded refusing stage and transport class. runtime_replay_preflight is the replay preflight judging a resend against its predecessor's lifecycle; owner_replay_preflight is the owner-forwarded preflight refusing a resend of the turn the session's owner still holds (a new turn refused by a busy owner gets the owner's own refusal and is not counted); websocket_turn_claim and native_http_turn_claim refuse a second reservation of a turn already claimed; native_replay_dispatch, client_retry_claim and compaction_retry_claim refuse a replay or retry whose binding or claim no longer holds. None of them writes a request row for the refused request, so request logs, admin stats and every database-derived error count miss them: this panel is where they are counted. The reason is in the matching replay rejection log line; its vocabulary is open, so it is not a label.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (stage, transport) (rate(codex_pooler_gateway_duplicate_turn_refused_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{stage}} {{transport}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 12, "y": 136 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Native Compaction Admission Clears", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Native compaction admissions ended on serving nodes, by bounded clear reason, the stage the admission was in, and the direct or owner-forwarded topology. Every successful native websocket turn arms an admission for a possible in-band compaction (stage armed). On direct, connection_closed on armed is the ordinary end of a socket and tracks websocket volume; on forwarded the owner keeps its admission past the socket, so a closing socket ends it as downstream_detached and connection_closed on armed means the provider closed the upstream connection. A completed in-band compaction ends no admission: the final request's own success re-arms it for the next compaction (stage armed), so no reason counts completed compactions; final_success is in the reason vocabulary but no serving path clears with it. connection_closed on compacting or finalizing is the upstream connection closing under the admission: a compaction already collected is still delivered and a final still to be sent runs as an ordinary turn, while one cut in flight fails. Any other reason on the compacting or finalizing stage is an in-band compaction that did not complete: compact_failure and final_failure are unsuccessful upstream exchanges, owner_drained, upstream_exited, stale_owner and handoff_timeout are owner lifecycle events cutting it short, downstream_detached and downstream_cancelled are the client leaving, and request_rejected is only the runtime's explicit clear after it rejected the compaction request. A clear on a connection that never held an admission is not counted. No Pool, API-key, session, request or lifecycle identifier is exported.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (reason, stage, topology) (rate(codex_pooler_gateway_native_compaction_admission_clear_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{reason}} {{stage}} {{topology}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 0, "y": 145 } }, { "type": "row", "collapsed": false, "id": 0, "panels": [], "title": "Saved-reset Post-consume Convergence", "gridPos": { "x": 0, "y": 154, "h": 1, "w": 24 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Saved-reset Convergence Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Committed post-consume transitions observed by scraped web nodes only. This panel is the native in-process share (via=in_process) while the job relay remains shadowed. Bounded source and outcome labels contain no identity, pool, account, request, or payload identifiers; this best-effort view is not a cluster-wide persisted-state inventory. Only web-role pods run the Prometheus reporter (it is disabled for OBAN_MODE=worker and scheduler), so transitions committed by the saved-reset redemption and account reconciliation workers are never exported here. An empty graph means this is not measured on those roles, not that convergence never happened; confirm worker-side transitions from the upstream identity saved_reset_redemption lifecycle metadata instead.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (source, outcome) (rate(codex_pooler_saved_reset_convergence_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\", via=\"in_process\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{source}} {{outcome}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 0, "y": 155 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Saved-reset Convergence p95", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Best-effort web-node timing for committed transitions. This panel is the native in-process share (via=in_process) while the job relay remains shadowed. Labels stay bounded to source and outcome; missing or invalid timing is omitted and does not indicate cross-role persistence latency. Only web-role pods run the Prometheus reporter (it is disabled for OBAN_MODE=worker and scheduler), so transitions committed by the saved-reset redemption and account reconciliation workers are never exported here. An empty graph means this is not measured on those roles, not that convergence never happened; confirm worker-side transitions from the upstream identity saved_reset_redemption lifecycle metadata instead.", "fieldConfig": { "defaults": { "unit": "s", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le, source, outcome) (rate(codex_pooler_saved_reset_convergence_applied_to_canonical_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\", via=\"in_process\"}[$__rate_interval])))", "refId": "", "legendFormat": "{{source}} {{outcome}} applied to canonical", "range": true, "instant": false, "editorMode": "code" }, { "expr": "histogram_quantile(0.95, sum by (le, source, outcome) (rate(codex_pooler_saved_reset_convergence_canonical_to_lifecycle_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\", via=\"in_process\"}[$__rate_interval])))", "refId": "", "legendFormat": "{{source}} {{outcome}} canonical to lifecycle", "range": true, "instant": false, "editorMode": "code" }, { "expr": "histogram_quantile(0.95, sum by (le, source, outcome) (rate(codex_pooler_saved_reset_convergence_applied_to_lifecycle_seconds_bucket{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\", via=\"in_process\"}[$__rate_interval])))", "refId": "", "legendFormat": "{{source}} {{outcome}} applied to lifecycle", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 12, "y": 155 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Routing Circuit Transitions", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Routing circuit transition rate by bounded transition, route class, and reason class. Compare open, half-open, and closed transitions.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (transition, route_class, reason_class) (rate(codex_pooler_gateway_routing_circuit_transition_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{transition}} {{route_class}} {{reason_class}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 0, "y": 164 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Affinity Stale Writes", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Affinity writes refused by the updated_at fence, by pod, bounded operation, and affinity kind. The fence orders writers on the event clock each writer stamps from its own node clock, so it is correct and a low cluster-wide rate is ordinary reordering between nodes. The pod label names the writer that was refused, not the writer that caused it: a sustained rate concentrated on one pod means that pod's clock lags its peers and it loses every affinity write it attempts, while a sustained rate spread across every pod but one means the remaining pod's clock runs ahead and its stamps refuse everyone else's writes. Either way the affected pods stop contributing prompt-cache steering while serving traffic normally; routing, retries, settlement, and accounting are unaffected. Compare the pods' clocks against each other.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (pod, operation, affinity_kind) (rate(codex_pooler_gateway_routing_affinity_stale_write_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{pod}} {{operation}} {{affinity_kind}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 12, "y": 164 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Ingress Firewall Denial Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Ingress firewall denials by bounded scope and reason. These denials happen before authenticated request accounting.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum by (scope, reason) (rate(codex_pooler_ingress_firewall_denied_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "{{scope}} {{reason}}", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 0, "y": 173 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Instance Heartbeat Write Failure Rate", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Failed instance-presence heartbeat writes on scraped pods. InstanceHeartbeat runs on every release role, but OBAN_MODE=worker and scheduler have no Prometheus reporter; this graph shows only the web/all share. An empty graph does not prove those roles are healthy. Check instance_presences.last_seen_at for durable presence freshness on every role.", "fieldConfig": { "defaults": { "unit": "ops", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "sum(rate(codex_pooler_instance_presence_heartbeat_failure_count{namespace=\"$namespace\", job=\"codex-pooler-app\", pod=~\"${pod:regex}\"}[$__rate_interval]))", "refId": "", "legendFormat": "failed writes", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 9, "w": 12, "x": 12, "y": 173 } }, { "type": "row", "collapsed": false, "id": 0, "panels": [], "title": "Job Telemetry Relay", "gridPos": { "x": 0, "y": 182, "h": 1, "w": 24 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Relay Unclaimed Backlog", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Shared unclaimed rows and samples, including expired backlog. Values are replicated observations: max avoids counting each app observer again. job_relay remains shadowed; in_process panels stay authoritative.", "fieldConfig": { "defaults": { "unit": "short", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "max(codex_pooler_telemetry_relay_backlog_rows{job=\"codex-pooler-app\", namespace=\"$namespace\", pod=~\"${pod:regex}\"})", "refId": "", "legendFormat": "rows", "range": true, "instant": false, "editorMode": "code" }, { "expr": "max(codex_pooler_telemetry_relay_backlog_samples{job=\"codex-pooler-app\", namespace=\"$namespace\", pod=~\"${pod:regex}\"})", "refId": "", "legendFormat": "samples", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 8, "w": 12, "x": 0, "y": 183 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Relay Fresh Consumers", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Non-quiesced consumers with a heartbeat in the last 60 seconds. This is consumer health, not the writer database-access insert gate. A missing series means no scraped observer, not zero backlog.", "fieldConfig": { "defaults": { "unit": "short", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "max(codex_pooler_telemetry_relay_consumers_fresh{job=\"codex-pooler-app\", namespace=\"$namespace\", pod=~\"${pod:regex}\"})", "refId": "", "legendFormat": "fresh consumers", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 8, "w": 12, "x": 12, "y": 183 } }, { "type": "timeseries", "transparent": false, "repeatDirection": "h", "title": "Relay Known Lost Samples", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Durable cumulative loss totals by fixed reason. max avoids duplicate observer counts. Includes expired unclaimed, buffer overflow and shutdown unflushed samples when persistence succeeds. Hard kills, unavailable loss persistence and post-claim/pre-scrape loss remain unquantified.", "fieldConfig": { "defaults": { "unit": "short", "custom": { "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "asc" } }, "targets": [ { "expr": "max by (reason) (codex_pooler_telemetry_relay_loss_samples{job=\"codex-pooler-app\", namespace=\"$namespace\", pod=~\"${pod:regex}\"})", "refId": "", "legendFormat": "{{reason}} samples", "range": true, "instant": false, "editorMode": "code" }, { "expr": "max by (reason) (codex_pooler_telemetry_relay_loss_rows{job=\"codex-pooler-app\", namespace=\"$namespace\", pod=~\"${pod:regex}\"})", "refId": "", "legendFormat": "{{reason}} rows", "range": true, "instant": false, "editorMode": "code" } ], "gridPos": { "h": 8, "w": 12, "x": 0, "y": 191 } } ] }