grafana: replace dashboard 13770 (Korean) with 6417 (English)
13770 'Kubernetes All-in-one Cluster Monitoring KR' has Korean panel titles. Swap in 6417 'Kubernetes Cluster (Prometheus)' which is fully English.
This commit is contained in:
parent
e1274bc63b
commit
a8e1f2fb73
3 changed files with 3131 additions and 2712 deletions
File diff suppressed because it is too large
Load diff
2635
ansible/roles/grafana/files/dashboards/k8s-cluster.json
Normal file
2635
ansible/roles/grafana/files/dashboards/k8s-cluster.json
Normal file
File diff suppressed because it is too large
Load diff
496
ansible/roles/grafana/files/dashboards/microwaveprop.json
Normal file
496
ansible/roles/grafana/files/dashboards/microwaveprop.json
Normal file
|
|
@ -0,0 +1,496 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {"type": "grafana", "uid": "-- Grafana --"},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "PromEx-exported metrics from microwaveprop (BEAM, Phoenix, Ecto, Microwaveprop.Instrument spans, Oban queue depth, ScoreCache/GridCache hit rates).",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"liveNow": false,
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["elixir", "phoenix", "promex", "microwaveprop"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {"selected": false, "text": "Prometheus", "value": "Prometheus"},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Datasource",
|
||||
"multi": false,
|
||||
"name": "datasource",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {"selected": true, "text": "All", "value": "$__all"},
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"definition": "label_values(microwaveprop_prom_ex_beam_stats_process_count, instance)",
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Pod",
|
||||
"multi": true,
|
||||
"name": "instance",
|
||||
"options": [],
|
||||
"query": {"query": "label_values(microwaveprop_prom_ex_beam_stats_process_count, instance)", "refId": "StandardVariableQuery"},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "Microwaveprop",
|
||||
"uid": "microwaveprop",
|
||||
"version": 1,
|
||||
"weekStart": "",
|
||||
"panels": [
|
||||
{
|
||||
"type": "row",
|
||||
"id": 100,
|
||||
"title": "Overview",
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"id": 1,
|
||||
"title": "Replicas Up",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 1},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "yellow", "value": 1}, {"color": "green", "value": 2}]}, "unit": "short"}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
||||
"targets": [{"expr": "count(up{job=\"prop\"} == 1)", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"id": 2,
|
||||
"title": "Uptime (Min Replica)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 1},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "yellow", "value": 60000}, {"color": "green", "value": 300000}]}, "unit": "ms"}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}, "textMode": "auto"},
|
||||
"targets": [{"expr": "min(microwaveprop_prom_ex_beam_stats_uptime_milliseconds_count{instance=~\"$instance\"})", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"id": 3,
|
||||
"title": "Memory (Total)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 1},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "bytes"}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}, "textMode": "auto"},
|
||||
"targets": [{"expr": "sum(microwaveprop_prom_ex_beam_memory_allocated_bytes{instance=~\"$instance\"})", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"id": 4,
|
||||
"title": "Process Count",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 12, "y": 1},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 5000}, {"color": "red", "value": 20000}]}, "unit": "short"}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [{"expr": "sum(microwaveprop_prom_ex_beam_stats_process_count{instance=~\"$instance\"})", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"id": 5,
|
||||
"title": "Run Queue",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 16, "y": 1},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 5}, {"color": "red", "value": 20}]}, "unit": "short"}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [{"expr": "max(microwaveprop_prom_ex_beam_stats_run_queue_count{instance=~\"$instance\"})", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"id": 6,
|
||||
"title": "Reductions/s",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 20, "y": 1},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [{"expr": "sum(rate(microwaveprop_prom_ex_beam_stats_reduction_count{instance=~\"$instance\"}[5m]))", "refId": "A"}]
|
||||
},
|
||||
|
||||
{"type": "row", "id": 101, "title": "Phoenix HTTP", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5}, "panels": []},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 10,
|
||||
"title": "Request Rate (req/s)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 6},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10, "lineWidth": 1}, "unit": "reqps"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "sum(rate(microwaveprop_prom_ex_phoenix_endpoint_stop_duration_milliseconds_count{instance=~\"$instance\"}[1m]))", "legendFormat": "total", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 11,
|
||||
"title": "Endpoint Latency (p50 / p95 / p99)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 6},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0, "lineWidth": 1}, "unit": "ms"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.50, sum by(le) (rate(microwaveprop_prom_ex_phoenix_endpoint_stop_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_prom_ex_phoenix_endpoint_stop_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.99, sum by(le) (rate(microwaveprop_prom_ex_phoenix_endpoint_stop_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 12,
|
||||
"title": "Top Routes by p95 Latency",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 14},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0, "lineWidth": 1}, "unit": "ms"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "topk(8, histogram_quantile(0.95, sum by(le, route) (rate(microwaveprop_prom_ex_phoenix_router_dispatch_stop_duration_milliseconds_bucket{instance=~\"$instance\"}[5m]))))", "legendFormat": "{{route}}", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 13,
|
||||
"title": "Top Routes by Request Rate",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 14},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10, "lineWidth": 1}, "unit": "reqps"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "topk(8, sum by(route) (rate(microwaveprop_prom_ex_phoenix_router_dispatch_stop_duration_milliseconds_count{instance=~\"$instance\"}[5m])))", "legendFormat": "{{route}}", "refId": "A"}
|
||||
]
|
||||
},
|
||||
|
||||
{"type": "row", "id": 102, "title": "BEAM Internals", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 22}, "panels": []},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 20,
|
||||
"title": "Memory by Type",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 23},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 30, "lineWidth": 1, "stacking": {"mode": "normal"}}, "unit": "bytes"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "sum(microwaveprop_prom_ex_beam_memory_processes_total_bytes{instance=~\"$instance\"})", "legendFormat": "processes", "refId": "A"},
|
||||
{"expr": "sum(microwaveprop_prom_ex_beam_memory_binary_total_bytes{instance=~\"$instance\"})", "legendFormat": "binary", "refId": "B"},
|
||||
{"expr": "sum(microwaveprop_prom_ex_beam_memory_ets_total_bytes{instance=~\"$instance\"})", "legendFormat": "ets", "refId": "C"},
|
||||
{"expr": "sum(microwaveprop_prom_ex_beam_memory_code_total_bytes{instance=~\"$instance\"})", "legendFormat": "code", "refId": "D"},
|
||||
{"expr": "sum(microwaveprop_prom_ex_beam_memory_atom_total_bytes{instance=~\"$instance\"})", "legendFormat": "atom", "refId": "E"},
|
||||
{"expr": "sum(microwaveprop_prom_ex_beam_memory_persistent_term_total_bytes{instance=~\"$instance\"})", "legendFormat": "persistent_term", "refId": "F"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 21,
|
||||
"title": "Process / Port / ETS / Atom Counts",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 23},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0, "lineWidth": 1}, "unit": "short"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "sum(microwaveprop_prom_ex_beam_stats_process_count{instance=~\"$instance\"})", "legendFormat": "processes", "refId": "A"},
|
||||
{"expr": "sum(microwaveprop_prom_ex_beam_stats_port_count{instance=~\"$instance\"})", "legendFormat": "ports", "refId": "B"},
|
||||
{"expr": "sum(microwaveprop_prom_ex_beam_stats_ets_count{instance=~\"$instance\"})", "legendFormat": "ets tables", "refId": "C"},
|
||||
{"expr": "sum(microwaveprop_prom_ex_beam_stats_atom_count{instance=~\"$instance\"})", "legendFormat": "atoms", "refId": "D"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 22,
|
||||
"title": "GC: Reclaimed Bytes/s + Count/s",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 31},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10, "lineWidth": 1}, "unit": "Bps"}, "overrides": [{"matcher": {"id": "byName", "options": "gc count/s"}, "properties": [{"id": "unit", "value": "short"}, {"id": "custom.axisPlacement", "value": "right"}]}]},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "sum(rate(microwaveprop_prom_ex_beam_stats_gc_reclaimed_bytes{instance=~\"$instance\"}[5m]))", "legendFormat": "reclaimed bytes/s", "refId": "A"},
|
||||
{"expr": "sum(rate(microwaveprop_prom_ex_beam_stats_gc_count{instance=~\"$instance\"}[5m]))", "legendFormat": "gc count/s", "refId": "B"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 23,
|
||||
"title": "Run Queue / Active Tasks / Context Switches",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 31},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0, "lineWidth": 1}, "unit": "short"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "max(microwaveprop_prom_ex_beam_stats_run_queue_count{instance=~\"$instance\"})", "legendFormat": "run queue", "refId": "A"},
|
||||
{"expr": "max(microwaveprop_prom_ex_beam_stats_active_task_count{instance=~\"$instance\"})", "legendFormat": "active tasks", "refId": "B"},
|
||||
{"expr": "sum(rate(microwaveprop_prom_ex_beam_stats_context_switch_count{instance=~\"$instance\"}[5m]))", "legendFormat": "context switches/s", "refId": "C"}
|
||||
]
|
||||
},
|
||||
|
||||
{"type": "row", "id": 103, "title": "Ecto / Postgres", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 39}, "panels": []},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 30,
|
||||
"title": "Query Rate (queries/s)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 40},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "ops"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "sum(rate(microwaveprop_prom_ex_ecto_repo_query_total_time_milliseconds_count{instance=~\"$instance\"}[5m]))", "legendFormat": "queries/s", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 31,
|
||||
"title": "Query Latency (p50 / p95 / p99)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 40},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.50, sum by(le) (rate(microwaveprop_prom_ex_ecto_repo_query_total_time_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_prom_ex_ecto_repo_query_total_time_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.99, sum by(le) (rate(microwaveprop_prom_ex_ecto_repo_query_total_time_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 32,
|
||||
"title": "Queue Time (waiting for connection) p95",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 48},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "ms"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.95, sum by(le, instance) (rate(microwaveprop_prom_ex_ecto_repo_query_queue_time_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "p95 — {{instance}}", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 33,
|
||||
"title": "Pool Size (configured)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 48},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "short"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "max(microwaveprop_prom_ex_ecto_repo_init_pool_size{instance=~\"$instance\"})", "legendFormat": "pool size", "refId": "A"}]
|
||||
},
|
||||
|
||||
{"type": "row", "id": 104, "title": "Upstream HTTP Clients (Microwaveprop.Instrument)", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 56}, "panels": []},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 40,
|
||||
"title": "Weather/Forecast Fetch p95",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 57},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_hrrr_fetch_grid_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "hrrr.fetch_grid", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_hrrr_fetch_profile_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "hrrr.fetch_profile", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_hrrr_fetch_idx_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "hrrr.fetch_idx", "refId": "C"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_hrrr_download_grib_ranges_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "hrrr.download_grib_ranges", "refId": "D"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_gefs_fetch_grid_profiles_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "gefs.fetch_grid_profiles", "refId": "E"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_narr_fetch_profile_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "narr.fetch_profile", "refId": "F"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_rtma_fetch_observation_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "rtma.fetch_observation", "refId": "G"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 41,
|
||||
"title": "Surface/Sounding/Radar Fetch p95",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 57},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_iem_fetch_asos_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "iem.fetch_asos", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_iem_fetch_raob_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "iem.fetch_raob", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_iem_fetch_iemre_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "iem.fetch_iemre", "refId": "C"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_uwyo_fetch_sounding_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "uwyo.fetch_sounding", "refId": "D"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_ncei_fetch_metar_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "ncei.fetch_metar", "refId": "E"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_nexrad_fetch_frame_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "nexrad.fetch_frame", "refId": "F"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_nexrad_decode_png_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "nexrad.decode_png", "refId": "G"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_mrms_list_latest_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "mrms.list_latest", "refId": "H"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_mrms_download_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "mrms.download", "refId": "I"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 42,
|
||||
"title": "Ancillary Service Fetch p95",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 65},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_solar_fetch_indices_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "solar.fetch_indices", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_swpc_fetch_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "swpc.fetch", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_giro_fetch_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "giro.fetch", "refId": "C"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_qrz_request_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "qrz.request", "refId": "D"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_geocoder_geocode_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "geocoder.geocode", "refId": "E"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_elevation_fetch_profile_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "elevation.fetch_profile", "refId": "F"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_srtm_download_tile_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "srtm.download_tile", "refId": "G"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 43,
|
||||
"title": "Upstream Call Rate (calls/s)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 65},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "ops"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "topk(10, sum by(__name__) (rate({__name__=~\"microwaveprop_(hrrr|iem|nexrad|gefs|narr|uwyo|elevation|mrms|rtma|ncei|solar|swpc|giro|qrz|geocoder|srtm)_.+_count\", instance=~\"$instance\"}[5m])))", "legendFormat": "{{__name__}}", "refId": "A"}
|
||||
]
|
||||
},
|
||||
|
||||
{"type": "row", "id": 105, "title": "Workers / Pipeline Phases", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 73}, "panels": []},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 50,
|
||||
"title": "Worker Phase Duration p95",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 74},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_propagation_grid_worker_perform_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "propagation.grid_worker.perform", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_propagation_grid_score_band_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "propagation_grid.score_band", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_terrain_analyse_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "terrain.analyse", "refId": "C"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_radar_aggregate_stats_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "radar.aggregate_stats", "refId": "D"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_worker_terrain_profile_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "worker.terrain_profile", "refId": "E"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_worker_mechanism_classify_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "worker.mechanism_classify", "refId": "F"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_worker_mrms_fetch_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "worker.mrms_fetch", "refId": "G"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 51,
|
||||
"title": "PropagationGridWorker — Tasks Seeded + Exceptions",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 74},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "bars", "fillOpacity": 50}, "unit": "short"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "max(microwaveprop_propagation_grid_worker_seeded_queued_tasks{instance=~\"$instance\"})", "legendFormat": "queued tasks (last seed)", "refId": "A"},
|
||||
{"expr": "sum(rate(microwaveprop_propagation_grid_worker_exception_count{instance=~\"$instance\"}[5m]))", "legendFormat": "exceptions/s", "refId": "B"}
|
||||
]
|
||||
},
|
||||
|
||||
{"type": "row", "id": 106, "title": "LiveView Hot Path", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 82}, "panels": []},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 60,
|
||||
"title": "Map LiveView Calls p95",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 83},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_propagation_scores_at_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "scores_at", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_propagation_point_forecast_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "point_forecast", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_propagation_point_detail_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "point_detail", "refId": "C"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 61,
|
||||
"title": "Cache Hit Ratio (last 5m)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 83},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "percentunit", "min": 0, "max": 1}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "sum(rate(microwaveprop_propagation_scores_at_cache_count{hit=\"true\", instance=~\"$instance\"}[5m])) / clamp_min(sum(rate(microwaveprop_propagation_scores_at_cache_count{instance=~\"$instance\"}[5m])), 1e-9)", "legendFormat": "ScoreCache", "refId": "A"},
|
||||
{"expr": "sum(rate(microwaveprop_weather_grid_cache_lookup_count{hit=\"true\", instance=~\"$instance\"}[5m])) / clamp_min(sum(rate(microwaveprop_weather_grid_cache_lookup_count{instance=~\"$instance\"}[5m])), 1e-9)", "legendFormat": "Weather GridCache", "refId": "B"}
|
||||
]
|
||||
},
|
||||
|
||||
{"type": "row", "id": 107, "title": "DB Batches + Subprocesses", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 91}, "panels": []},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 70,
|
||||
"title": "DB Upsert Batch Duration p95",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 92},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_db_upsert_hrrr_profiles_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "upsert_hrrr_profiles", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_db_upsert_gefs_profiles_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "upsert_gefs_profiles", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_db_replace_scores_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "replace_scores", "refId": "C"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 71,
|
||||
"title": "wgrib2 Subprocess Duration p95",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 92},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "ms"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_wgrib2_extract_grid_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "extract_grid", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_wgrib2_extract_grid_from_file_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "extract_grid_from_file", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(microwaveprop_wgrib2_extract_grid_from_file_mapped_duration_milliseconds_bucket{instance=~\"$instance\"}[5m])))", "legendFormat": "extract_grid_from_file_mapped", "refId": "C"}
|
||||
]
|
||||
},
|
||||
|
||||
{"type": "row", "id": 108, "title": "Oban Queue Depth (sampled every 30s)", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 100}, "panels": []},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 80,
|
||||
"title": "Available Jobs by Queue",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 101},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10, "stacking": {"mode": "normal"}}, "unit": "short"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "max by(queue) (microwaveprop_oban_queue_count{state=\"available\", instance=~\"$instance\"})", "legendFormat": "{{queue}}", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 81,
|
||||
"title": "Executing Jobs by Queue",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 101},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10, "stacking": {"mode": "normal"}}, "unit": "short"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "max by(queue) (microwaveprop_oban_queue_count{state=\"executing\", instance=~\"$instance\"})", "legendFormat": "{{queue}}", "refId": "A"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 82,
|
||||
"title": "Retryable + Discarded Jobs by Queue",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 109},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "short"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "max by(queue, state) (microwaveprop_oban_queue_count{state=~\"retryable|discarded|cancelled\", instance=~\"$instance\"})", "legendFormat": "{{queue}} / {{state}}", "refId": "A"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue