prom: add app-specific grafana dashboards + per-pod job override

Dashboards: aprs-me, towerops-{application,beam,ecto,oban,phoenix,phoenix-live-view}.

prometheus.yml.j2: kubernetes-pods job now honors a `prometheus.io/job`
pod annotation so apps can claim a named job label (e.g. towerops)
instead of bucketing under job=kubernetes-pods.

ansible/roles/grafana/files/dashboards/aprs-me.json

@@ -0,0 +1,394 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Phoenix/Elixir application metrics for aprs.me — Phoenix HTTP, Ecto, BEAM VM, APRS packet pipeline, Spatial PubSub, and PostgreSQL.",
+  "editable": true,
+  "graphTooltip": 1,
+  "tags": ["aprs", "elixir", "phoenix", "prom_ex"],
+  "templating": {
+    "list": [
+      {
+        "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "datasource": "$datasource",
+        "definition": "label_values(aprsme_prom_ex_beam_stats_uptime_milliseconds_count, pod)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Pod",
+        "multi": true,
+        "name": "pod",
+        "query": { "query": "label_values(aprsme_prom_ex_beam_stats_uptime_milliseconds_count, pod)", "refId": "StandardVariableQuery" },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": { "from": "now-1h", "to": "now" },
+  "timepicker": {},
+  "timezone": "",
+  "title": "APRS.me — Application Metrics",
+  "uid": "aprs-me-app",
+  "schemaVersion": 38,
+  "version": 1,
+  "refresh": "30s",
+  "panels": [
+    {
+      "type": "row", "id": 100, "title": "Overview", "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "stat", "id": 1, "title": "Uptime", "datasource": "$datasource",
+      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
+      "fieldConfig": { "defaults": { "unit": "ms", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 60000 }] } } },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "graphMode": "area", "colorMode": "value" },
+      "targets": [{ "expr": "max(aprsme_prom_ex_beam_stats_uptime_milliseconds_count{pod=~\"$pod\"})", "refId": "A" }]
+    },
+    {
+      "type": "stat", "id": 2, "title": "Active pods", "datasource": "$datasource",
+      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
+      "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } } },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "value" },
+      "targets": [{ "expr": "count(count by (pod)(aprsme_prom_ex_beam_stats_uptime_milliseconds_count))", "refId": "A" }]
+    },
+    {
+      "type": "stat", "id": 3, "title": "HTTP req/s", "datasource": "$datasource",
+      "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
+      "fieldConfig": { "defaults": { "unit": "reqps" } },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "graphMode": "area", "colorMode": "value" },
+      "targets": [{ "expr": "sum(rate(aprsme_prom_ex_phoenix_http_requests_total{pod=~\"$pod\"}[5m]))", "refId": "A" }]
+    },
+    {
+      "type": "stat", "id": 4, "title": "HTTP error rate (5xx)", "datasource": "$datasource",
+      "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
+      "fieldConfig": { "defaults": { "unit": "percentunit", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 }] } } },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "graphMode": "area", "colorMode": "background" },
+      "targets": [{ "expr": "sum(rate(aprsme_prom_ex_phoenix_http_requests_total{status=~\"5..\",pod=~\"$pod\"}[5m])) / clamp_min(sum(rate(aprsme_prom_ex_phoenix_http_requests_total{pod=~\"$pod\"}[5m])), 0.0001)", "refId": "A" }]
+    },
+    {
+      "type": "stat", "id": 5, "title": "p99 latency", "datasource": "$datasource",
+      "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
+      "fieldConfig": { "defaults": { "unit": "ms" } },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "graphMode": "area", "colorMode": "value" },
+      "targets": [{ "expr": "histogram_quantile(0.99, sum by (le)(rate(aprsme_prom_ex_phoenix_http_request_duration_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "refId": "A" }]
+    },
+    {
+      "type": "stat", "id": 6, "title": "Packets/s", "datasource": "$datasource",
+      "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
+      "fieldConfig": { "defaults": { "unit": "short" } },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "graphMode": "area", "colorMode": "value" },
+      "targets": [{ "expr": "sum(rate(aprsme_packet_pipeline_batch_count{pod=~\"$pod\"}[5m]))", "refId": "A" }]
+    },
+
+    {
+      "type": "row", "id": 110, "title": "Phoenix HTTP", "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }
+    },
+    {
+      "type": "timeseries", "id": 11, "title": "Request rate by status", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+      "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineInterpolation": "linear", "fillOpacity": 10, "stacking": { "mode": "normal" } } } },
+      "options": { "tooltip": { "mode": "multi" }, "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [{ "expr": "sum by (status) (rate(aprsme_prom_ex_phoenix_http_requests_total{pod=~\"$pod\"}[1m]))", "legendFormat": "{{status}}", "refId": "A" }]
+    },
+    {
+      "type": "timeseries", "id": 12, "title": "Latency percentiles", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+      "fieldConfig": { "defaults": { "unit": "ms", "custom": { "drawStyle": "line", "fillOpacity": 5 } } },
+      "options": { "tooltip": { "mode": "multi" }, "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "histogram_quantile(0.50, sum by (le)(rate(aprsme_prom_ex_phoenix_http_request_duration_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "p50", "refId": "A" },
+        { "expr": "histogram_quantile(0.95, sum by (le)(rate(aprsme_prom_ex_phoenix_http_request_duration_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "p95", "refId": "B" },
+        { "expr": "histogram_quantile(0.99, sum by (le)(rate(aprsme_prom_ex_phoenix_http_request_duration_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "p99", "refId": "C" }
+      ]
+    },
+    {
+      "type": "timeseries", "id": 13, "title": "Top 10 routes by request rate", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 },
+      "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "fillOpacity": 5 } } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [{ "expr": "topk(10, sum by (route) (rate(aprsme_prom_ex_phoenix_http_requests_total{pod=~\"$pod\"}[5m])))", "legendFormat": "{{route}}", "refId": "A" }]
+    },
+    {
+      "type": "timeseries", "id": 14, "title": "Response size (p50/p95)", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 },
+      "fieldConfig": { "defaults": { "unit": "bytes" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "histogram_quantile(0.50, sum by (le)(rate(aprsme_prom_ex_phoenix_http_response_size_bytes_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "p50", "refId": "A" },
+        { "expr": "histogram_quantile(0.95, sum by (le)(rate(aprsme_prom_ex_phoenix_http_response_size_bytes_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "p95", "refId": "B" }
+      ]
+    },
+
+    {
+      "type": "row", "id": 120, "title": "BEAM VM", "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }
+    },
+    {
+      "type": "timeseries", "id": 21, "title": "Memory by type", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 },
+      "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "fillOpacity": 10, "stacking": { "mode": "normal" } } } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "sum(aprsme_prom_ex_beam_memory_processes_total_bytes{pod=~\"$pod\"})", "legendFormat": "processes", "refId": "A" },
+        { "expr": "sum(aprsme_prom_ex_beam_memory_binary_total_bytes{pod=~\"$pod\"})", "legendFormat": "binary", "refId": "B" },
+        { "expr": "sum(aprsme_prom_ex_beam_memory_ets_total_bytes{pod=~\"$pod\"})", "legendFormat": "ets", "refId": "C" },
+        { "expr": "sum(aprsme_prom_ex_beam_memory_atom_total_bytes{pod=~\"$pod\"})", "legendFormat": "atom", "refId": "D" },
+        { "expr": "sum(aprsme_prom_ex_beam_memory_code_total_bytes{pod=~\"$pod\"})", "legendFormat": "code", "refId": "E" }
+      ]
+    },
+    {
+      "type": "timeseries", "id": 22, "title": "Process / port / ETS counts", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 },
+      "fieldConfig": { "defaults": { "unit": "short" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "sum(aprsme_prom_ex_beam_stats_process_count{pod=~\"$pod\"})", "legendFormat": "processes", "refId": "A" },
+        { "expr": "sum(aprsme_prom_ex_beam_stats_port_count{pod=~\"$pod\"})", "legendFormat": "ports", "refId": "B" },
+        { "expr": "sum(aprsme_prom_ex_beam_stats_ets_count{pod=~\"$pod\"})", "legendFormat": "ETS tables", "refId": "C" },
+        { "expr": "sum(aprsme_prom_ex_beam_stats_atom_count{pod=~\"$pod\"})", "legendFormat": "atoms", "refId": "D" }
+      ]
+    },
+    {
+      "type": "timeseries", "id": 23, "title": "Run queue lengths", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 31 },
+      "fieldConfig": { "defaults": { "unit": "short" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [{ "expr": "sum(aprsme_prom_ex_beam_stats_run_queue_count{pod=~\"$pod\"}) by (pod)", "legendFormat": "{{pod}}", "refId": "A" }]
+    },
+    {
+      "type": "timeseries", "id": 24, "title": "Reductions / GC / context switches per second", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 31 },
+      "fieldConfig": { "defaults": { "unit": "ops" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "sum(rate(aprsme_prom_ex_beam_stats_reduction_count{pod=~\"$pod\"}[1m]))", "legendFormat": "reductions/s", "refId": "A" },
+        { "expr": "sum(rate(aprsme_prom_ex_beam_stats_gc_count{pod=~\"$pod\"}[1m]))", "legendFormat": "GC/s", "refId": "B" },
+        { "expr": "sum(rate(aprsme_prom_ex_beam_stats_context_switch_count{pod=~\"$pod\"}[1m]))", "legendFormat": "ctx switches/s", "refId": "C" }
+      ]
+    },
+
+    {
+      "type": "row", "id": 130, "title": "Ecto / Database", "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 }
+    },
+    {
+      "type": "timeseries", "id": 31, "title": "Query rate", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
+      "fieldConfig": { "defaults": { "unit": "ops" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [{ "expr": "sum by (source) (rate(aprsme_prom_ex_ecto_repo_query_total_time_milliseconds_count{pod=~\"$pod\"}[5m]))", "legendFormat": "{{source}}", "refId": "A" }]
+    },
+    {
+      "type": "timeseries", "id": 32, "title": "Query latency p50/p95/p99", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
+      "fieldConfig": { "defaults": { "unit": "ms" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "histogram_quantile(0.50, sum by (le)(rate(aprsme_prom_ex_ecto_repo_query_total_time_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "p50", "refId": "A" },
+        { "expr": "histogram_quantile(0.95, sum by (le)(rate(aprsme_prom_ex_ecto_repo_query_total_time_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "p95", "refId": "B" },
+        { "expr": "histogram_quantile(0.99, sum by (le)(rate(aprsme_prom_ex_ecto_repo_query_total_time_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "p99", "refId": "C" }
+      ]
+    },
+    {
+      "type": "timeseries", "id": 33, "title": "Connection pool", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 },
+      "fieldConfig": { "defaults": { "unit": "short" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "sum(aprsme_repo_pool_size{pod=~\"$pod\"})", "legendFormat": "size", "refId": "A" },
+        { "expr": "sum(aprsme_repo_pool_busy{pod=~\"$pod\"})", "legendFormat": "busy", "refId": "B" },
+        { "expr": "sum(aprsme_repo_pool_idle{pod=~\"$pod\"})", "legendFormat": "idle", "refId": "C" },
+        { "expr": "sum(aprsme_repo_pool_queue_length{pod=~\"$pod\"})", "legendFormat": "queued", "refId": "D" }
+      ]
+    },
+    {
+      "type": "timeseries", "id": 34, "title": "Queue / decode / idle time", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 48 },
+      "fieldConfig": { "defaults": { "unit": "ms" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "histogram_quantile(0.95, sum by (le)(rate(aprsme_prom_ex_ecto_repo_query_queue_time_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "queue p95", "refId": "A" },
+        { "expr": "histogram_quantile(0.95, sum by (le)(rate(aprsme_prom_ex_ecto_repo_query_decode_time_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "decode p95", "refId": "B" },
+        { "expr": "histogram_quantile(0.95, sum by (le)(rate(aprsme_prom_ex_ecto_repo_query_idle_time_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "idle p95", "refId": "C" }
+      ]
+    },
+
+    {
+      "type": "row", "id": 140, "title": "APRS Packet Pipeline", "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 56 }
+    },
+    {
+      "type": "timeseries", "id": 41, "title": "Pipeline throughput", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 57 },
+      "fieldConfig": { "defaults": { "unit": "ops" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "sum(rate(aprsme_packet_pipeline_batch_count{pod=~\"$pod\"}[1m]))", "legendFormat": "packets/s", "refId": "A" },
+        { "expr": "sum(rate(aprsme_packet_pipeline_batch_success{pod=~\"$pod\"}[1m]))", "legendFormat": "success/s", "refId": "B" },
+        { "expr": "sum(rate(aprsme_packet_pipeline_batch_error{pod=~\"$pod\"}[1m]))", "legendFormat": "errors/s", "refId": "C" }
+      ]
+    },
+    {
+      "type": "timeseries", "id": 42, "title": "Batch insert duration p50/p95/p99", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 57 },
+      "fieldConfig": { "defaults": { "unit": "ms" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "histogram_quantile(0.50, sum by (le)(rate(aprsme_packet_pipeline_batch_duration_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "p50", "refId": "A" },
+        { "expr": "histogram_quantile(0.95, sum by (le)(rate(aprsme_packet_pipeline_batch_duration_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "p95", "refId": "B" },
+        { "expr": "histogram_quantile(0.99, sum by (le)(rate(aprsme_packet_pipeline_batch_duration_milliseconds_bucket{pod=~\"$pod\"}[5m])))", "legendFormat": "p99", "refId": "C" }
+      ]
+    },
+    {
+      "type": "stat", "id": 43, "title": "Total packets processed", "datasource": "$datasource",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 65 },
+      "fieldConfig": { "defaults": { "unit": "short" } },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "graphMode": "area" },
+      "targets": [{ "expr": "sum(aprsme_packet_pipeline_batch_count)", "refId": "A" }]
+    },
+    {
+      "type": "stat", "id": 44, "title": "Total errors", "datasource": "$datasource",
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 65 },
+      "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 100 }, { "color": "red", "value": 1000 }] } } },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" },
+      "targets": [{ "expr": "sum(aprsme_packet_pipeline_batch_error)", "refId": "A" }]
+    },
+    {
+      "type": "stat", "id": 45, "title": "Success ratio", "datasource": "$datasource",
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 65 },
+      "fieldConfig": { "defaults": { "unit": "percentunit", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.95 }, { "color": "green", "value": 0.99 }] } } },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value" },
+      "targets": [{ "expr": "sum(rate(aprsme_packet_pipeline_batch_success[5m])) / clamp_min(sum(rate(aprsme_packet_pipeline_batch_count[5m])), 0.0001)", "refId": "A" }]
+    },
+    {
+      "type": "stat", "id": 46, "title": "Backpressure events", "datasource": "$datasource",
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 65 },
+      "fieldConfig": { "defaults": { "unit": "short" } },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "graphMode": "area" },
+      "targets": [{ "expr": "sum(increase(aprsme_packet_producer_backpressure_count[5m]))", "refId": "A" }]
+    },
+
+    {
+      "type": "row", "id": 150, "title": "Spatial PubSub", "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 69 }
+    },
+    {
+      "type": "timeseries", "id": 51, "title": "Connected clients", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 70 },
+      "fieldConfig": { "defaults": { "unit": "short" } },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom" } },
+      "targets": [{ "expr": "sum(aprsme_spatial_pubsub_clients_count{pod=~\"$pod\"})", "legendFormat": "clients", "refId": "A" }]
+    },
+    {
+      "type": "timeseries", "id": 52, "title": "Active grid cells / avg clients per cell", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 70 },
+      "fieldConfig": { "defaults": { "unit": "short" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "sum(aprsme_spatial_pubsub_clients_grid_cells{pod=~\"$pod\"})", "legendFormat": "grid cells", "refId": "A" },
+        { "expr": "avg(aprsme_spatial_pubsub_clients_avg_clients_per_cell{pod=~\"$pod\"})", "legendFormat": "avg clients/cell", "refId": "B" }
+      ]
+    },
+    {
+      "type": "timeseries", "id": 53, "title": "Broadcast efficiency ratio", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 70 },
+      "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1 } },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom" } },
+      "targets": [{ "expr": "avg(aprsme_spatial_pubsub_efficiency_ratio{pod=~\"$pod\"})", "legendFormat": "efficiency", "refId": "A" }]
+    },
+    {
+      "type": "timeseries", "id": 54, "title": "Broadcast rates", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 78 },
+      "fieldConfig": { "defaults": { "unit": "ops" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "sum(rate(aprsme_spatial_pubsub_broadcasts_total{pod=~\"$pod\"}[1m]))", "legendFormat": "sent/s", "refId": "A" },
+        { "expr": "sum(rate(aprsme_spatial_pubsub_broadcasts_filtered{pod=~\"$pod\"}[1m]))", "legendFormat": "filtered/s", "refId": "B" },
+        { "expr": "sum(rate(aprsme_spatial_pubsub_broadcasts_packets{pod=~\"$pod\"}[1m]))", "legendFormat": "packets/s", "refId": "C" }
+      ]
+    },
+    {
+      "type": "timeseries", "id": 55, "title": "Saved broadcasts (cumulative rate)", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 78 },
+      "fieldConfig": { "defaults": { "unit": "ops" } },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom" } },
+      "targets": [{ "expr": "sum(rate(aprsme_spatial_pubsub_efficiency_saved_broadcasts{pod=~\"$pod\"}[5m]))", "legendFormat": "saved/s", "refId": "A" }]
+    },
+
+    {
+      "type": "row", "id": 160, "title": "PostgreSQL", "collapsed": false,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 86 }
+    },
+    {
+      "type": "timeseries", "id": 61, "title": "Postgres connections", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 87 },
+      "fieldConfig": { "defaults": { "unit": "short" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "max(aprsme_postgres_connections_total)", "legendFormat": "total", "refId": "A" },
+        { "expr": "max(aprsme_postgres_connections_active)", "legendFormat": "active", "refId": "B" },
+        { "expr": "max(aprsme_postgres_connections_idle)", "legendFormat": "idle", "refId": "C" },
+        { "expr": "max(aprsme_postgres_connections_idle_in_transaction)", "legendFormat": "idle in tx", "refId": "D" },
+        { "expr": "max(aprsme_postgres_connections_waiting)", "legendFormat": "waiting", "refId": "E" }
+      ]
+    },
+    {
+      "type": "timeseries", "id": 62, "title": "Packets table tuples (live vs dead)", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 87 },
+      "fieldConfig": { "defaults": { "unit": "short" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "max(aprsme_postgres_packets_table_live_tuples)", "legendFormat": "live", "refId": "A" },
+        { "expr": "max(aprsme_postgres_packets_table_dead_tuples)", "legendFormat": "dead", "refId": "B" }
+      ]
+    },
+    {
+      "type": "timeseries", "id": 63, "title": "Database / table / index sizes", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 95 },
+      "fieldConfig": { "defaults": { "unit": "bytes" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "max(aprsme_postgres_database_size_bytes)", "legendFormat": "DB total", "refId": "A" },
+        { "expr": "max(aprsme_postgres_packets_table_table_size_bytes)", "legendFormat": "packets table", "refId": "B" },
+        { "expr": "max(aprsme_postgres_packets_table_indexes_size_bytes)", "legendFormat": "packets indexes", "refId": "C" }
+      ]
+    },
+    {
+      "type": "timeseries", "id": 64, "title": "Insert / update / delete rate", "datasource": "$datasource",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 95 },
+      "fieldConfig": { "defaults": { "unit": "ops" } },
+      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "lastNotNull"] } },
+      "targets": [
+        { "expr": "sum(rate(aprsme_postgres_packets_table_total_inserts[5m]))", "legendFormat": "inserts/s", "refId": "A" },
+        { "expr": "sum(rate(aprsme_postgres_packets_table_total_updates[5m]))", "legendFormat": "updates/s", "refId": "B" },
+        { "expr": "sum(rate(aprsme_postgres_packets_table_total_deletes[5m]))", "legendFormat": "deletes/s", "refId": "C" }
+      ]
+    }
+  ]
+}

ansible/roles/grafana/files/dashboards/towerops-application.json

@@ -0,0 +1,633 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      },
+      {
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "#73BF69",
+        "limit": 100,
+        "name": "PromEx service start",
+        "showIn": 0,
+        "tags": [
+          "prom_ex",
+          "towerops",
+          "start"
+        ],
+        "type": "tags"
+      },
+      {
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "#FF9830",
+        "limit": 100,
+        "name": "PromEx service stop",
+        "showIn": 0,
+        "tags": [
+          "prom_ex",
+          "towerops",
+          "stop"
+        ],
+        "type": "tags"
+      }
+    ]
+  },
+  "description": "All the data that is presented here is captured by the PromEx Application plugin (https://github.com/akoutmos/prom_ex/blob/master/lib/prom_ex/plugins/application.ex)",
+  "editable": false,
+  "gnetId": null,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [
+    {
+      "asDropdown": false,
+      "icon": "bolt",
+      "includeVars": false,
+      "keepTime": false,
+      "tags": [],
+      "targetBlank": true,
+      "title": "Sponsor PromEx",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://github.com/sponsors/akoutmos"
+    },
+    {
+      "asDropdown": false,
+      "icon": "doc",
+      "includeVars": false,
+      "keepTime": false,
+      "tags": [],
+      "targetBlank": true,
+      "title": "Application Plugin Docs",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://hexdocs.pm/prom_ex/PromEx.Plugins.Application.html"
+    }
+  ],
+  "panels": [
+    {
+      "datasource": "Prometheus",
+      "description": "The amount of time that the application has been running.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "decimals": 1,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "dtdurationms"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "id": 6,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.1.3",
+      "targets": [
+        {
+          "expr": "towerops_prom_ex_application_uptime_milliseconds_count{job=\"$job\", instance=\"$instance\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Application Uptime",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "description": "The data is populated by the PromEx Application plugin and provides information regarding your application's dependencies.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "left",
+            "displayMode": "auto"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Status"
+            },
+            "properties": [
+              {
+                "id": "custom.displayMode",
+                "value": "color-background"
+              },
+              {
+                "id": "mappings",
+                "value": [
+                  {
+                    "from": "",
+                    "id": 0,
+                    "text": "Started",
+                    "to": "",
+                    "type": 1,
+                    "value": "1"
+                  },
+                  {
+                    "from": "",
+                    "id": 1,
+                    "text": "Loaded",
+                    "to": "",
+                    "type": 1,
+                    "value": "0"
+                  }
+                ]
+              },
+              {
+                "id": "custom.align",
+                "value": "center"
+              },
+              {
+                "id": "custom.width",
+                "value": 202
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Name"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 349
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Version"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 187
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 36,
+        "w": 16,
+        "x": 8,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": false,
+            "displayName": "Name"
+          }
+        ]
+      },
+      "pluginVersion": "7.1.3",
+      "targets": [
+        {
+          "expr": "towerops_prom_ex_application_dependency_info{job=\"$job\", instance=\"$instance\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Dependency Information",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "__name__": true,
+              "instance": true,
+              "job": true,
+              "Value": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "Value": 4,
+              "__name__": 1,
+              "instance": 2,
+              "job": 3,
+              "modules": 7,
+              "name": 5,
+              "version": 6
+            },
+            "renameByName": {
+              "Value": "Status",
+              "modules": "Number of Modules Loaded",
+              "name": "Name",
+              "version": "Version"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": "Prometheus",
+      "description": "The name of the primary application that is running.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 6
+      },
+      "id": 11,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "/^name$/",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.1.3",
+      "targets": [
+        {
+          "expr": "towerops_prom_ex_application_primary_info{job=\"$job\", instance=\"$instance\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Application Name",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "description": "The Git SHA of the application.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 12
+      },
+      "id": 10,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "/^sha$/",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.1.3",
+      "targets": [
+        {
+          "expr": "towerops_prom_ex_application_git_sha_info{job=\"$job\", instance=\"$instance\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Application Git SHA",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "description": "The author of the application's last Git commit.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 18
+      },
+      "id": 12,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "/^author$/",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.1.3",
+      "targets": [
+        {
+          "expr": "towerops_prom_ex_application_git_author_info{job=\"$job\", instance=\"$instance\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Application Git Author",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "description": "The version of the primary application that is running.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 24
+      },
+      "id": 7,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "/^version$/",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.1.3",
+      "targets": [
+        {
+          "expr": "towerops_prom_ex_application_primary_info{job=\"$job\", instance=\"$instance\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Application Version",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "description": "The number of modules loaded by the primary application that is running.",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 30
+      },
+      "id": 9,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "/^modules$/",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.1.3",
+      "targets": [
+        {
+          "expr": "towerops_prom_ex_application_primary_info{job=\"$job\", instance=\"$instance\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Application Modules Loaded",
+      "type": "stat"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 26,
+  "style": "dark",
+  "tags": ["PromEx", "Application", "towerops"],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "datasource": "Prometheus",
+        "definition": "label_values(towerops_prom_ex_status_info, job)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Prometheus Job",
+        "multi": false,
+        "name": "job",
+        "options": [],
+        "query": "label_values(towerops_prom_ex_status_info, job)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 6,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "datasource": "Prometheus",
+        "definition": "label_values(towerops_prom_ex_status_info, instance)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Application Instance",
+        "multi": false,
+        "name": "instance",
+        "options": [],
+        "query": "label_values(towerops_prom_ex_status_info{job=\"$job\"}, instance)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m"
+    ]
+  },
+  "timezone": "",
+  "title": "Towerops · Application",
+  "uid": "towerops-application",
+  "version": 1
+}

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -116,6 +116,12 @@ scrape_configs:
       - source_labels: [__meta_kubernetes_pod_phase]
         action: keep
         regex: Running
+      # Per-pod job override via `prometheus.io/job` annotation. Apps that want
+      # a named target (e.g. towerops) set this; otherwise pods stay under
+      # job=kubernetes-pods.
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_job]
+        regex: (.+)
+        target_label: job
       - target_label: __address__
         replacement: {{ prometheus_k8s_api_server | regex_replace('^https?://', '') }}
       - source_labels: