From 0970e29364d0c1c8f8b862fbce69d6d35b027c88 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Fri, 8 May 2026 11:09:14 -0500 Subject: [PATCH] ansible: fix two latent bugs surfaced by repo audit - grafana role: empty grafana_admin_password by default, and only emit admin_password into grafana.ini when the var is set. Previously the default 'admin' was rewritten on every run, clobbering passwords rotated via the Grafana UI. - bootstrap.yml: probe for sudo state before attempting su-as-root. The first play was using become_method=su unconditionally; on hosts where graham already has passwordless sudo (or where -K isn't passed), ansible blocked indefinitely waiting for a root password prompt. Now the install-via-su task only runs when sudo is actually missing, and the python3 task uses sudo when available. Also bring in latest microwaveprop dashboard tweaks. --- ansible/bootstrap.yml | 24 +++- ansible/roles/grafana/defaults/main.yml | 7 +- .../files/dashboards/microwaveprop.json | 130 ++++++++++++++++++ .../roles/grafana/templates/grafana.ini.j2 | 2 + 4 files changed, 158 insertions(+), 5 deletions(-) diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml index 4346645..41167fa 100644 --- a/ansible/bootstrap.yml +++ b/ansible/bootstrap.yml @@ -7,11 +7,26 @@ vars: ansible_user: graham tasks: - - name: Ensure sudo is present + # Probe before attempting privilege escalation. If sudo already works for + # graham, the install-via-su tasks below would otherwise block on a root + # password prompt that the Makefile's `-K` is designed to feed — and on a + # passwordless-sudo host with no `-K` provided, hang indefinitely. + - name: Probe sudo state raw: | if command -v sudo >/dev/null 2>&1; then - exit 0 + if sudo -n true 2>/dev/null; then + echo "sudo-passwordless" + else + echo "sudo-needs-password" + fi + else + echo "sudo-missing" fi + register: sudo_state + changed_when: false + + - name: Ensure sudo is present + raw: | if command -v apt-get >/dev/null 2>&1; then apt-get update && apt-get install -y sudo elif command -v dnf >/dev/null 2>&1; then @@ -27,6 +42,7 @@ become: yes become_method: su become_user: root + when: "'sudo-missing' in sudo_state.stdout" - name: Ensure Python 3 is present raw: | @@ -47,7 +63,9 @@ fi changed_when: false become: yes - become_method: su + # Once sudo exists (newly installed or pre-existing) prefer it; only fall + # back to `su` if sudo really isn't there. + become_method: "{{ 'su' if 'sudo-missing' in sudo_state.stdout else 'sudo' }}" become_user: root - name: Bootstrap new host - Base configuration diff --git a/ansible/roles/grafana/defaults/main.yml b/ansible/roles/grafana/defaults/main.yml index 050cba3..3fc315b 100644 --- a/ansible/roles/grafana/defaults/main.yml +++ b/ansible/roles/grafana/defaults/main.yml @@ -8,8 +8,11 @@ grafana_http_port: 3000 grafana_root_url: "http://10.0.15.31:3000/" grafana_admin_user: admin -# Set in vault / host_vars; default kept for first-boot only. -grafana_admin_password: admin +# Empty by default so re-running ansible doesn't clobber a password that was +# rotated via the Grafana UI (or `grafana-cli admin reset-admin-password`). +# To bootstrap or enforce a specific password, set this in host_vars / vault; +# whatever you set will be written into grafana.ini on every subsequent run. +grafana_admin_password: "" # On-disk location for provisioned dashboard JSON files. Grafana watches this # path via the file-based dashboard provider configured in diff --git a/ansible/roles/grafana/files/dashboards/microwaveprop.json b/ansible/roles/grafana/files/dashboards/microwaveprop.json index 7020172..07f1f0d 100644 --- a/ansible/roles/grafana/files/dashboards/microwaveprop.json +++ b/ansible/roles/grafana/files/dashboards/microwaveprop.json @@ -491,6 +491,136 @@ "targets": [ {"expr": "max by(queue, state) (microwaveprop_oban_queue_count{state=~\"retryable|discarded|cancelled\", instance=~\"$instance\"})", "legendFormat": "{{queue}} / {{state}}", "refId": "A"} ] + }, + + {"type": "row", "id": 109, "title": "Rust Workers (prop-grid-rs + hrrr-point-rs)", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 117}, "panels": []}, + { + "type": "stat", + "id": 90, + "title": "prop-grid-rs Replicas", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 118}, + "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "yellow", "value": 1}, {"color": "green", "value": 2}]}, "unit": "short"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [{"expr": "count(up{job=\"prop-grid-rs\"} == 1)", "refId": "A"}] + }, + { + "type": "stat", + "id": 91, + "title": "hrrr-point-rs Replicas", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 118}, + "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "yellow", "value": 1}, {"color": "green", "value": 2}]}, "unit": "short"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [{"expr": "count(up{job=\"hrrr-point-rs\"} == 1)", "refId": "A"}] + }, + { + "type": "stat", + "id": 92, + "title": "Tasks In-Flight (sum)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 118}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "short"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [{"expr": "sum(prop_grid_rs_tasks_in_flight)", "refId": "A"}] + }, + { + "type": "stat", + "id": 93, + "title": "Chain Step Success Rate (5m)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 4, "w": 4, "x": 12, "y": 118}, + "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "yellow", "value": 0.9}, {"color": "green", "value": 0.99}]}, "unit": "percentunit", "min": 0, "max": 1}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [{"expr": "sum(rate(prop_grid_rs_chain_steps_total{outcome=\"success\"}[5m])) / clamp_min(sum(rate(prop_grid_rs_chain_steps_total[5m])), 1e-9)", "refId": "A"}] + }, + { + "type": "stat", + "id": 94, + "title": "Point Batch Success Rate (5m)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 4, "w": 4, "x": 16, "y": 118}, + "fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "yellow", "value": 0.9}, {"color": "green", "value": 0.99}]}, "unit": "percentunit", "min": 0, "max": 1}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [{"expr": "sum(rate(prop_grid_rs_point_batches_total{outcome=\"success\"}[5m])) / clamp_min(sum(rate(prop_grid_rs_point_batches_total[5m])), 1e-9)", "refId": "A"}] + }, + { + "type": "stat", + "id": 95, + "title": "Points Inserted/s", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 4, "w": 4, "x": 20, "y": 118}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "short"}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}}, + "targets": [{"expr": "sum(rate(prop_grid_rs_points_processed_total{kind=\"inserted\"}[5m]))", "refId": "A"}] + }, + { + "type": "timeseries", + "id": 96, + "title": "Chain Step Duration p50 / p95 / p99 (prop-grid-rs)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 122}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "s"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by(le, outcome) (rate(prop_grid_rs_chain_step_duration_seconds_bucket[5m])))", "legendFormat": "p50 — {{outcome}}", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by(le, outcome) (rate(prop_grid_rs_chain_step_duration_seconds_bucket[5m])))", "legendFormat": "p95 — {{outcome}}", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by(le, outcome) (rate(prop_grid_rs_chain_step_duration_seconds_bucket[5m])))", "legendFormat": "p99 — {{outcome}}", "refId": "C"} + ] + }, + { + "type": "timeseries", + "id": 97, + "title": "wgrib2 Decode Duration (prop-grid-rs)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 122}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "s"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by(le) (rate(prop_grid_rs_decode_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by(le) (rate(prop_grid_rs_decode_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by(le) (rate(prop_grid_rs_decode_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"} + ] + }, + { + "type": "timeseries", + "id": 98, + "title": "Point Batch Duration p50 / p95 / p99 (hrrr-point-rs)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 130}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "s"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by(le, outcome) (rate(prop_grid_rs_point_batch_duration_seconds_bucket[5m])))", "legendFormat": "p50 — {{outcome}}", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by(le, outcome) (rate(prop_grid_rs_point_batch_duration_seconds_bucket[5m])))", "legendFormat": "p95 — {{outcome}}", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by(le, outcome) (rate(prop_grid_rs_point_batch_duration_seconds_bucket[5m])))", "legendFormat": "p99 — {{outcome}}", "refId": "C"} + ] + }, + { + "type": "timeseries", + "id": 99, + "title": "Points Requested vs Inserted (hrrr-point-rs)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 130}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "short"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "sum(rate(prop_grid_rs_points_processed_total{kind=\"requested\"}[5m]))", "legendFormat": "requested/s", "refId": "A"}, + {"expr": "sum(rate(prop_grid_rs_points_processed_total{kind=\"inserted\"}[5m]))", "legendFormat": "inserted/s", "refId": "B"} + ] + }, + { + "type": "timeseries", + "id": 110, + "title": "Chain + Point Batch Throughput (per outcome)", + "datasource": {"type": "prometheus", "uid": "${datasource}"}, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 138}, + "fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "bars", "fillOpacity": 60, "stacking": {"mode": "normal"}}, "unit": "ops"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "sum by(outcome) (rate(prop_grid_rs_chain_steps_total[5m]))", "legendFormat": "grid-rs chain — {{outcome}}", "refId": "A"}, + {"expr": "sum by(outcome) (rate(prop_grid_rs_point_batches_total[5m]))", "legendFormat": "hrrr-point-rs batch — {{outcome}}", "refId": "B"} + ] } ] } diff --git a/ansible/roles/grafana/templates/grafana.ini.j2 b/ansible/roles/grafana/templates/grafana.ini.j2 index 71485d1..d0557d6 100644 --- a/ansible/roles/grafana/templates/grafana.ini.j2 +++ b/ansible/roles/grafana/templates/grafana.ini.j2 @@ -6,7 +6,9 @@ root_url = {{ grafana_root_url }} [security] admin_user = {{ grafana_admin_user }} +{% if grafana_admin_password %} admin_password = {{ grafana_admin_password }} +{% endif %} [analytics] reporting_enabled = false