ansible: fix two latent bugs surfaced by repo audit
- grafana role: empty grafana_admin_password by default, and only emit admin_password into grafana.ini when the var is set. Previously the default 'admin' was rewritten on every run, clobbering passwords rotated via the Grafana UI. - bootstrap.yml: probe for sudo state before attempting su-as-root. The first play was using become_method=su unconditionally; on hosts where graham already has passwordless sudo (or where -K isn't passed), ansible blocked indefinitely waiting for a root password prompt. Now the install-via-su task only runs when sudo is actually missing, and the python3 task uses sudo when available. Also bring in latest microwaveprop dashboard tweaks.
This commit is contained in:
parent
be96483fb0
commit
0970e29364
4 changed files with 158 additions and 5 deletions
|
|
@ -7,11 +7,26 @@
|
|||
vars:
|
||||
ansible_user: graham
|
||||
tasks:
|
||||
- name: Ensure sudo is present
|
||||
# Probe before attempting privilege escalation. If sudo already works for
|
||||
# graham, the install-via-su tasks below would otherwise block on a root
|
||||
# password prompt that the Makefile's `-K` is designed to feed — and on a
|
||||
# passwordless-sudo host with no `-K` provided, hang indefinitely.
|
||||
- name: Probe sudo state
|
||||
raw: |
|
||||
if command -v sudo >/dev/null 2>&1; then
|
||||
exit 0
|
||||
if sudo -n true 2>/dev/null; then
|
||||
echo "sudo-passwordless"
|
||||
else
|
||||
echo "sudo-needs-password"
|
||||
fi
|
||||
else
|
||||
echo "sudo-missing"
|
||||
fi
|
||||
register: sudo_state
|
||||
changed_when: false
|
||||
|
||||
- name: Ensure sudo is present
|
||||
raw: |
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update && apt-get install -y sudo
|
||||
elif command -v dnf >/dev/null 2>&1; then
|
||||
|
|
@ -27,6 +42,7 @@
|
|||
become: yes
|
||||
become_method: su
|
||||
become_user: root
|
||||
when: "'sudo-missing' in sudo_state.stdout"
|
||||
|
||||
- name: Ensure Python 3 is present
|
||||
raw: |
|
||||
|
|
@ -47,7 +63,9 @@
|
|||
fi
|
||||
changed_when: false
|
||||
become: yes
|
||||
become_method: su
|
||||
# Once sudo exists (newly installed or pre-existing) prefer it; only fall
|
||||
# back to `su` if sudo really isn't there.
|
||||
become_method: "{{ 'su' if 'sudo-missing' in sudo_state.stdout else 'sudo' }}"
|
||||
become_user: root
|
||||
|
||||
- name: Bootstrap new host - Base configuration
|
||||
|
|
|
|||
|
|
@ -8,8 +8,11 @@ grafana_http_port: 3000
|
|||
grafana_root_url: "http://10.0.15.31:3000/"
|
||||
|
||||
grafana_admin_user: admin
|
||||
# Set in vault / host_vars; default kept for first-boot only.
|
||||
grafana_admin_password: admin
|
||||
# Empty by default so re-running ansible doesn't clobber a password that was
|
||||
# rotated via the Grafana UI (or `grafana-cli admin reset-admin-password`).
|
||||
# To bootstrap or enforce a specific password, set this in host_vars / vault;
|
||||
# whatever you set will be written into grafana.ini on every subsequent run.
|
||||
grafana_admin_password: ""
|
||||
|
||||
# On-disk location for provisioned dashboard JSON files. Grafana watches this
|
||||
# path via the file-based dashboard provider configured in
|
||||
|
|
|
|||
|
|
@ -491,6 +491,136 @@
|
|||
"targets": [
|
||||
{"expr": "max by(queue, state) (microwaveprop_oban_queue_count{state=~\"retryable|discarded|cancelled\", instance=~\"$instance\"})", "legendFormat": "{{queue}} / {{state}}", "refId": "A"}
|
||||
]
|
||||
},
|
||||
|
||||
{"type": "row", "id": 109, "title": "Rust Workers (prop-grid-rs + hrrr-point-rs)", "collapsed": false, "gridPos": {"h": 1, "w": 24, "x": 0, "y": 117}, "panels": []},
|
||||
{
|
||||
"type": "stat",
|
||||
"id": 90,
|
||||
"title": "prop-grid-rs Replicas",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 118},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "yellow", "value": 1}, {"color": "green", "value": 2}]}, "unit": "short"}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [{"expr": "count(up{job=\"prop-grid-rs\"} == 1)", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"id": 91,
|
||||
"title": "hrrr-point-rs Replicas",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 118},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "yellow", "value": 1}, {"color": "green", "value": 2}]}, "unit": "short"}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [{"expr": "count(up{job=\"hrrr-point-rs\"} == 1)", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"id": 92,
|
||||
"title": "Tasks In-Flight (sum)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 118},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "short"}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [{"expr": "sum(prop_grid_rs_tasks_in_flight)", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"id": 93,
|
||||
"title": "Chain Step Success Rate (5m)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 12, "y": 118},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "yellow", "value": 0.9}, {"color": "green", "value": 0.99}]}, "unit": "percentunit", "min": 0, "max": 1}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [{"expr": "sum(rate(prop_grid_rs_chain_steps_total{outcome=\"success\"}[5m])) / clamp_min(sum(rate(prop_grid_rs_chain_steps_total[5m])), 1e-9)", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"id": 94,
|
||||
"title": "Point Batch Success Rate (5m)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 16, "y": 118},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "yellow", "value": 0.9}, {"color": "green", "value": 0.99}]}, "unit": "percentunit", "min": 0, "max": 1}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [{"expr": "sum(rate(prop_grid_rs_point_batches_total{outcome=\"success\"}[5m])) / clamp_min(sum(rate(prop_grid_rs_point_batches_total[5m])), 1e-9)", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"id": 95,
|
||||
"title": "Points Inserted/s",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 20, "y": 118},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "unit": "short"}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "area", "reduceOptions": {"calcs": ["lastNotNull"]}},
|
||||
"targets": [{"expr": "sum(rate(prop_grid_rs_points_processed_total{kind=\"inserted\"}[5m]))", "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 96,
|
||||
"title": "Chain Step Duration p50 / p95 / p99 (prop-grid-rs)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 122},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "s"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.50, sum by(le, outcome) (rate(prop_grid_rs_chain_step_duration_seconds_bucket[5m])))", "legendFormat": "p50 — {{outcome}}", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le, outcome) (rate(prop_grid_rs_chain_step_duration_seconds_bucket[5m])))", "legendFormat": "p95 — {{outcome}}", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.99, sum by(le, outcome) (rate(prop_grid_rs_chain_step_duration_seconds_bucket[5m])))", "legendFormat": "p99 — {{outcome}}", "refId": "C"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 97,
|
||||
"title": "wgrib2 Decode Duration (prop-grid-rs)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 122},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "s"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.50, sum by(le) (rate(prop_grid_rs_decode_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le) (rate(prop_grid_rs_decode_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.99, sum by(le) (rate(prop_grid_rs_decode_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 98,
|
||||
"title": "Point Batch Duration p50 / p95 / p99 (hrrr-point-rs)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 130},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 0}, "unit": "s"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.50, sum by(le, outcome) (rate(prop_grid_rs_point_batch_duration_seconds_bucket[5m])))", "legendFormat": "p50 — {{outcome}}", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by(le, outcome) (rate(prop_grid_rs_point_batch_duration_seconds_bucket[5m])))", "legendFormat": "p95 — {{outcome}}", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.99, sum by(le, outcome) (rate(prop_grid_rs_point_batch_duration_seconds_bucket[5m])))", "legendFormat": "p99 — {{outcome}}", "refId": "C"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 99,
|
||||
"title": "Points Requested vs Inserted (hrrr-point-rs)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 130},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "line", "fillOpacity": 10}, "unit": "short"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "sum(rate(prop_grid_rs_points_processed_total{kind=\"requested\"}[5m]))", "legendFormat": "requested/s", "refId": "A"},
|
||||
{"expr": "sum(rate(prop_grid_rs_points_processed_total{kind=\"inserted\"}[5m]))", "legendFormat": "inserted/s", "refId": "B"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"id": 110,
|
||||
"title": "Chain + Point Batch Throughput (per outcome)",
|
||||
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 138},
|
||||
"fieldConfig": {"defaults": {"color": {"mode": "palette-classic"}, "custom": {"drawStyle": "bars", "fillOpacity": 60, "stacking": {"mode": "normal"}}, "unit": "ops"}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "sum by(outcome) (rate(prop_grid_rs_chain_steps_total[5m]))", "legendFormat": "grid-rs chain — {{outcome}}", "refId": "A"},
|
||||
{"expr": "sum by(outcome) (rate(prop_grid_rs_point_batches_total[5m]))", "legendFormat": "hrrr-point-rs batch — {{outcome}}", "refId": "B"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@ root_url = {{ grafana_root_url }}
|
|||
|
||||
[security]
|
||||
admin_user = {{ grafana_admin_user }}
|
||||
{% if grafana_admin_password %}
|
||||
admin_password = {{ grafana_admin_password }}
|
||||
{% endif %}
|
||||
|
||||
[analytics]
|
||||
reporting_enabled = false
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue