Graham McIntire
922552059f
Provisions prom.w5isp.com (10.0.15.31) as the home-cluster monitoring host. Prometheus 3.6.0 scrapes the Talos cluster API/nodes/cAdvisor via an external SA token; Grafana ships with the Prometheus datasource pre-provisioned. host_vars opens 9090/9093/9100/3000 to 10.0.0.0/16 plus SSH from the home LAN so the firewall role doesn't lock out admin access.
32 lines
1.1 KiB
Django/Jinja
32 lines
1.1 KiB
Django/Jinja
{{ ansible_managed | comment }}
|
|
groups:
|
|
- name: instance-health
|
|
rules:
|
|
- alert: InstanceDown
|
|
expr: up == 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Instance {{ '{{' }} $labels.instance {{ '}}' }} down"
|
|
description: "{{ '{{' }} $labels.instance {{ '}}' }} ({{ '{{' }} $labels.job {{ '}}' }}) has been unreachable for more than 5 minutes."
|
|
|
|
- name: node-health
|
|
rules:
|
|
- alert: NodeFilesystemAlmostFull
|
|
expr: |
|
|
(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} /
|
|
node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) < 0.10
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Filesystem {{ '{{' }} $labels.mountpoint {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }} below 10%"
|
|
|
|
- alert: NodeHighLoad
|
|
expr: node_load5 > (count by (instance) (node_cpu_seconds_total{mode="idle"}) * 2)
|
|
for: 15m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: "High load on {{ '{{' }} $labels.instance {{ '}}' }}"
|