graham/infra
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

monitoring: collect Unbound metrics via Telegraf into Prometheus + Grafana

Browse source 
Add a telegraf role (applied to the resolvers) that runs unbound-control
via the inputs.unbound plugin and exposes the stats as a Prometheus
/metrics endpoint on :9273, firewalled to the 204.110.188.0/22 scrape
network. Imports the InfluxData signing key (influxdata-archive.key; the
_compat key expired 2026-01-17) so dnf can validate the package.

- prom.w5isp.com: scrape job "unbound" for resolver1/2:9273
- grafana: "Unbound Resolvers" dashboard (queries, cache hit ratio,
  rcodes, query types, recursion time, request list, cache memory)
- resolvers: disable unused shm-enable to silence the startup warning
This commit is contained in:
Graham McIntire 2026-06-07 13:34:24 -05:00
parent 43730bf5e7
commit 66986e294b
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
8 changed files with 988 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
ansible/host_vars/prom.w5isp.com.yml
View file

@ -36,3 +36,11 @@ firewall_allow_rules:
proto: tcp
from_ip: 10.0.0.0/16
comment: Loki HTTP (push + query)
# Scrape the Unbound resolvers' Telegraf exporters (port 9273, plain HTTP,
# firewalled on the resolvers to the 204.110.188.0/22 scrape network).
prometheus_extra_scrape_configs:
- job_name: unbound
targets:
- resolver1.vntx.net:9273
- resolver2.vntx.net:9273

1
ansible/playbook.yml
View file

@ -33,6 +33,7 @@
roles:
- resolvers
- caddy
- telegraf
- name: Install and configure NetBox
hosts: netbox_servers

866
ansible/roles/grafana/files/dashboards/unbound.json Normal file
View file

@ -0,0 +1,866 @@
{
"uid": "unbound-resolvers",
"title": "Unbound Resolvers",
"tags": [
"unbound",
"dns",
"resolvers"
],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"editable": true,
"refresh": "30s",
"time": {
"from": "now-6h",
"to": "now"
},
"templating": {
"list": [
{
"name": "ds_prometheus",
"type": "datasource",
"query": "prometheus",
"current": {},
"hide": 0,
"refresh": 1,
"regex": "",
"label": "Data source"
},
{
"name": "instance",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"query": "label_values(unbound_time_up, instance)",
"refresh": 2,
"includeAll": true,
"multi": true,
"current": {
"text": "All",
"value": "$__all"
},
"label": "Resolver",
"sort": 1
}
]
},
"panels": [
{
"type": "stat",
"title": "Uptime",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 6,
"h": 4
},
"fieldConfig": {
"defaults": {
"unit": "s",
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "max(unbound_time_up{instance=~\"$instance\"})",
"legendFormat": "",
"refId": "A"
}
]
},
{
"type": "stat",
"title": "Queries / sec",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"gridPos": {
"x": 6,
"y": 0,
"w": 6,
"h": 4
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_total_num_queries{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "",
"refId": "A"
}
]
},
{
"type": "stat",
"title": "Cache hit ratio",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 6,
"h": 4
},
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_total_num_cachehits{instance=~\"$instance\"}[$__rate_interval])) / clamp_min(sum(rate(unbound_total_num_cachehits{instance=~\"$instance\"}[$__rate_interval])) + sum(rate(unbound_total_num_cachemiss{instance=~\"$instance\"}[$__rate_interval])), 1)",
"legendFormat": "",
"refId": "A"
}
]
},
{
"type": "stat",
"title": "Avg recursion time",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"gridPos": {
"x": 18,
"y": 0,
"w": 6,
"h": 4
},
"fieldConfig": {
"defaults": {
"unit": "s",
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "avg(unbound_total_recursion_time_avg{instance=~\"$instance\"})",
"legendFormat": "",
"refId": "A"
}
]
},
{
"type": "timeseries",
"title": "Query rate",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"gridPos": {
"x": 0,
"y": 4,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"stacking": {
"mode": "none"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "rate(unbound_total_num_queries{instance=~\"$instance\"}[$__rate_interval])",
"legendFormat": "{{instance}}",
"refId": "A"
}
]
},
{
"type": "timeseries",
"title": "Cache hits vs misses (rate)",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"gridPos": {
"x": 12,
"y": 4,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"stacking": {
"mode": "none"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_total_num_cachehits{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "hits",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_total_num_cachemiss{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "misses",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_total_num_prefetch{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "prefetch",
"refId": "C"
}
]
},
{
"type": "timeseries",
"title": "Queries by type (rate)",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"gridPos": {
"x": 0,
"y": 12,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_query_type_A{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "A",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_query_type_AAAA{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "AAAA",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_query_type_PTR{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "PTR",
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_query_type_HTTPS{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "HTTPS",
"refId": "D"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_query_type_SRV{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "SRV",
"refId": "E"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_query_type_TXT{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "TXT",
"refId": "F"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_query_type_SOA{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "SOA",
"refId": "G"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_query_type_NS{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "NS",
"refId": "H"
}
]
},
{
"type": "timeseries",
"title": "Answers by rcode (rate)",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"gridPos": {
"x": 12,
"y": 12,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_answer_rcode_NOERROR{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "NOERROR",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_answer_rcode_NXDOMAIN{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "NXDOMAIN",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_answer_rcode_SERVFAIL{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "SERVFAIL",
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_answer_rcode_REFUSED{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "REFUSED",
"refId": "D"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_answer_rcode_FORMERR{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "FORMERR",
"refId": "E"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "sum(rate(unbound_num_answer_rcode_NOTIMPL{instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "NOTIMPL",
"refId": "F"
}
]
},
{
"type": "timeseries",
"title": "Recursion time",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"gridPos": {
"x": 0,
"y": 20,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"stacking": {
"mode": "none"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "unbound_total_recursion_time_avg{instance=~\"$instance\"}",
"legendFormat": "{{instance}} avg",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "unbound_total_recursion_time_median{instance=~\"$instance\"}",
"legendFormat": "{{instance}} median",
"refId": "B"
}
]
},
{
"type": "timeseries",
"title": "Request list",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"gridPos": {
"x": 12,
"y": 20,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"stacking": {
"mode": "none"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "unbound_total_requestlist_current_all{instance=~\"$instance\"}",
"legendFormat": "{{instance}} current",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "unbound_total_requestlist_current_user{instance=~\"$instance\"}",
"legendFormat": "{{instance}} user",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "unbound_total_requestlist_max{instance=~\"$instance\"}",
"legendFormat": "{{instance}} max",
"refId": "C"
}
]
},
{
"type": "timeseries",
"title": "Cache memory",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"gridPos": {
"x": 0,
"y": 28,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "bytes",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"stacking": {
"mode": "none"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "unbound_mem_cache_rrset{instance=~\"$instance\"}",
"legendFormat": "{{instance}} rrset",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "unbound_mem_cache_message{instance=~\"$instance\"}",
"legendFormat": "{{instance}} message",
"refId": "B"
}
]
},
{
"type": "timeseries",
"title": "Recursive replies (rate)",
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"gridPos": {
"x": 12,
"y": 28,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"stacking": {
"mode": "none"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${ds_prometheus}"
},
"expr": "rate(unbound_total_num_recursivereplies{instance=~\"$instance\"}[$__rate_interval])",
"legendFormat": "{{instance}}",
"refId": "A"
}
]
}
]
}

6
ansible/roles/resolvers/templates/unbound.conf.j2
View file

@ -26,8 +26,10 @@ server:
# enable shm for stats, default no. if you enable also enable
# statistics-interval, every time it also writes stats to the
# shared memory segment keyed with shm-key.
# shm-enable: no
shm-enable: yes
# Disabled: nothing consumes the shm stats segment, and leaving it on
# with statistics-interval: 0 logs a warning at startup. Re-enable (with
# a non-zero statistics-interval) if an unbound stats exporter is added.
shm-enable: no
# shm for stats uses this key, and key+1 for the shared mem segment.
# shm-key: 11777

22
ansible/roles/telegraf/defaults/main.yml Normal file
View file

@ -0,0 +1,22 @@
---
# Telegraf agent: collects Unbound stats via unbound-control and exposes them
# as a Prometheus /metrics endpoint for the prom.w5isp.com server to scrape.
# (unbound-control runs fine as the unprivileged telegraf user because the
# resolvers' control interface is plaintext on localhost.)
telegraf_interval: "30s"
telegraf_unbound_binary: /usr/sbin/unbound-control
# Prometheus output — Prometheus scrapes http://<resolver>:<port>/metrics.
telegraf_prometheus_port: 9273
telegraf_prometheus_listen: "0.0.0.0:{{ telegraf_prometheus_port }}"
# Only this source network may reach the metrics port (the routed scrape
# path from prom.w5isp.com). Keeps Unbound stats off the public internet.
telegraf_scrape_allow_cidr: "204.110.188.0/22"
# InfluxData package repository (EL/RedHat).
# Use influxdata-archive.key (not _compat): the _compat key's primary expired
# 2026-01-17, while current packages are signed by a 2025 subkey only present
# in this keyring.
telegraf_influxdata_baseurl: "https://repos.influxdata.com/stable/$basearch/main"
telegraf_influxdata_gpgkey: "https://repos.influxdata.com/influxdata-archive.key"

5
ansible/roles/telegraf/handlers/main.yml Normal file
View file

@ -0,0 +1,5 @@
---
- name: Restart telegraf
ansible.builtin.systemd:
name: telegraf
state: restarted

53
ansible/roles/telegraf/tasks/main.yml Normal file
View file

@ -0,0 +1,53 @@
---
# Installs Telegraf and configures it to collect Unbound stats (via
# unbound-control) and expose them as Prometheus metrics on
# telegraf_prometheus_port for the central Prometheus server to scrape.
- name: Import InfluxData package signing key (RedHat)
ansible.builtin.rpm_key:
key: "{{ telegraf_influxdata_gpgkey }}"
state: present
when: ansible_facts['os_family'] == "RedHat"
- name: Add InfluxData package repository (RedHat)
ansible.builtin.yum_repository:
name: influxdata
description: InfluxData Repository - Stable
baseurl: "{{ telegraf_influxdata_baseurl }}"
gpgcheck: true
gpgkey: "{{ telegraf_influxdata_gpgkey }}"
enabled: true
when: ansible_facts['os_family'] == "RedHat"
- name: Install Telegraf (RedHat)
ansible.builtin.dnf:
name: telegraf
state: present
when: ansible_facts['os_family'] == "RedHat"
- name: Deploy Telegraf configuration
ansible.builtin.template:
src: telegraf.conf.j2
dest: /etc/telegraf/telegraf.conf
owner: root
group: root
mode: "0644"
notify: Restart telegraf
- name: Allow Prometheus scrape of the metrics port from the scrape subnet (firewalld)
ansible.posix.firewalld:
zone: public
rich_rule: >-
rule family="ipv4" source address="{{ telegraf_scrape_allow_cidr }}"
port port="{{ telegraf_prometheus_port }}" protocol="tcp" accept
permanent: true
immediate: true
state: enabled
when: ansible_facts['os_family'] == "RedHat"
- name: Enable and start Telegraf
ansible.builtin.systemd:
name: telegraf
state: started
enabled: true
daemon_reload: true

29
ansible/roles/telegraf/templates/telegraf.conf.j2 Normal file
View file

@ -0,0 +1,29 @@
# {{ ansible_managed }}
# Managed by Ansible (roles/telegraf). Collects Unbound stats via
# unbound-control and exposes them as a Prometheus /metrics endpoint that
# prom.w5isp.com scrapes. This file fully replaces the packaged default so
# the stock [[outputs.influxdb]] section never runs.
[agent]
interval = "{{ telegraf_interval }}"
round_interval = true
metric_batch_size = 1000
metric_buffer_limit = 10000
collection_jitter = "0s"
flush_interval = "10s"
flush_jitter = "0s"
omit_hostname = false
# Read Unbound statistics (runs `unbound-control stats_noreset`).
[[inputs.unbound]]
binary = "{{ telegraf_unbound_binary }}"
use_sudo = false
thread_as_tag = false
# Expose collected metrics for Prometheus to scrape (measurement "unbound"
# becomes unbound_<field>, e.g. unbound_total_num_queries).
[[outputs.prometheus_client]]
listen = "{{ telegraf_prometheus_listen }}"
metric_version = 2
path = "/metrics"
expiration_interval = "60s"