prom: add prometheus stack (prometheus, alertmanager, grafana, node_exporter)
Provisions prom.w5isp.com (10.0.15.31) as the home-cluster monitoring host. Prometheus 3.6.0 scrapes the Talos cluster API/nodes/cAdvisor via an external SA token; Grafana ships with the Prometheus datasource pre-provisioned. host_vars opens 9090/9093/9100/3000 to 10.0.0.0/16 plus SSH from the home LAN so the firewall role doesn't lock out admin access.
This commit is contained in:
parent
20896be049
commit
922552059f
25 changed files with 1024 additions and 0 deletions
35
ansible/host_vars/prom.w5isp.com.yml
Normal file
35
ansible/host_vars/prom.w5isp.com.yml
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
---
|
||||
ansible_user: ansible
|
||||
ansible_host: 10.0.15.31
|
||||
ansible_python_interpreter: /usr/bin/python3
|
||||
ansible_ssh_private_key_file: /Users/graham/.ssh/ansible
|
||||
ansible_ssh_common_args: '-o StrictHostKeyChecking=accept-new'
|
||||
|
||||
network:
|
||||
skip: true
|
||||
|
||||
# Open Prometheus stack ports to the home LAN.
|
||||
# Trusted subnets (Tailscale, VNTX) are already wide-open via firewall role defaults;
|
||||
# this adds the 10.0.15.0/24 home cluster subnet so the k8s nodes can hit node_exporter
|
||||
# and so any LAN client can reach the Grafana UI.
|
||||
firewall_allow_rules:
|
||||
- port: 22
|
||||
proto: tcp
|
||||
from_ip: 10.0.0.0/16
|
||||
comment: SSH from home LAN
|
||||
- port: 9090
|
||||
proto: tcp
|
||||
from_ip: 10.0.0.0/16
|
||||
comment: Prometheus UI/API
|
||||
- port: 9093
|
||||
proto: tcp
|
||||
from_ip: 10.0.0.0/16
|
||||
comment: Alertmanager
|
||||
- port: 9100
|
||||
proto: tcp
|
||||
from_ip: 10.0.0.0/16
|
||||
comment: node_exporter
|
||||
- port: 3000
|
||||
proto: tcp
|
||||
from_ip: 10.0.0.0/16
|
||||
comment: Grafana UI
|
||||
|
|
@ -25,6 +25,13 @@ mail.mcintire.me ansible_host=mail
|
|||
dokku.w5isp.com
|
||||
aprs.w5isp.com
|
||||
staging.towerops.net
|
||||
prom.w5isp.com ansible_host=10.0.15.31
|
||||
|
||||
[prometheus_servers]
|
||||
prom.w5isp.com
|
||||
|
||||
[node_exporter_servers:children]
|
||||
prometheus_servers
|
||||
|
||||
[dokku_servers]
|
||||
staging.towerops.net
|
||||
|
|
|
|||
|
|
@ -308,3 +308,27 @@
|
|||
- dokku
|
||||
roles:
|
||||
- dokku
|
||||
|
||||
- name: Configure node_exporter targets
|
||||
hosts: node_exporter_servers
|
||||
become: yes
|
||||
gather_facts: true
|
||||
tags:
|
||||
- node_exporter
|
||||
- prometheus_stack
|
||||
roles:
|
||||
- node_exporter
|
||||
|
||||
- name: Configure Prometheus monitoring stack
|
||||
hosts: prometheus_servers
|
||||
become: yes
|
||||
gather_facts: true
|
||||
tags:
|
||||
- prometheus_stack
|
||||
roles:
|
||||
- role: alertmanager
|
||||
tags: alertmanager
|
||||
- role: prometheus
|
||||
tags: prometheus
|
||||
- role: grafana
|
||||
tags: grafana
|
||||
|
|
|
|||
28
ansible/roles/alertmanager/defaults/main.yml
Normal file
28
ansible/roles/alertmanager/defaults/main.yml
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
---
|
||||
alertmanager_version: "0.28.1"
|
||||
alertmanager_arch: amd64
|
||||
|
||||
alertmanager_user: alertmanager
|
||||
alertmanager_group: alertmanager
|
||||
|
||||
alertmanager_install_dir: /usr/local/bin
|
||||
alertmanager_config_dir: /etc/alertmanager
|
||||
alertmanager_data_dir: /var/lib/alertmanager
|
||||
|
||||
alertmanager_listen_address: "0.0.0.0:9093"
|
||||
alertmanager_cluster_listen: "" # "" disables clustering (single-node)
|
||||
alertmanager_external_url: ""
|
||||
|
||||
# Default config: a single null receiver (alerts are accepted and discarded).
|
||||
# Override `alertmanager_route` and `alertmanager_receivers` to wire up real receivers.
|
||||
alertmanager_route:
|
||||
receiver: "null"
|
||||
group_by: ["alertname", "cluster", "service"]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 3h
|
||||
|
||||
alertmanager_receivers:
|
||||
- name: "null"
|
||||
|
||||
alertmanager_inhibit_rules: []
|
||||
11
ansible/roles/alertmanager/handlers/main.yml
Normal file
11
ansible/roles/alertmanager/handlers/main.yml
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
---
|
||||
- name: Restart alertmanager
|
||||
ansible.builtin.systemd:
|
||||
name: alertmanager
|
||||
state: restarted
|
||||
daemon_reload: true
|
||||
|
||||
- name: Reload alertmanager
|
||||
ansible.builtin.systemd:
|
||||
name: alertmanager
|
||||
state: reloaded
|
||||
97
ansible/roles/alertmanager/tasks/main.yml
Normal file
97
ansible/roles/alertmanager/tasks/main.yml
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
---
|
||||
- name: Create alertmanager group
|
||||
ansible.builtin.group:
|
||||
name: "{{ alertmanager_group }}"
|
||||
system: true
|
||||
state: present
|
||||
|
||||
- name: Create alertmanager user
|
||||
ansible.builtin.user:
|
||||
name: "{{ alertmanager_user }}"
|
||||
group: "{{ alertmanager_group }}"
|
||||
system: true
|
||||
shell: /usr/sbin/nologin
|
||||
home: "{{ alertmanager_data_dir }}"
|
||||
create_home: false
|
||||
state: present
|
||||
|
||||
- name: Create alertmanager directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ alertmanager_user }}"
|
||||
group: "{{ alertmanager_group }}"
|
||||
mode: "0755"
|
||||
loop:
|
||||
- "{{ alertmanager_config_dir }}"
|
||||
- "{{ alertmanager_data_dir }}"
|
||||
|
||||
- name: Check installed alertmanager version
|
||||
ansible.builtin.command: "{{ alertmanager_install_dir }}/alertmanager --version"
|
||||
register: alertmanager_installed
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
|
||||
- name: Decide whether alertmanager needs (re)install
|
||||
ansible.builtin.set_fact:
|
||||
alertmanager_needs_install: >-
|
||||
{{
|
||||
alertmanager_installed.rc != 0
|
||||
or alertmanager_version not in (alertmanager_installed.stdout | default(''))
|
||||
and alertmanager_version not in (alertmanager_installed.stderr | default(''))
|
||||
}}
|
||||
|
||||
- name: Download alertmanager tarball
|
||||
ansible.builtin.get_url:
|
||||
url: "https://github.com/prometheus/alertmanager/releases/download/v{{ alertmanager_version }}/alertmanager-{{ alertmanager_version }}.linux-{{ alertmanager_arch }}.tar.gz"
|
||||
dest: "/tmp/alertmanager-{{ alertmanager_version }}.tar.gz"
|
||||
mode: "0644"
|
||||
when: alertmanager_needs_install
|
||||
|
||||
- name: Extract alertmanager
|
||||
ansible.builtin.unarchive:
|
||||
src: "/tmp/alertmanager-{{ alertmanager_version }}.tar.gz"
|
||||
dest: /tmp/
|
||||
remote_src: true
|
||||
creates: "/tmp/alertmanager-{{ alertmanager_version }}.linux-{{ alertmanager_arch }}/alertmanager"
|
||||
when: alertmanager_needs_install
|
||||
|
||||
- name: Install alertmanager binaries
|
||||
ansible.builtin.copy:
|
||||
src: "/tmp/alertmanager-{{ alertmanager_version }}.linux-{{ alertmanager_arch }}/{{ item }}"
|
||||
dest: "{{ alertmanager_install_dir }}/{{ item }}"
|
||||
remote_src: true
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
loop:
|
||||
- alertmanager
|
||||
- amtool
|
||||
when: alertmanager_needs_install
|
||||
notify: Restart alertmanager
|
||||
|
||||
- name: Render alertmanager.yml
|
||||
ansible.builtin.template:
|
||||
src: alertmanager.yml.j2
|
||||
dest: "{{ alertmanager_config_dir }}/alertmanager.yml"
|
||||
owner: "{{ alertmanager_user }}"
|
||||
group: "{{ alertmanager_group }}"
|
||||
mode: "0640"
|
||||
validate: "{{ alertmanager_install_dir }}/amtool check-config %s"
|
||||
notify: Reload alertmanager
|
||||
|
||||
- name: Install alertmanager systemd unit
|
||||
ansible.builtin.template:
|
||||
src: alertmanager.service.j2
|
||||
dest: /etc/systemd/system/alertmanager.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: Restart alertmanager
|
||||
|
||||
- name: Enable and start alertmanager
|
||||
ansible.builtin.systemd:
|
||||
name: alertmanager
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: true
|
||||
33
ansible/roles/alertmanager/templates/alertmanager.service.j2
Normal file
33
ansible/roles/alertmanager/templates/alertmanager.service.j2
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
[Unit]
|
||||
Description=Prometheus Alertmanager
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ alertmanager_user }}
|
||||
Group={{ alertmanager_group }}
|
||||
ExecStart={{ alertmanager_install_dir }}/alertmanager \
|
||||
--config.file={{ alertmanager_config_dir }}/alertmanager.yml \
|
||||
--storage.path={{ alertmanager_data_dir }} \
|
||||
--web.listen-address={{ alertmanager_listen_address }} \
|
||||
{% if alertmanager_external_url %}
|
||||
--web.external-url={{ alertmanager_external_url }} \
|
||||
{% endif %}
|
||||
{% if alertmanager_cluster_listen %}
|
||||
--cluster.listen-address={{ alertmanager_cluster_listen }}
|
||||
{% else %}
|
||||
--cluster.listen-address=
|
||||
{% endif %}
|
||||
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
NoNewPrivileges=true
|
||||
ProtectHome=true
|
||||
ProtectSystem=full
|
||||
ReadWritePaths={{ alertmanager_data_dir }}
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
9
ansible/roles/alertmanager/templates/alertmanager.yml.j2
Normal file
9
ansible/roles/alertmanager/templates/alertmanager.yml.j2
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{{ ansible_managed | comment }}
|
||||
route:
|
||||
{{ alertmanager_route | to_nice_yaml(indent=2) | indent(2, true) }}
|
||||
receivers:
|
||||
{{ alertmanager_receivers | to_nice_yaml(indent=2) | indent(2, true) }}
|
||||
{% if alertmanager_inhibit_rules %}
|
||||
inhibit_rules:
|
||||
{{ alertmanager_inhibit_rules | to_nice_yaml(indent=2) | indent(2, true) }}
|
||||
{% endif %}
|
||||
26
ansible/roles/grafana/defaults/main.yml
Normal file
26
ansible/roles/grafana/defaults/main.yml
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
---
|
||||
grafana_apt_repo: "deb https://apt.grafana.com stable main"
|
||||
grafana_apt_key_url: "https://apt.grafana.com/gpg.key"
|
||||
grafana_apt_keyring: "/etc/apt/keyrings/grafana.gpg"
|
||||
|
||||
grafana_listen_address: "0.0.0.0"
|
||||
grafana_http_port: 3000
|
||||
grafana_root_url: "http://10.0.15.31:3000/"
|
||||
|
||||
grafana_admin_user: admin
|
||||
# Set in vault / host_vars; default kept for first-boot only.
|
||||
grafana_admin_password: admin
|
||||
|
||||
# Datasources provisioned at startup. The first one is marked default.
|
||||
grafana_datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: "http://127.0.0.1:9090"
|
||||
isDefault: true
|
||||
- name: Alertmanager
|
||||
type: alertmanager
|
||||
access: proxy
|
||||
url: "http://127.0.0.1:9093"
|
||||
jsonData:
|
||||
implementation: prometheus
|
||||
6
ansible/roles/grafana/handlers/main.yml
Normal file
6
ansible/roles/grafana/handlers/main.yml
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
- name: Restart grafana
|
||||
ansible.builtin.systemd:
|
||||
name: grafana-server
|
||||
state: restarted
|
||||
daemon_reload: true
|
||||
60
ansible/roles/grafana/tasks/main.yml
Normal file
60
ansible/roles/grafana/tasks/main.yml
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
---
|
||||
- name: Ensure /etc/apt/keyrings exists
|
||||
ansible.builtin.file:
|
||||
path: /etc/apt/keyrings
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: Install prerequisites
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
- apt-transport-https
|
||||
- gnupg
|
||||
state: present
|
||||
update_cache: true
|
||||
|
||||
- name: Add Grafana apt signing key
|
||||
ansible.builtin.get_url:
|
||||
url: "{{ grafana_apt_key_url }}"
|
||||
dest: "{{ grafana_apt_keyring }}"
|
||||
mode: "0644"
|
||||
force: false
|
||||
|
||||
- name: Add Grafana apt repository
|
||||
ansible.builtin.apt_repository:
|
||||
repo: "deb [signed-by={{ grafana_apt_keyring }}] https://apt.grafana.com stable main"
|
||||
filename: grafana
|
||||
state: present
|
||||
update_cache: true
|
||||
|
||||
- name: Install Grafana OSS
|
||||
ansible.builtin.apt:
|
||||
name: grafana
|
||||
state: present
|
||||
|
||||
- name: Configure grafana.ini
|
||||
ansible.builtin.template:
|
||||
src: grafana.ini.j2
|
||||
dest: /etc/grafana/grafana.ini
|
||||
owner: root
|
||||
group: grafana
|
||||
mode: "0640"
|
||||
notify: Restart grafana
|
||||
|
||||
- name: Provision Prometheus datasource
|
||||
ansible.builtin.template:
|
||||
src: datasources.yaml.j2
|
||||
dest: /etc/grafana/provisioning/datasources/prometheus.yaml
|
||||
owner: root
|
||||
group: grafana
|
||||
mode: "0640"
|
||||
notify: Restart grafana
|
||||
|
||||
- name: Enable and start grafana-server
|
||||
ansible.builtin.systemd:
|
||||
name: grafana-server
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: true
|
||||
18
ansible/roles/grafana/templates/datasources.yaml.j2
Normal file
18
ansible/roles/grafana/templates/datasources.yaml.j2
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{{ ansible_managed | comment }}
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
{% for ds in grafana_datasources %}
|
||||
- name: {{ ds.name }}
|
||||
type: {{ ds.type }}
|
||||
access: {{ ds.access | default('proxy') }}
|
||||
url: {{ ds.url }}
|
||||
isDefault: {{ ds.isDefault | default(false) | string | lower }}
|
||||
editable: true
|
||||
{% if ds.jsonData is defined %}
|
||||
jsonData:
|
||||
{% for k, v in ds.jsonData.items() %}
|
||||
{{ k }}: {{ v }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
21
ansible/roles/grafana/templates/grafana.ini.j2
Normal file
21
ansible/roles/grafana/templates/grafana.ini.j2
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{{ ansible_managed | comment }}
|
||||
[server]
|
||||
http_addr = {{ grafana_listen_address }}
|
||||
http_port = {{ grafana_http_port }}
|
||||
root_url = {{ grafana_root_url }}
|
||||
|
||||
[security]
|
||||
admin_user = {{ grafana_admin_user }}
|
||||
admin_password = {{ grafana_admin_password }}
|
||||
|
||||
[analytics]
|
||||
reporting_enabled = false
|
||||
check_for_updates = false
|
||||
check_for_plugin_updates = false
|
||||
|
||||
[users]
|
||||
allow_sign_up = false
|
||||
|
||||
[log]
|
||||
mode = console file
|
||||
level = info
|
||||
10
ansible/roles/node_exporter/defaults/main.yml
Normal file
10
ansible/roles/node_exporter/defaults/main.yml
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
---
|
||||
node_exporter_version: "1.9.1"
|
||||
node_exporter_arch: amd64
|
||||
node_exporter_user: node_exporter
|
||||
node_exporter_group: node_exporter
|
||||
node_exporter_install_dir: /usr/local/bin
|
||||
node_exporter_listen_address: "0.0.0.0:9100"
|
||||
|
||||
# Default --collector.* flags. Override per-host if needed.
|
||||
node_exporter_extra_args: []
|
||||
6
ansible/roles/node_exporter/handlers/main.yml
Normal file
6
ansible/roles/node_exporter/handlers/main.yml
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
- name: Restart node_exporter
|
||||
ansible.builtin.systemd:
|
||||
name: node_exporter
|
||||
state: restarted
|
||||
daemon_reload: true
|
||||
75
ansible/roles/node_exporter/tasks/main.yml
Normal file
75
ansible/roles/node_exporter/tasks/main.yml
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
---
|
||||
# Installs Prometheus node_exporter from the upstream tarball and runs it under systemd.
|
||||
|
||||
- name: Create node_exporter group
|
||||
ansible.builtin.group:
|
||||
name: "{{ node_exporter_group }}"
|
||||
system: true
|
||||
state: present
|
||||
|
||||
- name: Create node_exporter user
|
||||
ansible.builtin.user:
|
||||
name: "{{ node_exporter_user }}"
|
||||
group: "{{ node_exporter_group }}"
|
||||
system: true
|
||||
shell: /usr/sbin/nologin
|
||||
home: /nonexistent
|
||||
create_home: false
|
||||
state: present
|
||||
|
||||
- name: Check installed node_exporter version
|
||||
ansible.builtin.command: "{{ node_exporter_install_dir }}/node_exporter --version"
|
||||
register: node_exporter_installed
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
|
||||
- name: Set fact for installed version
|
||||
ansible.builtin.set_fact:
|
||||
node_exporter_needs_install: >-
|
||||
{{
|
||||
node_exporter_installed.rc != 0
|
||||
or (node_exporter_version not in (node_exporter_installed.stderr | default('')))
|
||||
and (node_exporter_version not in (node_exporter_installed.stdout | default('')))
|
||||
}}
|
||||
|
||||
- name: Download node_exporter tarball
|
||||
ansible.builtin.get_url:
|
||||
url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-{{ node_exporter_arch }}.tar.gz"
|
||||
dest: "/tmp/node_exporter-{{ node_exporter_version }}.tar.gz"
|
||||
mode: "0644"
|
||||
when: node_exporter_needs_install
|
||||
|
||||
- name: Extract node_exporter
|
||||
ansible.builtin.unarchive:
|
||||
src: "/tmp/node_exporter-{{ node_exporter_version }}.tar.gz"
|
||||
dest: /tmp/
|
||||
remote_src: true
|
||||
creates: "/tmp/node_exporter-{{ node_exporter_version }}.linux-{{ node_exporter_arch }}/node_exporter"
|
||||
when: node_exporter_needs_install
|
||||
|
||||
- name: Install node_exporter binary
|
||||
ansible.builtin.copy:
|
||||
src: "/tmp/node_exporter-{{ node_exporter_version }}.linux-{{ node_exporter_arch }}/node_exporter"
|
||||
dest: "{{ node_exporter_install_dir }}/node_exporter"
|
||||
remote_src: true
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
when: node_exporter_needs_install
|
||||
notify: Restart node_exporter
|
||||
|
||||
- name: Install node_exporter systemd unit
|
||||
ansible.builtin.template:
|
||||
src: node_exporter.service.j2
|
||||
dest: /etc/systemd/system/node_exporter.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: Restart node_exporter
|
||||
|
||||
- name: Enable and start node_exporter
|
||||
ansible.builtin.systemd:
|
||||
name: node_exporter
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: true
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
[Unit]
|
||||
Description=Prometheus node_exporter
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ node_exporter_user }}
|
||||
Group={{ node_exporter_group }}
|
||||
ExecStart={{ node_exporter_install_dir }}/node_exporter \
|
||||
--web.listen-address={{ node_exporter_listen_address }} \
|
||||
{% for arg in node_exporter_extra_args %}
|
||||
{{ arg }} \
|
||||
{% endfor %}
|
||||
$NODE_EXPORTER_EXTRA_ARGS
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
NoNewPrivileges=true
|
||||
ProtectHome=true
|
||||
ProtectSystem=full
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
61
ansible/roles/prometheus/README.md
Normal file
61
ansible/roles/prometheus/README.md
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
# prometheus role
|
||||
|
||||
Installs Prometheus from upstream tarballs, configures scraping for the
|
||||
home-cluster Kubernetes API, nodes, cAdvisor, and pods/services with
|
||||
`prometheus.io/scrape: "true"` annotations.
|
||||
|
||||
## One-time Kubernetes bootstrap
|
||||
|
||||
Prometheus runs *outside* the cluster on `prom.w5isp.com`, so it needs a
|
||||
ServiceAccount token + the cluster's CA cert.
|
||||
|
||||
1. Apply the RBAC manifest from your workstation:
|
||||
|
||||
```bash
|
||||
kubectl apply -f roles/prometheus/files/k8s-rbac.yaml
|
||||
```
|
||||
|
||||
2. Pull the token + CA out of the cluster:
|
||||
|
||||
```bash
|
||||
TOKEN=$(kubectl -n prometheus-external get secret prometheus-token \
|
||||
-o jsonpath='{.data.token}' | base64 -d)
|
||||
CA=$(kubectl -n prometheus-external get secret prometheus-token \
|
||||
-o jsonpath='{.data.ca\.crt}' | base64 -d)
|
||||
```
|
||||
|
||||
3. Copy them onto the prom host:
|
||||
|
||||
```bash
|
||||
ssh ansible@10.0.15.31 'sudo install -d -o prometheus -g prometheus -m 0750 /etc/prometheus/k8s'
|
||||
printf '%s' "$TOKEN" | ssh ansible@10.0.15.31 \
|
||||
'sudo tee /etc/prometheus/k8s/token >/dev/null && \
|
||||
sudo chown prometheus:prometheus /etc/prometheus/k8s/token && \
|
||||
sudo chmod 0640 /etc/prometheus/k8s/token'
|
||||
printf '%s' "$CA" | ssh ansible@10.0.15.31 \
|
||||
'sudo tee /etc/prometheus/k8s/ca.crt >/dev/null && \
|
||||
sudo chown prometheus:prometheus /etc/prometheus/k8s/ca.crt && \
|
||||
sudo chmod 0640 /etc/prometheus/k8s/ca.crt'
|
||||
```
|
||||
|
||||
4. Reload Prometheus:
|
||||
|
||||
```bash
|
||||
ssh ansible@10.0.15.31 'sudo systemctl reload prometheus'
|
||||
```
|
||||
|
||||
The token persists across pod restarts (it's a long-lived `kubernetes.io/service-account-token`
|
||||
Secret, not a projected token), so it does not need to be rotated unless you delete the Secret.
|
||||
|
||||
## Verifying scrape targets
|
||||
|
||||
Open `http://10.0.15.31:9090/targets` — the `kubernetes-apiservers`,
|
||||
`kubernetes-nodes`, and `kubernetes-cadvisor` jobs should all be UP.
|
||||
|
||||
If `kubernetes-pods` / `kubernetes-service-endpoints` are empty, that's expected
|
||||
until you annotate workloads with `prometheus.io/scrape: "true"`.
|
||||
|
||||
## Disabling k8s scraping
|
||||
|
||||
Set `prometheus_k8s_enabled: false` in `host_vars/prom.w5isp.com.yml` and re-run
|
||||
the playbook.
|
||||
34
ansible/roles/prometheus/defaults/main.yml
Normal file
34
ansible/roles/prometheus/defaults/main.yml
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
---
|
||||
prometheus_version: "3.6.0"
|
||||
prometheus_arch: amd64
|
||||
|
||||
prometheus_user: prometheus
|
||||
prometheus_group: prometheus
|
||||
|
||||
prometheus_install_dir: /usr/local/bin
|
||||
prometheus_config_dir: /etc/prometheus
|
||||
prometheus_data_dir: /var/lib/prometheus
|
||||
|
||||
prometheus_listen_address: "0.0.0.0:9090"
|
||||
prometheus_retention_time: "30d"
|
||||
prometheus_retention_size: "0" # 0 = unlimited; let retention_time bound it
|
||||
|
||||
prometheus_external_url: ""
|
||||
prometheus_scrape_interval: "30s"
|
||||
prometheus_evaluation_interval: "30s"
|
||||
|
||||
# Path to the Kubernetes ServiceAccount bearer token + CA on the prom host.
|
||||
# Bootstrap copies these from the cluster (see roles/prometheus/README.md).
|
||||
prometheus_k8s_enabled: true
|
||||
prometheus_k8s_api_server: "https://10.0.15.1:6443"
|
||||
prometheus_k8s_token_file: "{{ prometheus_config_dir }}/k8s/token"
|
||||
prometheus_k8s_ca_file: "{{ prometheus_config_dir }}/k8s/ca.crt"
|
||||
|
||||
# Alertmanager target (same host by default)
|
||||
prometheus_alertmanager_targets:
|
||||
- "127.0.0.1:9093"
|
||||
|
||||
# Extra static scrape targets, e.g. other node_exporters on home_servers
|
||||
# - job_name: node_exporter
|
||||
# targets: ["10.0.15.31:9100", "10.0.15.22:9100"]
|
||||
prometheus_extra_scrape_configs: []
|
||||
59
ansible/roles/prometheus/files/k8s-rbac.yaml
Normal file
59
ansible/roles/prometheus/files/k8s-rbac.yaml
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
---
|
||||
# RBAC for an external Prometheus on prom.w5isp.com to scrape home-cluster.
|
||||
# Apply once with `kubectl apply -f k8s-rbac.yaml`.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: prometheus-external
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: prometheus-external
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: prometheus-external
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources:
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
- nodes/metrics
|
||||
- services
|
||||
- endpoints
|
||||
- pods
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups:
|
||||
- extensions
|
||||
- networking.k8s.io
|
||||
resources:
|
||||
- ingresses
|
||||
verbs: ["get", "list", "watch"]
|
||||
- nonResourceURLs: ["/metrics", "/metrics/cadvisor"]
|
||||
verbs: ["get"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: prometheus-external
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: prometheus-external
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: prometheus
|
||||
namespace: prometheus-external
|
||||
---
|
||||
# Long-lived bearer token Secret for the SA (k8s 1.24+ no longer auto-creates one).
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: prometheus-token
|
||||
namespace: prometheus-external
|
||||
annotations:
|
||||
kubernetes.io/service-account.name: prometheus
|
||||
type: kubernetes.io/service-account-token
|
||||
11
ansible/roles/prometheus/handlers/main.yml
Normal file
11
ansible/roles/prometheus/handlers/main.yml
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
---
|
||||
- name: Restart prometheus
|
||||
ansible.builtin.systemd:
|
||||
name: prometheus
|
||||
state: restarted
|
||||
daemon_reload: true
|
||||
|
||||
- name: Reload prometheus
|
||||
ansible.builtin.systemd:
|
||||
name: prometheus
|
||||
state: reloaded
|
||||
128
ansible/roles/prometheus/tasks/main.yml
Normal file
128
ansible/roles/prometheus/tasks/main.yml
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
---
|
||||
# Installs Prometheus from the upstream tarball, configures scraping for the
|
||||
# home-cluster Kubernetes API + nodes + cAdvisor, and runs it under systemd.
|
||||
|
||||
- name: Create prometheus group
|
||||
ansible.builtin.group:
|
||||
name: "{{ prometheus_group }}"
|
||||
system: true
|
||||
state: present
|
||||
|
||||
- name: Create prometheus user
|
||||
ansible.builtin.user:
|
||||
name: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
system: true
|
||||
shell: /usr/sbin/nologin
|
||||
home: "{{ prometheus_data_dir }}"
|
||||
create_home: false
|
||||
state: present
|
||||
|
||||
- name: Create prometheus directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0755"
|
||||
loop:
|
||||
- "{{ prometheus_config_dir }}"
|
||||
- "{{ prometheus_config_dir }}/rules"
|
||||
- "{{ prometheus_config_dir }}/k8s"
|
||||
- "{{ prometheus_data_dir }}"
|
||||
|
||||
- name: Check installed prometheus version
|
||||
ansible.builtin.command: "{{ prometheus_install_dir }}/prometheus --version"
|
||||
register: prometheus_installed
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
|
||||
- name: Decide whether prometheus needs (re)install
|
||||
ansible.builtin.set_fact:
|
||||
prometheus_needs_install: >-
|
||||
{{
|
||||
prometheus_installed.rc != 0
|
||||
or prometheus_version not in (prometheus_installed.stdout | default(''))
|
||||
}}
|
||||
|
||||
- name: Download prometheus tarball
|
||||
ansible.builtin.get_url:
|
||||
url: "https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/prometheus-{{ prometheus_version }}.linux-{{ prometheus_arch }}.tar.gz"
|
||||
dest: "/tmp/prometheus-{{ prometheus_version }}.tar.gz"
|
||||
mode: "0644"
|
||||
when: prometheus_needs_install
|
||||
|
||||
- name: Extract prometheus
|
||||
ansible.builtin.unarchive:
|
||||
src: "/tmp/prometheus-{{ prometheus_version }}.tar.gz"
|
||||
dest: /tmp/
|
||||
remote_src: true
|
||||
creates: "/tmp/prometheus-{{ prometheus_version }}.linux-{{ prometheus_arch }}/prometheus"
|
||||
when: prometheus_needs_install
|
||||
|
||||
- name: Install prometheus binaries
|
||||
ansible.builtin.copy:
|
||||
src: "/tmp/prometheus-{{ prometheus_version }}.linux-{{ prometheus_arch }}/{{ item }}"
|
||||
dest: "{{ prometheus_install_dir }}/{{ item }}"
|
||||
remote_src: true
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
loop:
|
||||
- prometheus
|
||||
- promtool
|
||||
when: prometheus_needs_install
|
||||
notify: Restart prometheus
|
||||
|
||||
- name: Render prometheus.yml
|
||||
ansible.builtin.template:
|
||||
src: prometheus.yml.j2
|
||||
dest: "{{ prometheus_config_dir }}/prometheus.yml"
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0640"
|
||||
validate: "{{ prometheus_install_dir }}/promtool check config %s"
|
||||
notify: Reload prometheus
|
||||
|
||||
- name: Install default alert rules
|
||||
ansible.builtin.template:
|
||||
src: rules.yml.j2
|
||||
dest: "{{ prometheus_config_dir }}/rules/default.yml"
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0640"
|
||||
validate: "{{ prometheus_install_dir }}/promtool check rules %s"
|
||||
notify: Reload prometheus
|
||||
|
||||
- name: Warn if k8s scraping enabled but token file missing
|
||||
ansible.builtin.stat:
|
||||
path: "{{ prometheus_k8s_token_file }}"
|
||||
register: prom_k8s_token_stat
|
||||
when: prometheus_k8s_enabled
|
||||
|
||||
- name: Show k8s bootstrap reminder
|
||||
ansible.builtin.debug:
|
||||
msg: >
|
||||
Kubernetes scraping is enabled but {{ prometheus_k8s_token_file }} is missing.
|
||||
See roles/prometheus/README.md for the bootstrap steps (apply rbac.yaml in-cluster
|
||||
and copy the SA token + CA cert to {{ prometheus_config_dir }}/k8s/).
|
||||
Prometheus will start, but k8s scrape jobs will fail until the token is in place.
|
||||
when:
|
||||
- prometheus_k8s_enabled
|
||||
- not prom_k8s_token_stat.stat.exists
|
||||
|
||||
- name: Install prometheus systemd unit
|
||||
ansible.builtin.template:
|
||||
src: prometheus.service.j2
|
||||
dest: /etc/systemd/system/prometheus.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: Restart prometheus
|
||||
|
||||
- name: Enable and start prometheus
|
||||
ansible.builtin.systemd:
|
||||
name: prometheus
|
||||
enabled: true
|
||||
state: started
|
||||
daemon_reload: true
|
||||
33
ansible/roles/prometheus/templates/prometheus.service.j2
Normal file
33
ansible/roles/prometheus/templates/prometheus.service.j2
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
[Unit]
|
||||
Description=Prometheus
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ prometheus_user }}
|
||||
Group={{ prometheus_group }}
|
||||
ExecStart={{ prometheus_install_dir }}/prometheus \
|
||||
--config.file={{ prometheus_config_dir }}/prometheus.yml \
|
||||
--storage.tsdb.path={{ prometheus_data_dir }} \
|
||||
--storage.tsdb.retention.time={{ prometheus_retention_time }} \
|
||||
{% if prometheus_retention_size != "0" %}
|
||||
--storage.tsdb.retention.size={{ prometheus_retention_size }} \
|
||||
{% endif %}
|
||||
--web.listen-address={{ prometheus_listen_address }} \
|
||||
{% if prometheus_external_url %}
|
||||
--web.external-url={{ prometheus_external_url }} \
|
||||
{% endif %}
|
||||
--web.enable-lifecycle
|
||||
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
NoNewPrivileges=true
|
||||
ProtectHome=true
|
||||
ProtectSystem=full
|
||||
ReadWritePaths={{ prometheus_data_dir }}
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
172
ansible/roles/prometheus/templates/prometheus.yml.j2
Normal file
172
ansible/roles/prometheus/templates/prometheus.yml.j2
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
{{ ansible_managed | comment }}
|
||||
global:
|
||||
scrape_interval: {{ prometheus_scrape_interval }}
|
||||
evaluation_interval: {{ prometheus_evaluation_interval }}
|
||||
external_labels:
|
||||
monitor: prom-w5isp
|
||||
cluster: home-cluster
|
||||
|
||||
rule_files:
|
||||
- {{ prometheus_config_dir }}/rules/*.yml
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
{% for t in prometheus_alertmanager_targets %}
|
||||
- "{{ t }}"
|
||||
{% endfor %}
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["127.0.0.1:9090"]
|
||||
|
||||
- job_name: node_exporter_local
|
||||
static_configs:
|
||||
- targets: ["127.0.0.1:9100"]
|
||||
labels:
|
||||
instance: prom.w5isp.com
|
||||
|
||||
{% if prometheus_k8s_enabled %}
|
||||
# ---- Kubernetes home-cluster scraping ----
|
||||
# Uses a ServiceAccount bearer token + CA cert provisioned out-of-band; see role README.
|
||||
|
||||
- job_name: kubernetes-apiservers
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
api_server: {{ prometheus_k8s_api_server }}
|
||||
tls_config:
|
||||
ca_file: {{ prometheus_k8s_ca_file }}
|
||||
bearer_token_file: {{ prometheus_k8s_token_file }}
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: {{ prometheus_k8s_ca_file }}
|
||||
bearer_token_file: {{ prometheus_k8s_token_file }}
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
||||
action: keep
|
||||
regex: default;kubernetes;https
|
||||
|
||||
- job_name: kubernetes-nodes
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
api_server: {{ prometheus_k8s_api_server }}
|
||||
tls_config:
|
||||
ca_file: {{ prometheus_k8s_ca_file }}
|
||||
bearer_token_file: {{ prometheus_k8s_token_file }}
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: {{ prometheus_k8s_ca_file }}
|
||||
insecure_skip_verify: true
|
||||
bearer_token_file: {{ prometheus_k8s_token_file }}
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
- target_label: __address__
|
||||
replacement: {{ prometheus_k8s_api_server | regex_replace('^https?://', '') }}
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/${1}/proxy/metrics
|
||||
|
||||
- job_name: kubernetes-cadvisor
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
api_server: {{ prometheus_k8s_api_server }}
|
||||
tls_config:
|
||||
ca_file: {{ prometheus_k8s_ca_file }}
|
||||
bearer_token_file: {{ prometheus_k8s_token_file }}
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: {{ prometheus_k8s_ca_file }}
|
||||
insecure_skip_verify: true
|
||||
bearer_token_file: {{ prometheus_k8s_token_file }}
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
- target_label: __address__
|
||||
replacement: {{ prometheus_k8s_api_server | regex_replace('^https?://', '') }}
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
|
||||
|
||||
- job_name: kubernetes-pods
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
api_server: {{ prometheus_k8s_api_server }}
|
||||
tls_config:
|
||||
ca_file: {{ prometheus_k8s_ca_file }}
|
||||
bearer_token_file: {{ prometheus_k8s_token_file }}
|
||||
relabel_configs:
|
||||
# Only scrape pods with prometheus.io/scrape: "true"
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
||||
action: keep
|
||||
regex: "true"
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
||||
action: replace
|
||||
target_label: __metrics_path__
|
||||
regex: (.+)
|
||||
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
target_label: __address__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
action: replace
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
action: replace
|
||||
target_label: pod
|
||||
|
||||
- job_name: kubernetes-service-endpoints
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
api_server: {{ prometheus_k8s_api_server }}
|
||||
tls_config:
|
||||
ca_file: {{ prometheus_k8s_ca_file }}
|
||||
bearer_token_file: {{ prometheus_k8s_token_file }}
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
|
||||
action: keep
|
||||
regex: "true"
|
||||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
|
||||
action: replace
|
||||
target_label: __scheme__
|
||||
regex: (https?)
|
||||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
|
||||
action: replace
|
||||
target_label: __metrics_path__
|
||||
regex: (.+)
|
||||
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
target_label: __address__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_service_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
action: replace
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_service_name]
|
||||
action: replace
|
||||
target_label: service
|
||||
{% endif %}
|
||||
|
||||
{% for sc in prometheus_extra_scrape_configs %}
|
||||
- job_name: {{ sc.job_name }}
|
||||
static_configs:
|
||||
- targets:
|
||||
{% for t in sc.targets %}
|
||||
- "{{ t }}"
|
||||
{% endfor %}
|
||||
{% if sc.labels is defined %}
|
||||
labels:
|
||||
{% for k, v in sc.labels.items() %}
|
||||
{{ k }}: "{{ v }}"
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
32
ansible/roles/prometheus/templates/rules.yml.j2
Normal file
32
ansible/roles/prometheus/templates/rules.yml.j2
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
{{ ansible_managed | comment }}
|
||||
groups:
|
||||
- name: instance-health
|
||||
rules:
|
||||
- alert: InstanceDown
|
||||
expr: up == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Instance {{ '{{' }} $labels.instance {{ '}}' }} down"
|
||||
description: "{{ '{{' }} $labels.instance {{ '}}' }} ({{ '{{' }} $labels.job {{ '}}' }}) has been unreachable for more than 5 minutes."
|
||||
|
||||
- name: node-health
|
||||
rules:
|
||||
- alert: NodeFilesystemAlmostFull
|
||||
expr: |
|
||||
(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} /
|
||||
node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) < 0.10
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Filesystem {{ '{{' }} $labels.mountpoint {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }} below 10%"
|
||||
|
||||
- alert: NodeHighLoad
|
||||
expr: node_load5 > (count by (instance) (node_cpu_seconds_total{mode="idle"}) * 2)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "High load on {{ '{{' }} $labels.instance {{ '}}' }}"
|
||||
Loading…
Add table
Reference in a new issue