From 922552059f64e9542f8d8f675e72c2b5e9be5108 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Fri, 8 May 2026 09:50:31 -0500
Subject: [PATCH] prom: add prometheus stack (prometheus, alertmanager,
 grafana, node_exporter)

Provisions prom.w5isp.com (10.0.15.31) as the home-cluster monitoring host.
Prometheus 3.6.0 scrapes the Talos cluster API/nodes/cAdvisor via an external
SA token; Grafana ships with the Prometheus datasource pre-provisioned.

host_vars opens 9090/9093/9100/3000 to 10.0.0.0/16 plus SSH from the home LAN
so the firewall role doesn't lock out admin access.
---
 ansible/host_vars/prom.w5isp.com.yml          |  35 ++++
 ansible/hosts                                 |   7 +
 ansible/playbook.yml                          |  24 +++
 ansible/roles/alertmanager/defaults/main.yml  |  28 +++
 ansible/roles/alertmanager/handlers/main.yml  |  11 ++
 ansible/roles/alertmanager/tasks/main.yml     |  97 ++++++++++
 .../templates/alertmanager.service.j2         |  33 ++++
 .../templates/alertmanager.yml.j2             |   9 +
 ansible/roles/grafana/defaults/main.yml       |  26 +++
 ansible/roles/grafana/handlers/main.yml       |   6 +
 ansible/roles/grafana/tasks/main.yml          |  60 ++++++
 .../grafana/templates/datasources.yaml.j2     |  18 ++
 .../roles/grafana/templates/grafana.ini.j2    |  21 +++
 ansible/roles/node_exporter/defaults/main.yml |  10 +
 ansible/roles/node_exporter/handlers/main.yml |   6 +
 ansible/roles/node_exporter/tasks/main.yml    |  75 ++++++++
 .../templates/node_exporter.service.j2        |  28 +++
 ansible/roles/prometheus/README.md            |  61 +++++++
 ansible/roles/prometheus/defaults/main.yml    |  34 ++++
 ansible/roles/prometheus/files/k8s-rbac.yaml  |  59 ++++++
 ansible/roles/prometheus/handlers/main.yml    |  11 ++
 ansible/roles/prometheus/tasks/main.yml       | 128 +++++++++++++
 .../templates/prometheus.service.j2           |  33 ++++
 .../prometheus/templates/prometheus.yml.j2    | 172 ++++++++++++++++++
 .../roles/prometheus/templates/rules.yml.j2   |  32 ++++
 25 files changed, 1024 insertions(+)
 create mode 100644 ansible/host_vars/prom.w5isp.com.yml
 create mode 100644 ansible/roles/alertmanager/defaults/main.yml
 create mode 100644 ansible/roles/alertmanager/handlers/main.yml
 create mode 100644 ansible/roles/alertmanager/tasks/main.yml
 create mode 100644 ansible/roles/alertmanager/templates/alertmanager.service.j2
 create mode 100644 ansible/roles/alertmanager/templates/alertmanager.yml.j2
 create mode 100644 ansible/roles/grafana/defaults/main.yml
 create mode 100644 ansible/roles/grafana/handlers/main.yml
 create mode 100644 ansible/roles/grafana/tasks/main.yml
 create mode 100644 ansible/roles/grafana/templates/datasources.yaml.j2
 create mode 100644 ansible/roles/grafana/templates/grafana.ini.j2
 create mode 100644 ansible/roles/node_exporter/defaults/main.yml
 create mode 100644 ansible/roles/node_exporter/handlers/main.yml
 create mode 100644 ansible/roles/node_exporter/tasks/main.yml
 create mode 100644 ansible/roles/node_exporter/templates/node_exporter.service.j2
 create mode 100644 ansible/roles/prometheus/README.md
 create mode 100644 ansible/roles/prometheus/defaults/main.yml
 create mode 100644 ansible/roles/prometheus/files/k8s-rbac.yaml
 create mode 100644 ansible/roles/prometheus/handlers/main.yml
 create mode 100644 ansible/roles/prometheus/tasks/main.yml
 create mode 100644 ansible/roles/prometheus/templates/prometheus.service.j2
 create mode 100644 ansible/roles/prometheus/templates/prometheus.yml.j2
 create mode 100644 ansible/roles/prometheus/templates/rules.yml.j2

diff --git a/ansible/host_vars/prom.w5isp.com.yml b/ansible/host_vars/prom.w5isp.com.yml
new file mode 100644
index 0000000..da72880
--- /dev/null
+++ b/ansible/host_vars/prom.w5isp.com.yml
@@ -0,0 +1,35 @@
+---
+ansible_user: ansible
+ansible_host: 10.0.15.31
+ansible_python_interpreter: /usr/bin/python3
+ansible_ssh_private_key_file: /Users/graham/.ssh/ansible
+ansible_ssh_common_args: '-o StrictHostKeyChecking=accept-new'
+
+network:
+  skip: true
+
+# Open Prometheus stack ports to the home LAN.
+# Trusted subnets (Tailscale, VNTX) are already wide-open via firewall role defaults;
+# this adds the 10.0.15.0/24 home cluster subnet so the k8s nodes can hit node_exporter
+# and so any LAN client can reach the Grafana UI.
+firewall_allow_rules:
+  - port: 22
+    proto: tcp
+    from_ip: 10.0.0.0/16
+    comment: SSH from home LAN
+  - port: 9090
+    proto: tcp
+    from_ip: 10.0.0.0/16
+    comment: Prometheus UI/API
+  - port: 9093
+    proto: tcp
+    from_ip: 10.0.0.0/16
+    comment: Alertmanager
+  - port: 9100
+    proto: tcp
+    from_ip: 10.0.0.0/16
+    comment: node_exporter
+  - port: 3000
+    proto: tcp
+    from_ip: 10.0.0.0/16
+    comment: Grafana UI
diff --git a/ansible/hosts b/ansible/hosts
index 7b38603..1d5007c 100755
--- a/ansible/hosts
+++ b/ansible/hosts
@@ -25,6 +25,13 @@ mail.mcintire.me ansible_host=mail
 dokku.w5isp.com
 aprs.w5isp.com
 staging.towerops.net
+prom.w5isp.com ansible_host=10.0.15.31
+
+[prometheus_servers]
+prom.w5isp.com
+
+[node_exporter_servers:children]
+prometheus_servers
 
 [dokku_servers]
 staging.towerops.net
diff --git a/ansible/playbook.yml b/ansible/playbook.yml
index 3518f30..e635aec 100644
--- a/ansible/playbook.yml
+++ b/ansible/playbook.yml
@@ -308,3 +308,27 @@
     - dokku
   roles:
     - dokku
+
+- name: Configure node_exporter targets
+  hosts: node_exporter_servers
+  become: yes
+  gather_facts: true
+  tags:
+    - node_exporter
+    - prometheus_stack
+  roles:
+    - node_exporter
+
+- name: Configure Prometheus monitoring stack
+  hosts: prometheus_servers
+  become: yes
+  gather_facts: true
+  tags:
+    - prometheus_stack
+  roles:
+    - role: alertmanager
+      tags: alertmanager
+    - role: prometheus
+      tags: prometheus
+    - role: grafana
+      tags: grafana
diff --git a/ansible/roles/alertmanager/defaults/main.yml b/ansible/roles/alertmanager/defaults/main.yml
new file mode 100644
index 0000000..69d3c20
--- /dev/null
+++ b/ansible/roles/alertmanager/defaults/main.yml
@@ -0,0 +1,28 @@
+---
+alertmanager_version: "0.28.1"
+alertmanager_arch: amd64
+
+alertmanager_user: alertmanager
+alertmanager_group: alertmanager
+
+alertmanager_install_dir: /usr/local/bin
+alertmanager_config_dir: /etc/alertmanager
+alertmanager_data_dir: /var/lib/alertmanager
+
+alertmanager_listen_address: "0.0.0.0:9093"
+alertmanager_cluster_listen: ""  # "" disables clustering (single-node)
+alertmanager_external_url: ""
+
+# Default config: a single null receiver (alerts are accepted and discarded).
+# Override `alertmanager_route` and `alertmanager_receivers` to wire up real receivers.
+alertmanager_route:
+  receiver: "null"
+  group_by: ["alertname", "cluster", "service"]
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 3h
+
+alertmanager_receivers:
+  - name: "null"
+
+alertmanager_inhibit_rules: []
diff --git a/ansible/roles/alertmanager/handlers/main.yml b/ansible/roles/alertmanager/handlers/main.yml
new file mode 100644
index 0000000..d825e96
--- /dev/null
+++ b/ansible/roles/alertmanager/handlers/main.yml
@@ -0,0 +1,11 @@
+---
+- name: Restart alertmanager
+  ansible.builtin.systemd:
+    name: alertmanager
+    state: restarted
+    daemon_reload: true
+
+- name: Reload alertmanager
+  ansible.builtin.systemd:
+    name: alertmanager
+    state: reloaded
diff --git a/ansible/roles/alertmanager/tasks/main.yml b/ansible/roles/alertmanager/tasks/main.yml
new file mode 100644
index 0000000..2f77296
--- /dev/null
+++ b/ansible/roles/alertmanager/tasks/main.yml
@@ -0,0 +1,97 @@
+---
+- name: Create alertmanager group
+  ansible.builtin.group:
+    name: "{{ alertmanager_group }}"
+    system: true
+    state: present
+
+- name: Create alertmanager user
+  ansible.builtin.user:
+    name: "{{ alertmanager_user }}"
+    group: "{{ alertmanager_group }}"
+    system: true
+    shell: /usr/sbin/nologin
+    home: "{{ alertmanager_data_dir }}"
+    create_home: false
+    state: present
+
+- name: Create alertmanager directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ alertmanager_user }}"
+    group: "{{ alertmanager_group }}"
+    mode: "0755"
+  loop:
+    - "{{ alertmanager_config_dir }}"
+    - "{{ alertmanager_data_dir }}"
+
+- name: Check installed alertmanager version
+  ansible.builtin.command: "{{ alertmanager_install_dir }}/alertmanager --version"
+  register: alertmanager_installed
+  changed_when: false
+  failed_when: false
+
+- name: Decide whether alertmanager needs (re)install
+  ansible.builtin.set_fact:
+    alertmanager_needs_install: >-
+      {{
+        alertmanager_installed.rc != 0
+        or alertmanager_version not in (alertmanager_installed.stdout | default(''))
+        and alertmanager_version not in (alertmanager_installed.stderr | default(''))
+      }}
+
+- name: Download alertmanager tarball
+  ansible.builtin.get_url:
+    url: "https://github.com/prometheus/alertmanager/releases/download/v{{ alertmanager_version }}/alertmanager-{{ alertmanager_version }}.linux-{{ alertmanager_arch }}.tar.gz"
+    dest: "/tmp/alertmanager-{{ alertmanager_version }}.tar.gz"
+    mode: "0644"
+  when: alertmanager_needs_install
+
+- name: Extract alertmanager
+  ansible.builtin.unarchive:
+    src: "/tmp/alertmanager-{{ alertmanager_version }}.tar.gz"
+    dest: /tmp/
+    remote_src: true
+    creates: "/tmp/alertmanager-{{ alertmanager_version }}.linux-{{ alertmanager_arch }}/alertmanager"
+  when: alertmanager_needs_install
+
+- name: Install alertmanager binaries
+  ansible.builtin.copy:
+    src: "/tmp/alertmanager-{{ alertmanager_version }}.linux-{{ alertmanager_arch }}/{{ item }}"
+    dest: "{{ alertmanager_install_dir }}/{{ item }}"
+    remote_src: true
+    owner: root
+    group: root
+    mode: "0755"
+  loop:
+    - alertmanager
+    - amtool
+  when: alertmanager_needs_install
+  notify: Restart alertmanager
+
+- name: Render alertmanager.yml
+  ansible.builtin.template:
+    src: alertmanager.yml.j2
+    dest: "{{ alertmanager_config_dir }}/alertmanager.yml"
+    owner: "{{ alertmanager_user }}"
+    group: "{{ alertmanager_group }}"
+    mode: "0640"
+    validate: "{{ alertmanager_install_dir }}/amtool check-config %s"
+  notify: Reload alertmanager
+
+- name: Install alertmanager systemd unit
+  ansible.builtin.template:
+    src: alertmanager.service.j2
+    dest: /etc/systemd/system/alertmanager.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart alertmanager
+
+- name: Enable and start alertmanager
+  ansible.builtin.systemd:
+    name: alertmanager
+    enabled: true
+    state: started
+    daemon_reload: true
diff --git a/ansible/roles/alertmanager/templates/alertmanager.service.j2 b/ansible/roles/alertmanager/templates/alertmanager.service.j2
new file mode 100644
index 0000000..2c91640
--- /dev/null
+++ b/ansible/roles/alertmanager/templates/alertmanager.service.j2
@@ -0,0 +1,33 @@
+[Unit]
+Description=Prometheus Alertmanager
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ alertmanager_user }}
+Group={{ alertmanager_group }}
+ExecStart={{ alertmanager_install_dir }}/alertmanager \
+  --config.file={{ alertmanager_config_dir }}/alertmanager.yml \
+  --storage.path={{ alertmanager_data_dir }} \
+  --web.listen-address={{ alertmanager_listen_address }} \
+{% if alertmanager_external_url %}
+  --web.external-url={{ alertmanager_external_url }} \
+{% endif %}
+{% if alertmanager_cluster_listen %}
+  --cluster.listen-address={{ alertmanager_cluster_listen }}
+{% else %}
+  --cluster.listen-address=
+{% endif %}
+
+ExecReload=/bin/kill -HUP $MAINPID
+
+Restart=on-failure
+RestartSec=5
+NoNewPrivileges=true
+ProtectHome=true
+ProtectSystem=full
+ReadWritePaths={{ alertmanager_data_dir }}
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ansible/roles/alertmanager/templates/alertmanager.yml.j2 b/ansible/roles/alertmanager/templates/alertmanager.yml.j2
new file mode 100644
index 0000000..b0eb177
--- /dev/null
+++ b/ansible/roles/alertmanager/templates/alertmanager.yml.j2
@@ -0,0 +1,9 @@
+{{ ansible_managed | comment }}
+route:
+{{ alertmanager_route | to_nice_yaml(indent=2) | indent(2, true) }}
+receivers:
+{{ alertmanager_receivers | to_nice_yaml(indent=2) | indent(2, true) }}
+{% if alertmanager_inhibit_rules %}
+inhibit_rules:
+{{ alertmanager_inhibit_rules | to_nice_yaml(indent=2) | indent(2, true) }}
+{% endif %}
diff --git a/ansible/roles/grafana/defaults/main.yml b/ansible/roles/grafana/defaults/main.yml
new file mode 100644
index 0000000..4763b9d
--- /dev/null
+++ b/ansible/roles/grafana/defaults/main.yml
@@ -0,0 +1,26 @@
+---
+grafana_apt_repo: "deb https://apt.grafana.com stable main"
+grafana_apt_key_url: "https://apt.grafana.com/gpg.key"
+grafana_apt_keyring: "/etc/apt/keyrings/grafana.gpg"
+
+grafana_listen_address: "0.0.0.0"
+grafana_http_port: 3000
+grafana_root_url: "http://10.0.15.31:3000/"
+
+grafana_admin_user: admin
+# Set in vault / host_vars; default kept for first-boot only.
+grafana_admin_password: admin
+
+# Datasources provisioned at startup. The first one is marked default.
+grafana_datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: "http://127.0.0.1:9090"
+    isDefault: true
+  - name: Alertmanager
+    type: alertmanager
+    access: proxy
+    url: "http://127.0.0.1:9093"
+    jsonData:
+      implementation: prometheus
diff --git a/ansible/roles/grafana/handlers/main.yml b/ansible/roles/grafana/handlers/main.yml
new file mode 100644
index 0000000..f3a1a8d
--- /dev/null
+++ b/ansible/roles/grafana/handlers/main.yml
@@ -0,0 +1,6 @@
+---
+- name: Restart grafana
+  ansible.builtin.systemd:
+    name: grafana-server
+    state: restarted
+    daemon_reload: true
diff --git a/ansible/roles/grafana/tasks/main.yml b/ansible/roles/grafana/tasks/main.yml
new file mode 100644
index 0000000..175dbcb
--- /dev/null
+++ b/ansible/roles/grafana/tasks/main.yml
@@ -0,0 +1,60 @@
+---
+- name: Ensure /etc/apt/keyrings exists
+  ansible.builtin.file:
+    path: /etc/apt/keyrings
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Install prerequisites
+  ansible.builtin.apt:
+    name:
+      - apt-transport-https
+      - gnupg
+    state: present
+    update_cache: true
+
+- name: Add Grafana apt signing key
+  ansible.builtin.get_url:
+    url: "{{ grafana_apt_key_url }}"
+    dest: "{{ grafana_apt_keyring }}"
+    mode: "0644"
+    force: false
+
+- name: Add Grafana apt repository
+  ansible.builtin.apt_repository:
+    repo: "deb [signed-by={{ grafana_apt_keyring }}] https://apt.grafana.com stable main"
+    filename: grafana
+    state: present
+    update_cache: true
+
+- name: Install Grafana OSS
+  ansible.builtin.apt:
+    name: grafana
+    state: present
+
+- name: Configure grafana.ini
+  ansible.builtin.template:
+    src: grafana.ini.j2
+    dest: /etc/grafana/grafana.ini
+    owner: root
+    group: grafana
+    mode: "0640"
+  notify: Restart grafana
+
+- name: Provision Prometheus datasource
+  ansible.builtin.template:
+    src: datasources.yaml.j2
+    dest: /etc/grafana/provisioning/datasources/prometheus.yaml
+    owner: root
+    group: grafana
+    mode: "0640"
+  notify: Restart grafana
+
+- name: Enable and start grafana-server
+  ansible.builtin.systemd:
+    name: grafana-server
+    enabled: true
+    state: started
+    daemon_reload: true
diff --git a/ansible/roles/grafana/templates/datasources.yaml.j2 b/ansible/roles/grafana/templates/datasources.yaml.j2
new file mode 100644
index 0000000..56b4803
--- /dev/null
+++ b/ansible/roles/grafana/templates/datasources.yaml.j2
@@ -0,0 +1,18 @@
+{{ ansible_managed | comment }}
+apiVersion: 1
+
+datasources:
+{% for ds in grafana_datasources %}
+  - name: {{ ds.name }}
+    type: {{ ds.type }}
+    access: {{ ds.access | default('proxy') }}
+    url: {{ ds.url }}
+    isDefault: {{ ds.isDefault | default(false) | string | lower }}
+    editable: true
+{% if ds.jsonData is defined %}
+    jsonData:
+{% for k, v in ds.jsonData.items() %}
+      {{ k }}: {{ v }}
+{% endfor %}
+{% endif %}
+{% endfor %}
diff --git a/ansible/roles/grafana/templates/grafana.ini.j2 b/ansible/roles/grafana/templates/grafana.ini.j2
new file mode 100644
index 0000000..71485d1
--- /dev/null
+++ b/ansible/roles/grafana/templates/grafana.ini.j2
@@ -0,0 +1,21 @@
+{{ ansible_managed | comment }}
+[server]
+http_addr = {{ grafana_listen_address }}
+http_port = {{ grafana_http_port }}
+root_url = {{ grafana_root_url }}
+
+[security]
+admin_user = {{ grafana_admin_user }}
+admin_password = {{ grafana_admin_password }}
+
+[analytics]
+reporting_enabled = false
+check_for_updates = false
+check_for_plugin_updates = false
+
+[users]
+allow_sign_up = false
+
+[log]
+mode = console file
+level = info
diff --git a/ansible/roles/node_exporter/defaults/main.yml b/ansible/roles/node_exporter/defaults/main.yml
new file mode 100644
index 0000000..b7d6369
--- /dev/null
+++ b/ansible/roles/node_exporter/defaults/main.yml
@@ -0,0 +1,10 @@
+---
+node_exporter_version: "1.9.1"
+node_exporter_arch: amd64
+node_exporter_user: node_exporter
+node_exporter_group: node_exporter
+node_exporter_install_dir: /usr/local/bin
+node_exporter_listen_address: "0.0.0.0:9100"
+
+# Default --collector.* flags. Override per-host if needed.
+node_exporter_extra_args: []
diff --git a/ansible/roles/node_exporter/handlers/main.yml b/ansible/roles/node_exporter/handlers/main.yml
new file mode 100644
index 0000000..a162da6
--- /dev/null
+++ b/ansible/roles/node_exporter/handlers/main.yml
@@ -0,0 +1,6 @@
+---
+- name: Restart node_exporter
+  ansible.builtin.systemd:
+    name: node_exporter
+    state: restarted
+    daemon_reload: true
diff --git a/ansible/roles/node_exporter/tasks/main.yml b/ansible/roles/node_exporter/tasks/main.yml
new file mode 100644
index 0000000..22bcf33
--- /dev/null
+++ b/ansible/roles/node_exporter/tasks/main.yml
@@ -0,0 +1,75 @@
+---
+# Installs Prometheus node_exporter from the upstream tarball and runs it under systemd.
+
+- name: Create node_exporter group
+  ansible.builtin.group:
+    name: "{{ node_exporter_group }}"
+    system: true
+    state: present
+
+- name: Create node_exporter user
+  ansible.builtin.user:
+    name: "{{ node_exporter_user }}"
+    group: "{{ node_exporter_group }}"
+    system: true
+    shell: /usr/sbin/nologin
+    home: /nonexistent
+    create_home: false
+    state: present
+
+- name: Check installed node_exporter version
+  ansible.builtin.command: "{{ node_exporter_install_dir }}/node_exporter --version"
+  register: node_exporter_installed
+  changed_when: false
+  failed_when: false
+
+- name: Set fact for installed version
+  ansible.builtin.set_fact:
+    node_exporter_needs_install: >-
+      {{
+        node_exporter_installed.rc != 0
+        or (node_exporter_version not in (node_exporter_installed.stderr | default('')))
+        and (node_exporter_version not in (node_exporter_installed.stdout | default('')))
+      }}
+
+- name: Download node_exporter tarball
+  ansible.builtin.get_url:
+    url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-{{ node_exporter_arch }}.tar.gz"
+    dest: "/tmp/node_exporter-{{ node_exporter_version }}.tar.gz"
+    mode: "0644"
+  when: node_exporter_needs_install
+
+- name: Extract node_exporter
+  ansible.builtin.unarchive:
+    src: "/tmp/node_exporter-{{ node_exporter_version }}.tar.gz"
+    dest: /tmp/
+    remote_src: true
+    creates: "/tmp/node_exporter-{{ node_exporter_version }}.linux-{{ node_exporter_arch }}/node_exporter"
+  when: node_exporter_needs_install
+
+- name: Install node_exporter binary
+  ansible.builtin.copy:
+    src: "/tmp/node_exporter-{{ node_exporter_version }}.linux-{{ node_exporter_arch }}/node_exporter"
+    dest: "{{ node_exporter_install_dir }}/node_exporter"
+    remote_src: true
+    owner: root
+    group: root
+    mode: "0755"
+  when: node_exporter_needs_install
+  notify: Restart node_exporter
+
+- name: Install node_exporter systemd unit
+  ansible.builtin.template:
+    src: node_exporter.service.j2
+    dest: /etc/systemd/system/node_exporter.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart node_exporter
+
+- name: Enable and start node_exporter
+  ansible.builtin.systemd:
+    name: node_exporter
+    enabled: true
+    state: started
+    daemon_reload: true
diff --git a/ansible/roles/node_exporter/templates/node_exporter.service.j2 b/ansible/roles/node_exporter/templates/node_exporter.service.j2
new file mode 100644
index 0000000..d605e62
--- /dev/null
+++ b/ansible/roles/node_exporter/templates/node_exporter.service.j2
@@ -0,0 +1,28 @@
+[Unit]
+Description=Prometheus node_exporter
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ node_exporter_user }}
+Group={{ node_exporter_group }}
+ExecStart={{ node_exporter_install_dir }}/node_exporter \
+  --web.listen-address={{ node_exporter_listen_address }} \
+{% for arg in node_exporter_extra_args %}
+  {{ arg }} \
+{% endfor %}
+  $NODE_EXPORTER_EXTRA_ARGS
+
+Restart=on-failure
+RestartSec=5
+NoNewPrivileges=true
+ProtectHome=true
+ProtectSystem=full
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ansible/roles/prometheus/README.md b/ansible/roles/prometheus/README.md
new file mode 100644
index 0000000..c04635e
--- /dev/null
+++ b/ansible/roles/prometheus/README.md
@@ -0,0 +1,61 @@
+# prometheus role
+
+Installs Prometheus from upstream tarballs, configures scraping for the
+home-cluster Kubernetes API, nodes, cAdvisor, and pods/services with
+`prometheus.io/scrape: "true"` annotations.
+
+## One-time Kubernetes bootstrap
+
+Prometheus runs *outside* the cluster on `prom.w5isp.com`, so it needs a
+ServiceAccount token + the cluster's CA cert.
+
+1. Apply the RBAC manifest from your workstation:
+
+   ```bash
+   kubectl apply -f roles/prometheus/files/k8s-rbac.yaml
+   ```
+
+2. Pull the token + CA out of the cluster:
+
+   ```bash
+   TOKEN=$(kubectl -n prometheus-external get secret prometheus-token \
+     -o jsonpath='{.data.token}' | base64 -d)
+   CA=$(kubectl -n prometheus-external get secret prometheus-token \
+     -o jsonpath='{.data.ca\.crt}' | base64 -d)
+   ```
+
+3. Copy them onto the prom host:
+
+   ```bash
+   ssh ansible@10.0.15.31 'sudo install -d -o prometheus -g prometheus -m 0750 /etc/prometheus/k8s'
+   printf '%s' "$TOKEN" | ssh ansible@10.0.15.31 \
+     'sudo tee /etc/prometheus/k8s/token >/dev/null && \
+      sudo chown prometheus:prometheus /etc/prometheus/k8s/token && \
+      sudo chmod 0640 /etc/prometheus/k8s/token'
+   printf '%s' "$CA" | ssh ansible@10.0.15.31 \
+     'sudo tee /etc/prometheus/k8s/ca.crt >/dev/null && \
+      sudo chown prometheus:prometheus /etc/prometheus/k8s/ca.crt && \
+      sudo chmod 0640 /etc/prometheus/k8s/ca.crt'
+   ```
+
+4. Reload Prometheus:
+
+   ```bash
+   ssh ansible@10.0.15.31 'sudo systemctl reload prometheus'
+   ```
+
+The token persists across pod restarts (it's a long-lived `kubernetes.io/service-account-token`
+Secret, not a projected token), so it does not need to be rotated unless you delete the Secret.
+
+## Verifying scrape targets
+
+Open `http://10.0.15.31:9090/targets` — the `kubernetes-apiservers`,
+`kubernetes-nodes`, and `kubernetes-cadvisor` jobs should all be UP.
+
+If `kubernetes-pods` / `kubernetes-service-endpoints` are empty, that's expected
+until you annotate workloads with `prometheus.io/scrape: "true"`.
+
+## Disabling k8s scraping
+
+Set `prometheus_k8s_enabled: false` in `host_vars/prom.w5isp.com.yml` and re-run
+the playbook.
diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml
new file mode 100644
index 0000000..bcbe997
--- /dev/null
+++ b/ansible/roles/prometheus/defaults/main.yml
@@ -0,0 +1,34 @@
+---
+prometheus_version: "3.6.0"
+prometheus_arch: amd64
+
+prometheus_user: prometheus
+prometheus_group: prometheus
+
+prometheus_install_dir: /usr/local/bin
+prometheus_config_dir: /etc/prometheus
+prometheus_data_dir: /var/lib/prometheus
+
+prometheus_listen_address: "0.0.0.0:9090"
+prometheus_retention_time: "30d"
+prometheus_retention_size: "0"  # 0 = unlimited; let retention_time bound it
+
+prometheus_external_url: ""
+prometheus_scrape_interval: "30s"
+prometheus_evaluation_interval: "30s"
+
+# Path to the Kubernetes ServiceAccount bearer token + CA on the prom host.
+# Bootstrap copies these from the cluster (see roles/prometheus/README.md).
+prometheus_k8s_enabled: true
+prometheus_k8s_api_server: "https://10.0.15.1:6443"
+prometheus_k8s_token_file: "{{ prometheus_config_dir }}/k8s/token"
+prometheus_k8s_ca_file: "{{ prometheus_config_dir }}/k8s/ca.crt"
+
+# Alertmanager target (same host by default)
+prometheus_alertmanager_targets:
+  - "127.0.0.1:9093"
+
+# Extra static scrape targets, e.g. other node_exporters on home_servers
+# - job_name: node_exporter
+#   targets: ["10.0.15.31:9100", "10.0.15.22:9100"]
+prometheus_extra_scrape_configs: []
diff --git a/ansible/roles/prometheus/files/k8s-rbac.yaml b/ansible/roles/prometheus/files/k8s-rbac.yaml
new file mode 100644
index 0000000..4231f7f
--- /dev/null
+++ b/ansible/roles/prometheus/files/k8s-rbac.yaml
@@ -0,0 +1,59 @@
+---
+# RBAC for an external Prometheus on prom.w5isp.com to scrape home-cluster.
+# Apply once with `kubectl apply -f k8s-rbac.yaml`.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: prometheus-external
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus
+  namespace: prometheus-external
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus-external
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - nodes/proxy
+      - nodes/metrics
+      - services
+      - endpoints
+      - pods
+    verbs: ["get", "list", "watch"]
+  - apiGroups:
+      - extensions
+      - networking.k8s.io
+    resources:
+      - ingresses
+    verbs: ["get", "list", "watch"]
+  - nonResourceURLs: ["/metrics", "/metrics/cadvisor"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus-external
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus-external
+subjects:
+  - kind: ServiceAccount
+    name: prometheus
+    namespace: prometheus-external
+---
+# Long-lived bearer token Secret for the SA (k8s 1.24+ no longer auto-creates one).
+apiVersion: v1
+kind: Secret
+metadata:
+  name: prometheus-token
+  namespace: prometheus-external
+  annotations:
+    kubernetes.io/service-account.name: prometheus
+type: kubernetes.io/service-account-token
diff --git a/ansible/roles/prometheus/handlers/main.yml b/ansible/roles/prometheus/handlers/main.yml
new file mode 100644
index 0000000..ad2cdb4
--- /dev/null
+++ b/ansible/roles/prometheus/handlers/main.yml
@@ -0,0 +1,11 @@
+---
+- name: Restart prometheus
+  ansible.builtin.systemd:
+    name: prometheus
+    state: restarted
+    daemon_reload: true
+
+- name: Reload prometheus
+  ansible.builtin.systemd:
+    name: prometheus
+    state: reloaded
diff --git a/ansible/roles/prometheus/tasks/main.yml b/ansible/roles/prometheus/tasks/main.yml
new file mode 100644
index 0000000..5d14b29
--- /dev/null
+++ b/ansible/roles/prometheus/tasks/main.yml
@@ -0,0 +1,128 @@
+---
+# Installs Prometheus from the upstream tarball, configures scraping for the
+# home-cluster Kubernetes API + nodes + cAdvisor, and runs it under systemd.
+
+- name: Create prometheus group
+  ansible.builtin.group:
+    name: "{{ prometheus_group }}"
+    system: true
+    state: present
+
+- name: Create prometheus user
+  ansible.builtin.user:
+    name: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    system: true
+    shell: /usr/sbin/nologin
+    home: "{{ prometheus_data_dir }}"
+    create_home: false
+    state: present
+
+- name: Create prometheus directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0755"
+  loop:
+    - "{{ prometheus_config_dir }}"
+    - "{{ prometheus_config_dir }}/rules"
+    - "{{ prometheus_config_dir }}/k8s"
+    - "{{ prometheus_data_dir }}"
+
+- name: Check installed prometheus version
+  ansible.builtin.command: "{{ prometheus_install_dir }}/prometheus --version"
+  register: prometheus_installed
+  changed_when: false
+  failed_when: false
+
+- name: Decide whether prometheus needs (re)install
+  ansible.builtin.set_fact:
+    prometheus_needs_install: >-
+      {{
+        prometheus_installed.rc != 0
+        or prometheus_version not in (prometheus_installed.stdout | default(''))
+      }}
+
+- name: Download prometheus tarball
+  ansible.builtin.get_url:
+    url: "https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/prometheus-{{ prometheus_version }}.linux-{{ prometheus_arch }}.tar.gz"
+    dest: "/tmp/prometheus-{{ prometheus_version }}.tar.gz"
+    mode: "0644"
+  when: prometheus_needs_install
+
+- name: Extract prometheus
+  ansible.builtin.unarchive:
+    src: "/tmp/prometheus-{{ prometheus_version }}.tar.gz"
+    dest: /tmp/
+    remote_src: true
+    creates: "/tmp/prometheus-{{ prometheus_version }}.linux-{{ prometheus_arch }}/prometheus"
+  when: prometheus_needs_install
+
+- name: Install prometheus binaries
+  ansible.builtin.copy:
+    src: "/tmp/prometheus-{{ prometheus_version }}.linux-{{ prometheus_arch }}/{{ item }}"
+    dest: "{{ prometheus_install_dir }}/{{ item }}"
+    remote_src: true
+    owner: root
+    group: root
+    mode: "0755"
+  loop:
+    - prometheus
+    - promtool
+  when: prometheus_needs_install
+  notify: Restart prometheus
+
+- name: Render prometheus.yml
+  ansible.builtin.template:
+    src: prometheus.yml.j2
+    dest: "{{ prometheus_config_dir }}/prometheus.yml"
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0640"
+    validate: "{{ prometheus_install_dir }}/promtool check config %s"
+  notify: Reload prometheus
+
+- name: Install default alert rules
+  ansible.builtin.template:
+    src: rules.yml.j2
+    dest: "{{ prometheus_config_dir }}/rules/default.yml"
+    owner: "{{ prometheus_user }}"
+    group: "{{ prometheus_group }}"
+    mode: "0640"
+    validate: "{{ prometheus_install_dir }}/promtool check rules %s"
+  notify: Reload prometheus
+
+- name: Warn if k8s scraping enabled but token file missing
+  ansible.builtin.stat:
+    path: "{{ prometheus_k8s_token_file }}"
+  register: prom_k8s_token_stat
+  when: prometheus_k8s_enabled
+
+- name: Show k8s bootstrap reminder
+  ansible.builtin.debug:
+    msg: >
+      Kubernetes scraping is enabled but {{ prometheus_k8s_token_file }} is missing.
+      See roles/prometheus/README.md for the bootstrap steps (apply rbac.yaml in-cluster
+      and copy the SA token + CA cert to {{ prometheus_config_dir }}/k8s/).
+      Prometheus will start, but k8s scrape jobs will fail until the token is in place.
+  when:
+    - prometheus_k8s_enabled
+    - not prom_k8s_token_stat.stat.exists
+
+- name: Install prometheus systemd unit
+  ansible.builtin.template:
+    src: prometheus.service.j2
+    dest: /etc/systemd/system/prometheus.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart prometheus
+
+- name: Enable and start prometheus
+  ansible.builtin.systemd:
+    name: prometheus
+    enabled: true
+    state: started
+    daemon_reload: true
diff --git a/ansible/roles/prometheus/templates/prometheus.service.j2 b/ansible/roles/prometheus/templates/prometheus.service.j2
new file mode 100644
index 0000000..8ec6fde
--- /dev/null
+++ b/ansible/roles/prometheus/templates/prometheus.service.j2
@@ -0,0 +1,33 @@
+[Unit]
+Description=Prometheus
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ prometheus_user }}
+Group={{ prometheus_group }}
+ExecStart={{ prometheus_install_dir }}/prometheus \
+  --config.file={{ prometheus_config_dir }}/prometheus.yml \
+  --storage.tsdb.path={{ prometheus_data_dir }} \
+  --storage.tsdb.retention.time={{ prometheus_retention_time }} \
+{% if prometheus_retention_size != "0" %}
+  --storage.tsdb.retention.size={{ prometheus_retention_size }} \
+{% endif %}
+  --web.listen-address={{ prometheus_listen_address }} \
+{% if prometheus_external_url %}
+  --web.external-url={{ prometheus_external_url }} \
+{% endif %}
+  --web.enable-lifecycle
+
+ExecReload=/bin/kill -HUP $MAINPID
+
+Restart=on-failure
+RestartSec=5
+NoNewPrivileges=true
+ProtectHome=true
+ProtectSystem=full
+ReadWritePaths={{ prometheus_data_dir }}
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ansible/roles/prometheus/templates/prometheus.yml.j2 b/ansible/roles/prometheus/templates/prometheus.yml.j2
new file mode 100644
index 0000000..271516a
--- /dev/null
+++ b/ansible/roles/prometheus/templates/prometheus.yml.j2
@@ -0,0 +1,172 @@
+{{ ansible_managed | comment }}
+global:
+  scrape_interval: {{ prometheus_scrape_interval }}
+  evaluation_interval: {{ prometheus_evaluation_interval }}
+  external_labels:
+    monitor: prom-w5isp
+    cluster: home-cluster
+
+rule_files:
+  - {{ prometheus_config_dir }}/rules/*.yml
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+{% for t in prometheus_alertmanager_targets %}
+            - "{{ t }}"
+{% endfor %}
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ["127.0.0.1:9090"]
+
+  - job_name: node_exporter_local
+    static_configs:
+      - targets: ["127.0.0.1:9100"]
+        labels:
+          instance: prom.w5isp.com
+
+{% if prometheus_k8s_enabled %}
+  # ---- Kubernetes home-cluster scraping ----
+  # Uses a ServiceAccount bearer token + CA cert provisioned out-of-band; see role README.
+
+  - job_name: kubernetes-apiservers
+    kubernetes_sd_configs:
+      - role: endpoints
+        api_server: {{ prometheus_k8s_api_server }}
+        tls_config:
+          ca_file: {{ prometheus_k8s_ca_file }}
+        bearer_token_file: {{ prometheus_k8s_token_file }}
+    scheme: https
+    tls_config:
+      ca_file: {{ prometheus_k8s_ca_file }}
+    bearer_token_file: {{ prometheus_k8s_token_file }}
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+        action: keep
+        regex: default;kubernetes;https
+
+  - job_name: kubernetes-nodes
+    kubernetes_sd_configs:
+      - role: node
+        api_server: {{ prometheus_k8s_api_server }}
+        tls_config:
+          ca_file: {{ prometheus_k8s_ca_file }}
+        bearer_token_file: {{ prometheus_k8s_token_file }}
+    scheme: https
+    tls_config:
+      ca_file: {{ prometheus_k8s_ca_file }}
+      insecure_skip_verify: true
+    bearer_token_file: {{ prometheus_k8s_token_file }}
+    relabel_configs:
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      - target_label: __address__
+        replacement: {{ prometheus_k8s_api_server | regex_replace('^https?://', '') }}
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/${1}/proxy/metrics
+
+  - job_name: kubernetes-cadvisor
+    kubernetes_sd_configs:
+      - role: node
+        api_server: {{ prometheus_k8s_api_server }}
+        tls_config:
+          ca_file: {{ prometheus_k8s_ca_file }}
+        bearer_token_file: {{ prometheus_k8s_token_file }}
+    scheme: https
+    tls_config:
+      ca_file: {{ prometheus_k8s_ca_file }}
+      insecure_skip_verify: true
+    bearer_token_file: {{ prometheus_k8s_token_file }}
+    relabel_configs:
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      - target_label: __address__
+        replacement: {{ prometheus_k8s_api_server | regex_replace('^https?://', '') }}
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+
+  - job_name: kubernetes-pods
+    kubernetes_sd_configs:
+      - role: pod
+        api_server: {{ prometheus_k8s_api_server }}
+        tls_config:
+          ca_file: {{ prometheus_k8s_ca_file }}
+        bearer_token_file: {{ prometheus_k8s_token_file }}
+    relabel_configs:
+      # Only scrape pods with prometheus.io/scrape: "true"
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+        action: keep
+        regex: "true"
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+      - action: labelmap
+        regex: __meta_kubernetes_pod_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        target_label: namespace
+      - source_labels: [__meta_kubernetes_pod_name]
+        action: replace
+        target_label: pod
+
+  - job_name: kubernetes-service-endpoints
+    kubernetes_sd_configs:
+      - role: endpoints
+        api_server: {{ prometheus_k8s_api_server }}
+        tls_config:
+          ca_file: {{ prometheus_k8s_ca_file }}
+        bearer_token_file: {{ prometheus_k8s_token_file }}
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+        action: keep
+        regex: "true"
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
+        action: replace
+        target_label: __scheme__
+        regex: (https?)
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+      - action: labelmap
+        regex: __meta_kubernetes_service_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        target_label: namespace
+      - source_labels: [__meta_kubernetes_service_name]
+        action: replace
+        target_label: service
+{% endif %}
+
+{% for sc in prometheus_extra_scrape_configs %}
+  - job_name: {{ sc.job_name }}
+    static_configs:
+      - targets:
+{% for t in sc.targets %}
+          - "{{ t }}"
+{% endfor %}
+{% if sc.labels is defined %}
+        labels:
+{% for k, v in sc.labels.items() %}
+          {{ k }}: "{{ v }}"
+{% endfor %}
+{% endif %}
+{% endfor %}
diff --git a/ansible/roles/prometheus/templates/rules.yml.j2 b/ansible/roles/prometheus/templates/rules.yml.j2
new file mode 100644
index 0000000..d576715
--- /dev/null
+++ b/ansible/roles/prometheus/templates/rules.yml.j2
@@ -0,0 +1,32 @@
+{{ ansible_managed | comment }}
+groups:
+  - name: instance-health
+    rules:
+      - alert: InstanceDown
+        expr: up == 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Instance {{ '{{' }} $labels.instance {{ '}}' }} down"
+          description: "{{ '{{' }} $labels.instance {{ '}}' }} ({{ '{{' }} $labels.job {{ '}}' }}) has been unreachable for more than 5 minutes."
+
+  - name: node-health
+    rules:
+      - alert: NodeFilesystemAlmostFull
+        expr: |
+          (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} /
+           node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) < 0.10
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Filesystem {{ '{{' }} $labels.mountpoint {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }} below 10%"
+
+      - alert: NodeHighLoad
+        expr: node_load5 > (count by (instance) (node_cpu_seconds_total{mode="idle"}) * 2)
+        for: 15m
+        labels:
+          severity: info
+        annotations:
+          summary: "High load on {{ '{{' }} $labels.instance {{ '}}' }}"