Graham McIntire
e1274bc63b
Cluster-side: - Deploy kube-state-metrics (kube-system) for cluster-state series. - Deploy node-exporter DaemonSet on all 7 nodes via the new monitoring ns (privileged PSS, hostNetwork, read-only host mounts). - Both managed by ArgoCD apps under home/cluster/argocd/apps/. Prometheus-side: - Replace direct pod-IP scraping (which can't reach the 10.244.0.0/16 pod network from outside the cluster) with API-server-proxy scraping for both Services and Pods. Now one target per pod, all healthy. - Extend external Prometheus RBAC with services/proxy + pods/proxy. Grafana-side: - Add file-based dashboard provisioner; ship 5 standard dashboards (Node Exporter Full, k8s cluster-via-prometheus, k8s all-in-one, k8s monitoring, Prometheus 2.0 overview). prom.w5isp.com:3000 now shows live data for all 7 cluster nodes plus the prom host itself.
209 lines
7.6 KiB
Django/Jinja
209 lines
7.6 KiB
Django/Jinja
{{ ansible_managed | comment }}
|
|
global:
|
|
scrape_interval: {{ prometheus_scrape_interval }}
|
|
evaluation_interval: {{ prometheus_evaluation_interval }}
|
|
external_labels:
|
|
monitor: prom-w5isp
|
|
cluster: home-cluster
|
|
|
|
rule_files:
|
|
- {{ prometheus_config_dir }}/rules/*.yml
|
|
|
|
alerting:
|
|
alertmanagers:
|
|
- static_configs:
|
|
- targets:
|
|
{% for t in prometheus_alertmanager_targets %}
|
|
- "{{ t }}"
|
|
{% endfor %}
|
|
|
|
scrape_configs:
|
|
- job_name: prometheus
|
|
static_configs:
|
|
- targets: ["127.0.0.1:9090"]
|
|
|
|
- job_name: node_exporter_local
|
|
static_configs:
|
|
- targets: ["127.0.0.1:9100"]
|
|
labels:
|
|
instance: prom.w5isp.com
|
|
|
|
{% if prometheus_k8s_enabled %}
|
|
# ---- Kubernetes home-cluster scraping ----
|
|
# Uses a ServiceAccount bearer token + CA cert provisioned out-of-band; see role README.
|
|
|
|
- job_name: kubernetes-apiservers
|
|
kubernetes_sd_configs:
|
|
- role: endpoints
|
|
api_server: {{ prometheus_k8s_api_server }}
|
|
tls_config:
|
|
ca_file: {{ prometheus_k8s_ca_file }}
|
|
bearer_token_file: {{ prometheus_k8s_token_file }}
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: {{ prometheus_k8s_ca_file }}
|
|
bearer_token_file: {{ prometheus_k8s_token_file }}
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
|
action: keep
|
|
regex: default;kubernetes;https
|
|
|
|
- job_name: kubernetes-nodes
|
|
kubernetes_sd_configs:
|
|
- role: node
|
|
api_server: {{ prometheus_k8s_api_server }}
|
|
tls_config:
|
|
ca_file: {{ prometheus_k8s_ca_file }}
|
|
bearer_token_file: {{ prometheus_k8s_token_file }}
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: {{ prometheus_k8s_ca_file }}
|
|
insecure_skip_verify: true
|
|
bearer_token_file: {{ prometheus_k8s_token_file }}
|
|
relabel_configs:
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_node_label_(.+)
|
|
- target_label: __address__
|
|
replacement: {{ prometheus_k8s_api_server | regex_replace('^https?://', '') }}
|
|
- source_labels: [__meta_kubernetes_node_name]
|
|
regex: (.+)
|
|
target_label: __metrics_path__
|
|
replacement: /api/v1/nodes/${1}/proxy/metrics
|
|
|
|
- job_name: kubernetes-cadvisor
|
|
kubernetes_sd_configs:
|
|
- role: node
|
|
api_server: {{ prometheus_k8s_api_server }}
|
|
tls_config:
|
|
ca_file: {{ prometheus_k8s_ca_file }}
|
|
bearer_token_file: {{ prometheus_k8s_token_file }}
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: {{ prometheus_k8s_ca_file }}
|
|
insecure_skip_verify: true
|
|
bearer_token_file: {{ prometheus_k8s_token_file }}
|
|
relabel_configs:
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_node_label_(.+)
|
|
- target_label: __address__
|
|
replacement: {{ prometheus_k8s_api_server | regex_replace('^https?://', '') }}
|
|
- source_labels: [__meta_kubernetes_node_name]
|
|
regex: (.+)
|
|
target_label: __metrics_path__
|
|
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
|
|
|
|
# Scrape annotated pods via the API server proxy (one target per pod).
|
|
# Requires `prometheus.io/scrape: "true"` and `prometheus.io/port: "<n>"` on the Pod.
|
|
- job_name: kubernetes-pods
|
|
kubernetes_sd_configs:
|
|
- role: pod
|
|
api_server: {{ prometheus_k8s_api_server }}
|
|
tls_config:
|
|
ca_file: {{ prometheus_k8s_ca_file }}
|
|
bearer_token_file: {{ prometheus_k8s_token_file }}
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: {{ prometheus_k8s_ca_file }}
|
|
insecure_skip_verify: true
|
|
bearer_token_file: {{ prometheus_k8s_token_file }}
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
|
action: keep
|
|
regex: "true"
|
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
|
|
action: keep
|
|
regex: .+
|
|
- source_labels: [__meta_kubernetes_pod_phase]
|
|
action: keep
|
|
regex: Running
|
|
- target_label: __address__
|
|
replacement: {{ prometheus_k8s_api_server | regex_replace('^https?://', '') }}
|
|
- source_labels:
|
|
- __meta_kubernetes_namespace
|
|
- __meta_kubernetes_pod_name
|
|
- __meta_kubernetes_pod_annotation_prometheus_io_port
|
|
- __meta_kubernetes_pod_annotation_prometheus_io_path
|
|
regex: (.+);(.+);(.+);(.+)
|
|
target_label: __metrics_path__
|
|
replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy$4
|
|
- source_labels:
|
|
- __meta_kubernetes_namespace
|
|
- __meta_kubernetes_pod_name
|
|
- __meta_kubernetes_pod_annotation_prometheus_io_port
|
|
- __meta_kubernetes_pod_annotation_prometheus_io_path
|
|
regex: (.+);(.+);(.+);
|
|
target_label: __metrics_path__
|
|
replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy/metrics
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_pod_label_(.+)
|
|
- source_labels: [__meta_kubernetes_namespace]
|
|
target_label: namespace
|
|
- source_labels: [__meta_kubernetes_pod_name]
|
|
target_label: pod
|
|
- source_labels: [__meta_kubernetes_pod_node_name]
|
|
target_label: instance
|
|
|
|
# Scrape annotated services via the API server proxy. Pod IPs (10.244.0.0/16)
|
|
# aren't routable from outside the cluster, so we tunnel through the apiserver.
|
|
# Requires `prometheus.io/scrape: "true"` and `prometheus.io/port: "<n>"` on the Service.
|
|
# `prometheus.io/path` is optional; defaults to /metrics.
|
|
- job_name: kubernetes-services
|
|
kubernetes_sd_configs:
|
|
- role: service
|
|
api_server: {{ prometheus_k8s_api_server }}
|
|
tls_config:
|
|
ca_file: {{ prometheus_k8s_ca_file }}
|
|
bearer_token_file: {{ prometheus_k8s_token_file }}
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: {{ prometheus_k8s_ca_file }}
|
|
insecure_skip_verify: true
|
|
bearer_token_file: {{ prometheus_k8s_token_file }}
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
|
|
action: keep
|
|
regex: "true"
|
|
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_port]
|
|
action: keep
|
|
regex: .+
|
|
- target_label: __address__
|
|
replacement: {{ prometheus_k8s_api_server | regex_replace('^https?://', '') }}
|
|
# With a path annotation, use it; otherwise fall through to the next rule (default /metrics).
|
|
- source_labels:
|
|
- __meta_kubernetes_namespace
|
|
- __meta_kubernetes_service_name
|
|
- __meta_kubernetes_service_annotation_prometheus_io_port
|
|
- __meta_kubernetes_service_annotation_prometheus_io_path
|
|
regex: (.+);(.+);(.+);(.+)
|
|
target_label: __metrics_path__
|
|
replacement: /api/v1/namespaces/$1/services/$2:$3/proxy$4
|
|
- source_labels:
|
|
- __meta_kubernetes_namespace
|
|
- __meta_kubernetes_service_name
|
|
- __meta_kubernetes_service_annotation_prometheus_io_port
|
|
- __meta_kubernetes_service_annotation_prometheus_io_path
|
|
regex: (.+);(.+);(.+);
|
|
target_label: __metrics_path__
|
|
replacement: /api/v1/namespaces/$1/services/$2:$3/proxy/metrics
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_service_label_(.+)
|
|
- source_labels: [__meta_kubernetes_namespace]
|
|
target_label: namespace
|
|
- source_labels: [__meta_kubernetes_service_name]
|
|
target_label: service
|
|
{% endif %}
|
|
|
|
{% for sc in prometheus_extra_scrape_configs %}
|
|
- job_name: {{ sc.job_name }}
|
|
static_configs:
|
|
- targets:
|
|
{% for t in sc.targets %}
|
|
- "{{ t }}"
|
|
{% endfor %}
|
|
{% if sc.labels is defined %}
|
|
labels:
|
|
{% for k, v in sc.labels.items() %}
|
|
{{ k }}: "{{ v }}"
|
|
{% endfor %}
|
|
{% endif %}
|
|
{% endfor %}
|