From abb7e90d2db1546045c11a4c90705e489e3d1139 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Fri, 8 May 2026 10:23:30 -0500 Subject: [PATCH] prom: annotate hot pods for k8s-pods scrape via apiserver proxy External Prometheus at 10.0.15.31 already runs the kubernetes-pods job in prometheus.yml.j2; pods need prometheus.io/scrape + port + path + job annotations to be picked up. The prop-metrics NodePort selector also tightened to tier=hot so backfill (PHX_SERVER=false, no :5000 listener) is excluded from the scrape pool. --- k8s/deployment.yaml | 16 ++++++++++++++++ k8s/metrics-service.yaml | 6 +++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index 94c3e734..c1dddbc4 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -30,6 +30,22 @@ spec: # doesn't listen on :5000. app: prop tier: hot + annotations: + # External Prometheus at 10.0.15.31 (prom.w5isp.com) discovers + # these pods via the `kubernetes-pods` job in + # `infra/ansible/roles/prometheus/templates/prometheus.yml.j2`, + # which scrapes through the apiserver proxy at 10.0.15.1:6443 + # using a bound ServiceAccount token. PromEx exposes `/metrics` + # on the same Bandit listener as the app (port 5000). Backfill + # pods (deployment-backfill.yaml) intentionally lack these + # annotations because they don't run a web server. + prometheus.io/scrape: "true" + prometheus.io/port: "5000" + prometheus.io/path: "/metrics" + # `prometheus.io/job` is a per-pod override added in + # `prometheus.yml.j2` so app-specific dashboards can filter on + # `job="prop"` instead of `kubernetes-pods`. + prometheus.io/job: "prop" spec: tolerations: - key: node-role.kubernetes.io/control-plane diff --git a/k8s/metrics-service.yaml b/k8s/metrics-service.yaml index eb4d2d92..d340e97e 100644 --- a/k8s/metrics-service.yaml +++ b/k8s/metrics-service.yaml @@ -15,10 +15,14 @@ spec: sessionAffinity: None selector: app: prop + # Restrict to hot pods. prop-backfill carries the same `app: prop` + # label but runs PHX_SERVER=false (no listener on :5000), so an + # unscoped selector lands ~50% of scrapes on a closed port. + tier: hot ports: - name: metrics # Fixed high-numbered NodePort so the external Prometheus at - # 10.0.15.25 can scrape `nodeIP:30090` without needing k8s SD. + # 10.0.15.31 can scrape `nodeIP:30090` without needing k8s SD. nodePort: 30090 port: 5000 targetPort: 5000