diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index 94c3e734..c1dddbc4 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -30,6 +30,22 @@ spec: # doesn't listen on :5000. app: prop tier: hot + annotations: + # External Prometheus at 10.0.15.31 (prom.w5isp.com) discovers + # these pods via the `kubernetes-pods` job in + # `infra/ansible/roles/prometheus/templates/prometheus.yml.j2`, + # which scrapes through the apiserver proxy at 10.0.15.1:6443 + # using a bound ServiceAccount token. PromEx exposes `/metrics` + # on the same Bandit listener as the app (port 5000). Backfill + # pods (deployment-backfill.yaml) intentionally lack these + # annotations because they don't run a web server. + prometheus.io/scrape: "true" + prometheus.io/port: "5000" + prometheus.io/path: "/metrics" + # `prometheus.io/job` is a per-pod override added in + # `prometheus.yml.j2` so app-specific dashboards can filter on + # `job="prop"` instead of `kubernetes-pods`. + prometheus.io/job: "prop" spec: tolerations: - key: node-role.kubernetes.io/control-plane diff --git a/k8s/metrics-service.yaml b/k8s/metrics-service.yaml index eb4d2d92..d340e97e 100644 --- a/k8s/metrics-service.yaml +++ b/k8s/metrics-service.yaml @@ -15,10 +15,14 @@ spec: sessionAffinity: None selector: app: prop + # Restrict to hot pods. prop-backfill carries the same `app: prop` + # label but runs PHX_SERVER=false (no listener on :5000), so an + # unscoped selector lands ~50% of scrapes on a closed port. + tier: hot ports: - name: metrics # Fixed high-numbered NodePort so the external Prometheus at - # 10.0.15.25 can scrape `nodeIP:30090` without needing k8s SD. + # 10.0.15.31 can scrape `nodeIP:30090` without needing k8s SD. nodePort: 30090 port: 5000 targetPort: 5000