prom: annotate hot pods for k8s-pods scrape via apiserver proxy

External Prometheus at 10.0.15.31 already runs the kubernetes-pods job
in prometheus.yml.j2; pods need prometheus.io/scrape + port + path +
job annotations to be picked up. The prop-metrics NodePort selector
also tightened to tier=hot so backfill (PHX_SERVER=false, no :5000
listener) is excluded from the scrape pool.

k8s/deployment.yaml

@@ -30,6 +30,22 @@ spec:
         # doesn't listen on :5000.
         app: prop
         tier: hot
+      annotations:
+        # External Prometheus at 10.0.15.31 (prom.w5isp.com) discovers
+        # these pods via the `kubernetes-pods` job in
+        # `infra/ansible/roles/prometheus/templates/prometheus.yml.j2`,
+        # which scrapes through the apiserver proxy at 10.0.15.1:6443
+        # using a bound ServiceAccount token. PromEx exposes `/metrics`
+        # on the same Bandit listener as the app (port 5000). Backfill
+        # pods (deployment-backfill.yaml) intentionally lack these
+        # annotations because they don't run a web server.
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "5000"
+        prometheus.io/path: "/metrics"
+        # `prometheus.io/job` is a per-pod override added in
+        # `prometheus.yml.j2` so app-specific dashboards can filter on
+        # `job="prop"` instead of `kubernetes-pods`.
+        prometheus.io/job: "prop"
     spec:
       tolerations:
         - key: node-role.kubernetes.io/control-plane

k8s/metrics-service.yaml

@@ -15,10 +15,14 @@ spec:
   sessionAffinity: None
   selector:
     app: prop
+    # Restrict to hot pods. prop-backfill carries the same `app: prop`
+    # label but runs PHX_SERVER=false (no listener on :5000), so an
+    # unscoped selector lands ~50% of scrapes on a closed port.
+    tier: hot
   ports:
     - name: metrics
       # Fixed high-numbered NodePort so the external Prometheus at
-      # 10.0.15.25 can scrape `nodeIP:30090` without needing k8s SD.
+      # 10.0.15.31 can scrape `nodeIP:30090` without needing k8s SD.
       nodePort: 30090
       port: 5000
       targetPort: 5000