prom: feed home-cluster metrics + provision standard k8s dashboards

Cluster-side:
- Deploy kube-state-metrics (kube-system) for cluster-state series.
- Deploy node-exporter DaemonSet on all 7 nodes via the new monitoring ns
  (privileged PSS, hostNetwork, read-only host mounts).
- Both managed by ArgoCD apps under home/cluster/argocd/apps/.

Prometheus-side:
- Replace direct pod-IP scraping (which can't reach the 10.244.0.0/16 pod
  network from outside the cluster) with API-server-proxy scraping for
  both Services and Pods. Now one target per pod, all healthy.
- Extend external Prometheus RBAC with services/proxy + pods/proxy.

Grafana-side:
- Add file-based dashboard provisioner; ship 5 standard dashboards
  (Node Exporter Full, k8s cluster-via-prometheus, k8s all-in-one,
  k8s monitoring, Prometheus 2.0 overview).

prom.w5isp.com:3000 now shows live data for all 7 cluster nodes plus
the prom host itself.

ansible/roles/grafana/defaults/main.yml

@@ -11,6 +11,11 @@ grafana_admin_user: admin
 # Set in vault / host_vars; default kept for first-boot only.
 grafana_admin_password: admin
 
+# On-disk location for provisioned dashboard JSON files. Grafana watches this
+# path via the file-based dashboard provider configured in
+# /etc/grafana/provisioning/dashboards/.
+grafana_dashboards_dir: /var/lib/grafana/dashboards
+
 # Datasources provisioned at startup. The first one is marked default.
 grafana_datasources:
   - name: Prometheus

ansible/roles/grafana/tasks/main.yml

@@ -52,6 +52,32 @@
     mode: "0640"
   notify: Restart grafana
 
+- name: Create dashboards directory
+  ansible.builtin.file:
+    path: "{{ grafana_dashboards_dir }}"
+    state: directory
+    owner: grafana
+    group: grafana
+    mode: "0755"
+
+- name: Sync standard dashboard JSON files
+  ansible.builtin.copy:
+    src: dashboards/
+    dest: "{{ grafana_dashboards_dir }}/"
+    owner: grafana
+    group: grafana
+    mode: "0644"
+  notify: Restart grafana
+
+- name: Render dashboard provider config
+  ansible.builtin.template:
+    src: dashboard-providers.yaml.j2
+    dest: /etc/grafana/provisioning/dashboards/default.yaml
+    owner: root
+    group: grafana
+    mode: "0640"
+  notify: Restart grafana
+
 - name: Enable and start grafana-server
   ansible.builtin.systemd:
     name: grafana-server

ansible/roles/grafana/templates/dashboard-providers.yaml.j2

@@ -0,0 +1,14 @@
+{{ ansible_managed | comment }}
+apiVersion: 1
+
+providers:
+  - name: default
+    orgId: 1
+    folder: ""
+    type: file
+    disableDeletion: false
+    allowUiUpdates: true
+    updateIntervalSeconds: 30
+    options:
+      path: {{ grafana_dashboards_dir }}
+      foldersFromFilesStructure: false

ansible/roles/prometheus/files/k8s-rbac.yaml

@@ -23,8 +23,10 @@ rules:
       - nodes/proxy
       - nodes/metrics
       - services
+      - services/proxy
       - endpoints
       - pods
+      - pods/proxy
     verbs: ["get", "list", "watch"]
   - apiGroups:
       - extensions

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -92,6 +92,8 @@ scrape_configs:
         target_label: __metrics_path__
         replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
 
+  # Scrape annotated pods via the API server proxy (one target per pod).
+  # Requires `prometheus.io/scrape: "true"` and `prometheus.io/port: "<n>"` on the Pod.
   - job_name: kubernetes-pods
     kubernetes_sd_configs:
       - role: pod
@@ -99,60 +101,95 @@ scrape_configs:
         tls_config:
           ca_file: {{ prometheus_k8s_ca_file }}
         bearer_token_file: {{ prometheus_k8s_token_file }}
+    scheme: https
+    tls_config:
+      ca_file: {{ prometheus_k8s_ca_file }}
+      insecure_skip_verify: true
+    bearer_token_file: {{ prometheus_k8s_token_file }}
     relabel_configs:
-      # Only scrape pods with prometheus.io/scrape: "true"
       - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
         action: keep
         regex: "true"
-      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
-        action: replace
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
+        action: keep
+        regex: .+
+      - source_labels: [__meta_kubernetes_pod_phase]
+        action: keep
+        regex: Running
+      - target_label: __address__
+        replacement: {{ prometheus_k8s_api_server | regex_replace('^https?://', '') }}
+      - source_labels:
+          - __meta_kubernetes_namespace
+          - __meta_kubernetes_pod_name
+          - __meta_kubernetes_pod_annotation_prometheus_io_port
+          - __meta_kubernetes_pod_annotation_prometheus_io_path
+        regex: (.+);(.+);(.+);(.+)
         target_label: __metrics_path__
-        regex: (.+)
-      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
-        action: replace
-        regex: ([^:]+)(?::\d+)?;(\d+)
-        replacement: $1:$2
-        target_label: __address__
+        replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy$4
+      - source_labels:
+          - __meta_kubernetes_namespace
+          - __meta_kubernetes_pod_name
+          - __meta_kubernetes_pod_annotation_prometheus_io_port
+          - __meta_kubernetes_pod_annotation_prometheus_io_path
+        regex: (.+);(.+);(.+);
+        target_label: __metrics_path__
+        replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy/metrics
       - action: labelmap
         regex: __meta_kubernetes_pod_label_(.+)
       - source_labels: [__meta_kubernetes_namespace]
-        action: replace
         target_label: namespace
       - source_labels: [__meta_kubernetes_pod_name]
-        action: replace
         target_label: pod
+      - source_labels: [__meta_kubernetes_pod_node_name]
+        target_label: instance
 
-  - job_name: kubernetes-service-endpoints
+  # Scrape annotated services via the API server proxy. Pod IPs (10.244.0.0/16)
+  # aren't routable from outside the cluster, so we tunnel through the apiserver.
+  # Requires `prometheus.io/scrape: "true"` and `prometheus.io/port: "<n>"` on the Service.
+  # `prometheus.io/path` is optional; defaults to /metrics.
+  - job_name: kubernetes-services
     kubernetes_sd_configs:
-      - role: endpoints
+      - role: service
         api_server: {{ prometheus_k8s_api_server }}
         tls_config:
           ca_file: {{ prometheus_k8s_ca_file }}
         bearer_token_file: {{ prometheus_k8s_token_file }}
+    scheme: https
+    tls_config:
+      ca_file: {{ prometheus_k8s_ca_file }}
+      insecure_skip_verify: true
+    bearer_token_file: {{ prometheus_k8s_token_file }}
     relabel_configs:
       - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
         action: keep
         regex: "true"
-      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
-        action: replace
-        target_label: __scheme__
-        regex: (https?)
-      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
-        action: replace
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_port]
+        action: keep
+        regex: .+
+      - target_label: __address__
+        replacement: {{ prometheus_k8s_api_server | regex_replace('^https?://', '') }}
+      # With a path annotation, use it; otherwise fall through to the next rule (default /metrics).
+      - source_labels:
+          - __meta_kubernetes_namespace
+          - __meta_kubernetes_service_name
+          - __meta_kubernetes_service_annotation_prometheus_io_port
+          - __meta_kubernetes_service_annotation_prometheus_io_path
+        regex: (.+);(.+);(.+);(.+)
         target_label: __metrics_path__
-        regex: (.+)
-      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
-        action: replace
-        regex: ([^:]+)(?::\d+)?;(\d+)
-        replacement: $1:$2
-        target_label: __address__
+        replacement: /api/v1/namespaces/$1/services/$2:$3/proxy$4
+      - source_labels:
+          - __meta_kubernetes_namespace
+          - __meta_kubernetes_service_name
+          - __meta_kubernetes_service_annotation_prometheus_io_port
+          - __meta_kubernetes_service_annotation_prometheus_io_path
+        regex: (.+);(.+);(.+);
+        target_label: __metrics_path__
+        replacement: /api/v1/namespaces/$1/services/$2:$3/proxy/metrics
       - action: labelmap
         regex: __meta_kubernetes_service_label_(.+)
       - source_labels: [__meta_kubernetes_namespace]
-        action: replace
         target_label: namespace
       - source_labels: [__meta_kubernetes_service_name]
-        action: replace
         target_label: service
 {% endif %}
 

home/cluster/argocd/apps/kube-state-metrics.yaml

@@ -0,0 +1,22 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: kube-state-metrics
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://codeberg.org/gmcintire/infra.git
+    targetRevision: main
+    path: home/cluster/kube-state-metrics
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kube-system
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - ServerSideApply=true

home/cluster/argocd/apps/node-exporter.yaml

@@ -0,0 +1,23 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: node-exporter
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://codeberg.org/gmcintire/infra.git
+    targetRevision: main
+    path: home/cluster/node-exporter
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: monitoring
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

home/cluster/kube-state-metrics/cluster-role-binding.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: 2.13.0
+  name: kube-state-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics
+subjects:
+- kind: ServiceAccount
+  name: kube-state-metrics
+  namespace: kube-system

home/cluster/kube-state-metrics/cluster-role.yaml

@@ -0,0 +1,128 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: 2.13.0
+  name: kube-state-metrics
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - secrets
+  - nodes
+  - pods
+  - services
+  - serviceaccounts
+  - resourcequotas
+  - replicationcontrollers
+  - limitranges
+  - persistentvolumeclaims
+  - persistentvolumes
+  - namespaces
+  - endpoints
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - statefulsets
+  - daemonsets
+  - deployments
+  - replicasets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - batch
+  resources:
+  - cronjobs
+  - jobs
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - autoscaling
+  resources:
+  - horizontalpodautoscalers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+- apiGroups:
+  - policy
+  resources:
+  - poddisruptionbudgets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  - volumeattachments
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - mutatingwebhookconfigurations
+  - validatingwebhookconfigurations
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  - ingressclasses
+  - ingresses
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterrolebindings
+  - clusterroles
+  - rolebindings
+  - roles
+  verbs:
+  - list
+  - watch

home/cluster/kube-state-metrics/deployment.yaml

@@ -0,0 +1,55 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: 2.13.0
+  name: kube-state-metrics
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-state-metrics
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: exporter
+        app.kubernetes.io/name: kube-state-metrics
+        app.kubernetes.io/version: 2.13.0
+    spec:
+      automountServiceAccountToken: true
+      containers:
+      - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0
+        livenessProbe:
+          httpGet:
+            path: /livez
+            port: http-metrics
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        name: kube-state-metrics
+        ports:
+        - containerPort: 8080
+          name: http-metrics
+        - containerPort: 8081
+          name: telemetry
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: telemetry
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 65534
+          seccompProfile:
+            type: RuntimeDefault
+      nodeSelector:
+        kubernetes.io/os: linux
+      serviceAccountName: kube-state-metrics

home/cluster/kube-state-metrics/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - service-account.yaml
+  - cluster-role.yaml
+  - cluster-role-binding.yaml
+  - service.yaml
+  - deployment.yaml

home/cluster/kube-state-metrics/service-account.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+automountServiceAccountToken: false
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: 2.13.0
+  name: kube-state-metrics
+  namespace: kube-system

home/cluster/kube-state-metrics/service.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: 2.13.0
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "8080"
+    prometheus.io/path: /metrics
+  name: kube-state-metrics
+  namespace: kube-system
+spec:
+  clusterIP: None
+  ports:
+  - name: http-metrics
+    port: 8080
+    targetPort: http-metrics
+  - name: telemetry
+    port: 8081
+    targetPort: telemetry
+  selector:
+    app.kubernetes.io/name: kube-state-metrics

home/cluster/node-exporter/daemonset.yaml

@@ -0,0 +1,73 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: node-exporter
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: node-exporter
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: node-exporter
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: node-exporter
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9100"
+        prometheus.io/path: /metrics
+    spec:
+      hostNetwork: true
+      hostPID: true
+      tolerations:
+        - operator: Exists
+      containers:
+        - name: node-exporter
+          image: quay.io/prometheus/node-exporter:v1.8.2
+          args:
+            - --path.procfs=/host/proc
+            - --path.sysfs=/host/sys
+            - --path.rootfs=/host/root
+            - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/)
+            - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
+            - --web.listen-address=0.0.0.0:9100
+          ports:
+            - name: metrics
+              containerPort: 9100
+              hostPort: 9100
+              protocol: TCP
+          resources:
+            requests:
+              cpu: 50m
+              memory: 32Mi
+            limits:
+              memory: 128Mi
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65534
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: proc
+              mountPath: /host/proc
+              readOnly: true
+            - name: sys
+              mountPath: /host/sys
+              readOnly: true
+            - name: root
+              mountPath: /host/root
+              mountPropagation: HostToContainer
+              readOnly: true
+      volumes:
+        - name: proc
+          hostPath:
+            path: /proc
+        - name: sys
+          hostPath:
+            path: /sys
+        - name: root
+          hostPath:
+            path: /

home/cluster/node-exporter/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - daemonset.yaml
+  - service.yaml

home/cluster/node-exporter/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring
+  labels:
+    pod-security.kubernetes.io/enforce: privileged

home/cluster/node-exporter/service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: node-exporter
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: node-exporter
+  # Prometheus scrapes the DaemonSet pods (one target per node) via the
+  # kubernetes-pods job; scraping the headless service would only hit one
+  # pod per cycle, so no scrape annotations on the Service.
+spec:
+  clusterIP: None
+  selector:
+    app.kubernetes.io/name: node-exporter
+  ports:
+    - name: metrics
+      port: 9100
+      targetPort: 9100
+      protocol: TCP