diff --git a/ansible/roles/grafana/files/dashboards/tmobile-home-internet.json b/ansible/roles/grafana/files/dashboards/tmobile-home-internet.json index ee2c03b..d90a6fa 100644 --- a/ansible/roles/grafana/files/dashboards/tmobile-home-internet.json +++ b/ansible/roles/grafana/files/dashboards/tmobile-home-internet.json @@ -88,7 +88,7 @@ "textMode": "value" }, "targets": [ - {"expr": "tmobile_gateway_gateway_up", "refId": "A", "datasource": "$datasource"} + {"expr": "up{service=\"tmobile-exporter\"}", "refId": "A", "datasource": "$datasource"} ] }, { diff --git a/home/cluster/tmobile-exporter/configmap.yaml b/home/cluster/tmobile-exporter/configmap.yaml new file mode 100644 index 0000000..1b0497b --- /dev/null +++ b/home/cluster/tmobile-exporter/configmap.yaml @@ -0,0 +1,102 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: tmobile-exporter-config + namespace: monitoring +data: + # nginx reverse-proxies /metrics to json_exporter's /probe endpoint with the + # gateway target baked in, so the existing Prometheus kubernetes-services + # scrape job picks this up via a normal /metrics path annotation. + nginx.conf: | + worker_processes 1; + pid /tmp/nginx.pid; + error_log /dev/stderr warn; + events { worker_connections 64; } + http { + access_log off; + client_body_temp_path /tmp/nginx-client 1 2; + proxy_temp_path /tmp/nginx-proxy 1 2; + fastcgi_temp_path /tmp/nginx-fcgi 1 2; + uwsgi_temp_path /tmp/nginx-uwsgi 1 2; + scgi_temp_path /tmp/nginx-scgi 1 2; + server { + listen 9099; + location = /healthz { return 200 "ok\n"; } + location = /metrics { + proxy_pass http://127.0.0.1:7979/probe?module=tmhi&target=http://192.168.12.1/TMI/v1/gateway?get=all; + } + location / { return 404; } + } + } + + # json_exporter mapping for the G4AR's /TMI/v1/gateway?get=all response. + # JSONPath uses k8s client-go dialect, which requires bracket+single-quote + # notation for digit-prefixed keys (.signal['5g']). YAML single-quoted + # strings escape literal single quotes by doubling them. + # allow_missing_key keeps the scrape clean when blocks are absent (the + # G4AR omits the 4g block entirely on 5G-only deployments). + json-exporter.yml: | + modules: + tmhi: + metrics: + - name: tmobile_gateway_uptime_seconds + path: '{ .time.upTime }' + help: Gateway uptime in seconds. + + - name: tmobile_gateway + type: object + path: '{ .device }' + help: Static gateway metadata; value is always 1. + labels: + serial: '{ .serial }' + manufacturer: '{ .manufacturer }' + model: '{ .model }' + name: '{ .name }' + hardware_version: '{ .hardwareVersion }' + software_version: '{ .softwareVersion }' + mac: '{ .macId }' + values: + info: 1 + + - name: tmobile_gateway_rsrp + path: '{ .signal[''5g''].rsrp }' + allow_missing_key: true + help: Reference Signal Received Power (dBm). + labels: + type: "5g" + band: '{ .signal[''5g''].bands[0] }' + - name: tmobile_gateway_rsrq + path: '{ .signal[''5g''].rsrq }' + allow_missing_key: true + help: Reference Signal Received Quality (dB). + labels: + type: "5g" + band: '{ .signal[''5g''].bands[0] }' + - name: tmobile_gateway_rssi + path: '{ .signal[''5g''].rssi }' + allow_missing_key: true + help: Received Signal Strength Indicator (dBm). + labels: + type: "5g" + band: '{ .signal[''5g''].bands[0] }' + - name: tmobile_gateway_snr + path: '{ .signal[''5g''].sinr }' + allow_missing_key: true + help: Signal-to-noise ratio (dB). + labels: + type: "5g" + band: '{ .signal[''5g''].bands[0] }' + - name: tmobile_gateway_bars + path: '{ .signal[''5g''].bars }' + allow_missing_key: true + help: Signal strength in bars (0-5). + labels: + type: "5g" + band: '{ .signal[''5g''].bands[0] }' + - name: tmobile_gateway_channel_id + path: '{ .signal[''5g''].cid }' + allow_missing_key: true + help: Cellular channel ID. + labels: + type: "5g" + band: '{ .signal[''5g''].bands[0] }' diff --git a/home/cluster/tmobile-exporter/deployment.yaml b/home/cluster/tmobile-exporter/deployment.yaml index 0f2f1c8..6539e87 100644 --- a/home/cluster/tmobile-exporter/deployment.yaml +++ b/home/cluster/tmobile-exporter/deployment.yaml @@ -15,15 +15,44 @@ spec: labels: app.kubernetes.io/name: tmobile-exporter spec: - imagePullSecrets: - - name: forgejo-registry + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault containers: - - name: exporter - image: git.mcintire.me/graham/tmobile-exporter:v1 - imagePullPolicy: IfNotPresent + # json_exporter scrapes the gateway JSON on demand and exposes /probe. + # The nginx sidecar below rewrites the public /metrics URL onto it. + - name: json-exporter + image: quay.io/prometheuscommunity/json-exporter:v0.7.0 args: - - -target=192.168.12.1 - - -listen=:9099 + - --config.file=/etc/json_exporter/json-exporter.yml + ports: + - name: probe + containerPort: 7979 + protocol: TCP + resources: + requests: + cpu: 10m + memory: 24Mi + limits: + memory: 96Mi + securityContext: + runAsUser: 65534 + runAsGroup: 65534 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: config + mountPath: /etc/json_exporter + readOnly: true + + # nginx rewrites GET /metrics → GET /probe?module=tmhi&target=. + # nginxinc/nginx-unprivileged runs as UID 101 (non-root) without file caps, + # so it survives no_new_privs + readOnlyRootFilesystem cleanly. + - name: nginx + image: nginxinc/nginx-unprivileged:1.27-alpine-slim ports: - name: metrics containerPort: 9099 @@ -35,12 +64,32 @@ spec: limits: memory: 64Mi securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 + runAsUser: 101 + runAsGroup: 101 allowPrivilegeEscalation: false readOnlyRootFilesystem: true capabilities: drop: ["ALL"] - seccompProfile: - type: RuntimeDefault + volumeMounts: + - name: config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true + - name: nginx-tmp + mountPath: /tmp + readinessProbe: + httpGet: + path: /healthz + port: metrics + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: metrics + periodSeconds: 30 + volumes: + - name: config + configMap: + name: tmobile-exporter-config + - name: nginx-tmp + emptyDir: {} diff --git a/home/cluster/tmobile-exporter/kustomization.yaml b/home/cluster/tmobile-exporter/kustomization.yaml index a33121c..ff5298c 100644 --- a/home/cluster/tmobile-exporter/kustomization.yaml +++ b/home/cluster/tmobile-exporter/kustomization.yaml @@ -1,5 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - configmap.yaml - deployment.yaml - service.yaml diff --git a/home/cluster/tmobile-exporter/service.yaml b/home/cluster/tmobile-exporter/service.yaml index 9216f65..06607c8 100644 --- a/home/cluster/tmobile-exporter/service.yaml +++ b/home/cluster/tmobile-exporter/service.yaml @@ -8,7 +8,6 @@ metadata: annotations: prometheus.io/scrape: "true" prometheus.io/port: "9099" - prometheus.io/job: tmobile_gateway spec: selector: app.kubernetes.io/name: tmobile-exporter diff --git a/home/cluster/tmobile-exporter/src/Dockerfile b/home/cluster/tmobile-exporter/src/Dockerfile deleted file mode 100644 index da9c47c..0000000 --- a/home/cluster/tmobile-exporter/src/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM golang:1.23-alpine AS build -WORKDIR /src -COPY go.mod ./ -RUN go mod download -COPY . . -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/tmobile-exporter . - -FROM gcr.io/distroless/static-debian12:nonroot -COPY --from=build /out/tmobile-exporter /tmobile-exporter -USER 65532:65532 -EXPOSE 9099 -ENTRYPOINT ["/tmobile-exporter"] diff --git a/home/cluster/tmobile-exporter/src/go.mod b/home/cluster/tmobile-exporter/src/go.mod deleted file mode 100644 index 81f29cc..0000000 --- a/home/cluster/tmobile-exporter/src/go.mod +++ /dev/null @@ -1,17 +0,0 @@ -module github.com/gmcintire/infra/tmobile-exporter - -go 1.23 - -require github.com/prometheus/client_golang v1.20.5 - -require ( - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - golang.org/x/sys v0.22.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect -) diff --git a/home/cluster/tmobile-exporter/src/go.sum b/home/cluster/tmobile-exporter/src/go.sum deleted file mode 100644 index d5318cf..0000000 --- a/home/cluster/tmobile-exporter/src/go.sum +++ /dev/null @@ -1,24 +0,0 @@ -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/home/cluster/tmobile-exporter/src/main.go b/home/cluster/tmobile-exporter/src/main.go deleted file mode 100644 index 753cdcf..0000000 --- a/home/cluster/tmobile-exporter/src/main.go +++ /dev/null @@ -1,225 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "log" - "net/http" - "os" - "os/signal" - "sync" - "syscall" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -const ns = "tmobile_gateway" - -type device struct { - Manufacturer string `json:"manufacturer"` - Name string `json:"name"` - Model string `json:"model"` - HardwareVersion string `json:"hardwareVersion"` - SoftwareVersion string `json:"softwareVersion"` - Serial string `json:"serial"` - MacID string `json:"macId"` - Role string `json:"role"` -} - -type timeBlock struct { - UpTime int64 `json:"upTime"` -} - -// signalBlock is shared by both the 4g and 5g objects. All numeric fields -// are decoded as float64 — the G4AR returns `bars` as a float (e.g. 4.0) -// while older Sagemcom gateways return it as an int, and json.Unmarshal -// into float64 accepts both shapes. -type signalBlock struct { - Cid float64 `json:"cid"` - Sinr float64 `json:"sinr"` - Rsrp float64 `json:"rsrp"` - Rsrq float64 `json:"rsrq"` - Rssi float64 `json:"rssi"` - Bars float64 `json:"bars"` - Bands []string `json:"bands"` -} - -type signalGroup struct { - FourG *signalBlock `json:"4g,omitempty"` - FiveG *signalBlock `json:"5g,omitempty"` -} - -type gatewayStatus struct { - Device device `json:"device"` - Time timeBlock `json:"time"` - Signal signalGroup `json:"signal"` -} - -var ( - gatewayUp = promauto.NewGauge(prometheus.GaugeOpts{ - Namespace: ns, Name: "gateway_up", - Help: "1 if the last scrape of the gateway succeeded, else 0.", - }) - scrapeErrors = promauto.NewCounter(prometheus.CounterOpts{ - Namespace: ns, Name: "scrape_errors_total", - Help: "Number of failed gateway scrapes.", - }) - uptimeSeconds = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: ns, Name: "uptime_seconds", - Help: "Gateway uptime in seconds.", - }, []string{"serial"}) - deviceInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: ns, Name: "device_info", - Help: "Static gateway metadata; value is always 1.", - }, []string{"serial", "manufacturer", "model", "name", "hardware_version", "software_version", "mac"}) - - radio = map[string]*prometheus.GaugeVec{} -) - -func init() { - for _, m := range []string{"channel_id", "rsrp", "rsrq", "rssi", "snr", "bars"} { - radio[m] = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: ns, Name: m, - Help: "Radio " + m + " per access tech / band.", - }, []string{"type", "band"}) - } -} - -type collector struct { - target string - client *http.Client - mu sync.Mutex - last map[string]map[string]struct{} // type -> set of bands seen -} - -func newCollector(target string) *collector { - return &collector{ - target: target, - client: &http.Client{Timeout: 5 * time.Second}, - last: map[string]map[string]struct{}{"4g": {}, "5g": {}}, - } -} - -func (c *collector) url() string { - return fmt.Sprintf("http://%s/TMI/v1/gateway?get=all", c.target) -} - -func (c *collector) scrape(ctx context.Context) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url(), nil) - if err != nil { - return err - } - resp, err := c.client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("gateway returned %d", resp.StatusCode) - } - var s gatewayStatus - if err := json.NewDecoder(resp.Body).Decode(&s); err != nil { - return fmt.Errorf("decode: %w", err) - } - - uptimeSeconds.WithLabelValues(s.Device.Serial).Set(float64(s.Time.UpTime)) - deviceInfo.Reset() - deviceInfo.WithLabelValues( - s.Device.Serial, s.Device.Manufacturer, s.Device.Model, s.Device.Name, - s.Device.HardwareVersion, s.Device.SoftwareVersion, s.Device.MacID, - ).Set(1) - - c.mu.Lock() - defer c.mu.Unlock() - c.observe("4g", s.Signal.FourG) - c.observe("5g", s.Signal.FiveG) - return nil -} - -// observe records signal metrics for one access tech (4g or 5g). It also -// drops any (type, band) series that were present on a previous scrape but -// are missing now — otherwise a dropped band would stick around forever. -func (c *collector) observe(tech string, b *signalBlock) { - seen := map[string]struct{}{} - if b != nil { - bands := b.Bands - if len(bands) == 0 { - bands = []string{""} - } - for _, band := range bands { - seen[band] = struct{}{} - radio["channel_id"].WithLabelValues(tech, band).Set(b.Cid) - radio["rsrp"].WithLabelValues(tech, band).Set(b.Rsrp) - radio["rsrq"].WithLabelValues(tech, band).Set(b.Rsrq) - radio["rssi"].WithLabelValues(tech, band).Set(b.Rssi) - radio["snr"].WithLabelValues(tech, band).Set(b.Sinr) - radio["bars"].WithLabelValues(tech, band).Set(b.Bars) - } - } - for band := range c.last[tech] { - if _, ok := seen[band]; !ok { - for _, g := range radio { - g.DeleteLabelValues(tech, band) - } - } - } - c.last[tech] = seen -} - -func (c *collector) run(ctx context.Context, every time.Duration) { - t := time.NewTicker(every) - defer t.Stop() - for { - if err := c.scrape(ctx); err != nil { - scrapeErrors.Inc() - gatewayUp.Set(0) - log.Printf("scrape failed: %v", err) - } else { - gatewayUp.Set(1) - } - select { - case <-ctx.Done(): - return - case <-t.C: - } - } -} - -func main() { - listen := flag.String("listen", ":9099", "exporter listen address") - target := flag.String("target", "192.168.12.1", "gateway IP or host") - every := flag.Duration("scrape-frequency", 10*time.Second, "how often to poll the gateway") - flag.Parse() - - ctx, cancel := context.WithCancel(context.Background()) - sigC := make(chan os.Signal, 1) - signal.Notify(sigC, syscall.SIGINT, syscall.SIGTERM) - go func() { <-sigC; cancel() }() - - c := newCollector(*target) - go c.run(ctx, *every) - - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) - mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("ok\n")) - }) - - srv := &http.Server{Addr: *listen, Handler: mux, ReadHeaderTimeout: 5 * time.Second} - log.Printf("tmobile-exporter listening on %s, target=%s", *listen, *target) - go func() { - if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { - log.Fatalf("listen: %v", err) - } - }() - <-ctx.Done() - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) - defer shutdownCancel() - _ = srv.Shutdown(shutdownCtx) -}