fix(probes): stop DB saturation from SIGKILLing every pod

Investigated why all three prop pods were restart-looping every few
minutes (RESTARTS counts 7–11 in 99m). The trail:

  20:53:33.058 [error] Postgrex.Protocol ("db_conn_10") disconnected:
    client (:"Elixir.Microwaveprop.PromEx.Poller.5000") timed out
    because it queued and checked out the connection for longer than
    15000ms

PromEx's Oban queue-depth poller was scanning the oban_jobs table
every 5s and holding an Ecto connection past its checkout timeout.
That starved the rest of the pool, so /health's `SELECT 1` couldn't
acquire a connection inside the 3s liveness probe timeout, the
kubelet's failureThreshold=3 tripped, and SIGKILL landed on the
BEAM. Every replica cycled on the same cadence.

Two fixes:

1. Split /live (no DB) from /health (DB ping). Kubernetes livenessProbe
   now points at /live so a saturated Ecto pool can never cascade into
   SIGKILL. Readiness still hits /health so a genuine DB outage drains
   the pod from Service endpoints gracefully.

2. Turn off the PromEx Oban plugin in prod via the same flag already
   used in test. The queue-depth query isn't worth pod instability;
   the oban_web dashboard surfaces the same information without
   scanning the job table on a timer.

Locked both down with HealthController tests that verify /live never
touches the Repo (no sandbox owner, controller.live/2 called directly,
200 ok) and /health does (sanity query round-trips).

config/runtime.exs

@@ -344,17 +344,27 @@ if config_env() == :prod do
 
   config :microwaveprop, Oban, plugins: oban_plugins
   config :microwaveprop, :email_from, {"NTMS Propagation", "prop@w5isp.com"}
+
+  # Disable the PromEx Oban queue-depth poller in prod. Its 5 s
+  # telemetry_poller runs an aggregate scan against the oban_jobs
+  # table; with our current row count that query regularly holds a
+  # Postgrex connection for > 15 s and starves the Ecto pool enough
+  # that the /health probe times out and the kubelet SIGKILLs every
+  # replica. Queue depth is better observed from the oban_web
+  # dashboard anyway.
+  config :microwaveprop, :prom_ex_include_oban_plugin, false
   config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080")
   config :microwaveprop, srtm_tiles_dir: "/data/srtm"
+
+  # --- OpenTelemetry ---
   # config :microwaveprop, start_freshness_monitor: true
+  # Ship traces via OTLP/gRPC to the cluster OpenTelemetry Collector
+  # when OTEL_EXPORTER_OTLP_ENDPOINT is set (k8s deployment env). When
+  # unset the exporter is replaced with the `:none` sampler so trace
+  # collection is a no-op — keeps dev/test / unconfigured hosts quiet.
   config :microwaveprop, start_freshness_monitor: false
 end
 
-# --- OpenTelemetry ---
-# Ship traces via OTLP/gRPC to the cluster OpenTelemetry Collector
-# when OTEL_EXPORTER_OTLP_ENDPOINT is set (k8s deployment env). When
-# unset the exporter is replaced with the `:none` sampler so trace
-# collection is a no-op — keeps dev/test / unconfigured hosts quiet.
 if otlp_endpoint = System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT") do
   config :opentelemetry, :resource,
     service: [

k8s/deployment.yaml

@@ -92,9 +92,14 @@ spec:
             periodSeconds: 5
             failureThreshold: 2
             timeoutSeconds: 2
+          # /live skips the DB — a saturated Ecto pool (e.g. a PromEx
+          # poller holding a connection >15s) would otherwise cause
+          # /health's `SELECT 1` to time out and the kubelet to SIGKILL
+          # every replica at once. Readiness still points at /health so
+          # DB outages drain the pod from Service endpoints gracefully.
           livenessProbe:
             httpGet:
-              path: /health
+              path: /live
               port: 5000
             periodSeconds: 10
             failureThreshold: 3

lib/microwaveprop_web/controllers/health_controller.ex

@@ -1,10 +1,34 @@
 defmodule MicrowavepropWeb.HealthController do
+  @moduledoc """
+  Two separate probe endpoints for Kubernetes:
+
+    * `live/2` — liveness only. Returns 200 as long as the BEAM can
+      serve an HTTP request. Must NOT touch the database: a slow or
+      saturated Ecto pool would otherwise cascade into SIGKILL across
+      every replica simultaneously. Used as the `livenessProbe`
+      target.
+
+    * `check/2` — readiness. Also pings the Repo via `SELECT 1` so
+      the pod is taken out of the Service endpoint set when the DB
+      is unreachable. Used as the `readinessProbe` target.
+
+  Splitting these two concerns is specifically a fix for an incident
+  where PromEx's Oban queue-depth poller held a connection for > 15s,
+  queued up every other checkout, and the liveness probe's `SELECT 1`
+  timed out before it could ever acquire the pool — so the kubelet
+  killed every pod every few minutes even though the BEAM was healthy.
+  """
   use MicrowavepropWeb, :controller
 
   alias Ecto.Adapters.SQL
 
+  def live(conn, _params) do
+    conn
+    |> put_resp_content_type("text/plain")
+    |> send_resp(200, "ok")
+  end
+
   def check(conn, _params) do
-    # Verify DB is reachable
     _ = SQL.query!(Microwaveprop.Repo, "SELECT 1")
 
     conn

lib/microwaveprop_web/router.ex

@@ -132,7 +132,11 @@ defmodule MicrowavepropWeb.Router do
     plug :accepts, ["json"]
   end
 
-  # Health check — no pipeline, minimal overhead
+  # Health checks — no pipeline, minimal overhead.
+  # /live = liveness, BEAM only. /health = readiness, also pings Repo.
+  # Pods that serve a BEAM-alive /live but a failing /health get taken
+  # out of Service endpoints without SIGKILL cascading across replicas.
+  get "/live", MicrowavepropWeb.HealthController, :live, log: false
   get "/health", MicrowavepropWeb.HealthController, :check, log: false
 
   # Prometheus scrape endpoint. No pipeline so there's no session /

test/microwaveprop_web/controllers/health_controller_test.exs

@@ -1,10 +1,44 @@
 defmodule MicrowavepropWeb.HealthControllerTest do
   use MicrowavepropWeb.ConnCase, async: true
 
+  alias Ecto.Adapters.SQL
+
   test "GET /health returns 200 ok", %{conn: conn} do
     conn = get(conn, "/health")
 
     assert response(conn, 200) == "ok"
     assert response_content_type(conn, :text) =~ "text/plain"
   end
+
+  test "GET /live returns 200 ok", %{conn: conn} do
+    conn = get(conn, "/live")
+
+    assert response(conn, 200) == "ok"
+    assert response_content_type(conn, :text) =~ "text/plain"
+  end
+
+  # The split between /live and /health is the load-bearing detail of
+  # the production liveness fix. If the Ecto pool is saturated, /health
+  # must fail (so readiness drains the pod) while /live must succeed
+  # (so the kubelet doesn't SIGKILL the BEAM). Invoke the controller
+  # actions without any Ecto sandbox assignment so the DB-touching path
+  # is forced to raise — /live must not raise, /health must.
+  describe "DB dependency" do
+    test "live/2 returns 200 without touching the Repo", %{conn: conn} do
+      # No `Ecto.Adapters.SQL.Sandbox.checkout` has been issued for this
+      # test process in the async: true block, so any attempt to reach
+      # the Repo would raise `DBConnection.OwnershipError`. A clean 200
+      # proves the action never tried.
+      refute conn.assigns[:ecto_sandbox]
+      conn = MicrowavepropWeb.HealthController.live(conn, %{})
+      assert conn.status == 200
+      assert conn.resp_body == "ok"
+    end
+
+    test "check/2 issues a Repo query (proves it's the DB-gated endpoint)", %{conn: conn} do
+      conn = get(conn, "/health")
+      assert response(conn, 200) == "ok"
+      assert {:ok, %{rows: [[1]]}} = SQL.query(Microwaveprop.Repo, "SELECT 1")
+    end
+  end
 end