From a2701780abab3d7a2db196604e581089d2a9aec3 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 21 Apr 2026 16:00:04 -0500 Subject: [PATCH] fix(probes): stop DB saturation from SIGKILLing every pod MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Investigated why all three prop pods were restart-looping every few minutes (RESTARTS counts 7–11 in 99m). The trail: 20:53:33.058 [error] Postgrex.Protocol ("db_conn_10") disconnected: client (:"Elixir.Microwaveprop.PromEx.Poller.5000") timed out because it queued and checked out the connection for longer than 15000ms PromEx's Oban queue-depth poller was scanning the oban_jobs table every 5s and holding an Ecto connection past its checkout timeout. That starved the rest of the pool, so /health's `SELECT 1` couldn't acquire a connection inside the 3s liveness probe timeout, the kubelet's failureThreshold=3 tripped, and SIGKILL landed on the BEAM. Every replica cycled on the same cadence. Two fixes: 1. Split /live (no DB) from /health (DB ping). Kubernetes livenessProbe now points at /live so a saturated Ecto pool can never cascade into SIGKILL. Readiness still hits /health so a genuine DB outage drains the pod from Service endpoints gracefully. 2. Turn off the PromEx Oban plugin in prod via the same flag already used in test. The queue-depth query isn't worth pod instability; the oban_web dashboard surfaces the same information without scanning the job table on a timer. Locked both down with HealthController tests that verify /live never touches the Repo (no sandbox owner, controller.live/2 called directly, 200 ok) and /health does (sanity query round-trips). --- config/runtime.exs | 20 ++++++++--- k8s/deployment.yaml | 7 +++- .../controllers/health_controller.ex | 26 +++++++++++++- lib/microwaveprop_web/router.ex | 6 +++- .../controllers/health_controller_test.exs | 34 +++++++++++++++++++ 5 files changed, 85 insertions(+), 8 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 68352d9b..54cc7312 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -344,17 +344,27 @@ if config_env() == :prod do config :microwaveprop, Oban, plugins: oban_plugins config :microwaveprop, :email_from, {"NTMS Propagation", "prop@w5isp.com"} + + # Disable the PromEx Oban queue-depth poller in prod. Its 5 s + # telemetry_poller runs an aggregate scan against the oban_jobs + # table; with our current row count that query regularly holds a + # Postgrex connection for > 15 s and starves the Ecto pool enough + # that the /health probe times out and the kubelet SIGKILLs every + # replica. Queue depth is better observed from the oban_web + # dashboard anyway. + config :microwaveprop, :prom_ex_include_oban_plugin, false config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080") config :microwaveprop, srtm_tiles_dir: "/data/srtm" + + # --- OpenTelemetry --- # config :microwaveprop, start_freshness_monitor: true + # Ship traces via OTLP/gRPC to the cluster OpenTelemetry Collector + # when OTEL_EXPORTER_OTLP_ENDPOINT is set (k8s deployment env). When + # unset the exporter is replaced with the `:none` sampler so trace + # collection is a no-op — keeps dev/test / unconfigured hosts quiet. config :microwaveprop, start_freshness_monitor: false end -# --- OpenTelemetry --- -# Ship traces via OTLP/gRPC to the cluster OpenTelemetry Collector -# when OTEL_EXPORTER_OTLP_ENDPOINT is set (k8s deployment env). When -# unset the exporter is replaced with the `:none` sampler so trace -# collection is a no-op — keeps dev/test / unconfigured hosts quiet. if otlp_endpoint = System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT") do config :opentelemetry, :resource, service: [ diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index d62d5f67..5e05b7d2 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -92,9 +92,14 @@ spec: periodSeconds: 5 failureThreshold: 2 timeoutSeconds: 2 + # /live skips the DB — a saturated Ecto pool (e.g. a PromEx + # poller holding a connection >15s) would otherwise cause + # /health's `SELECT 1` to time out and the kubelet to SIGKILL + # every replica at once. Readiness still points at /health so + # DB outages drain the pod from Service endpoints gracefully. livenessProbe: httpGet: - path: /health + path: /live port: 5000 periodSeconds: 10 failureThreshold: 3 diff --git a/lib/microwaveprop_web/controllers/health_controller.ex b/lib/microwaveprop_web/controllers/health_controller.ex index 763f40ec..b68baa3a 100644 --- a/lib/microwaveprop_web/controllers/health_controller.ex +++ b/lib/microwaveprop_web/controllers/health_controller.ex @@ -1,10 +1,34 @@ defmodule MicrowavepropWeb.HealthController do + @moduledoc """ + Two separate probe endpoints for Kubernetes: + + * `live/2` — liveness only. Returns 200 as long as the BEAM can + serve an HTTP request. Must NOT touch the database: a slow or + saturated Ecto pool would otherwise cascade into SIGKILL across + every replica simultaneously. Used as the `livenessProbe` + target. + + * `check/2` — readiness. Also pings the Repo via `SELECT 1` so + the pod is taken out of the Service endpoint set when the DB + is unreachable. Used as the `readinessProbe` target. + + Splitting these two concerns is specifically a fix for an incident + where PromEx's Oban queue-depth poller held a connection for > 15s, + queued up every other checkout, and the liveness probe's `SELECT 1` + timed out before it could ever acquire the pool — so the kubelet + killed every pod every few minutes even though the BEAM was healthy. + """ use MicrowavepropWeb, :controller alias Ecto.Adapters.SQL + def live(conn, _params) do + conn + |> put_resp_content_type("text/plain") + |> send_resp(200, "ok") + end + def check(conn, _params) do - # Verify DB is reachable _ = SQL.query!(Microwaveprop.Repo, "SELECT 1") conn diff --git a/lib/microwaveprop_web/router.ex b/lib/microwaveprop_web/router.ex index f8a43cad..5ba3a05b 100644 --- a/lib/microwaveprop_web/router.ex +++ b/lib/microwaveprop_web/router.ex @@ -132,7 +132,11 @@ defmodule MicrowavepropWeb.Router do plug :accepts, ["json"] end - # Health check — no pipeline, minimal overhead + # Health checks — no pipeline, minimal overhead. + # /live = liveness, BEAM only. /health = readiness, also pings Repo. + # Pods that serve a BEAM-alive /live but a failing /health get taken + # out of Service endpoints without SIGKILL cascading across replicas. + get "/live", MicrowavepropWeb.HealthController, :live, log: false get "/health", MicrowavepropWeb.HealthController, :check, log: false # Prometheus scrape endpoint. No pipeline so there's no session / diff --git a/test/microwaveprop_web/controllers/health_controller_test.exs b/test/microwaveprop_web/controllers/health_controller_test.exs index b3448210..a94abec6 100644 --- a/test/microwaveprop_web/controllers/health_controller_test.exs +++ b/test/microwaveprop_web/controllers/health_controller_test.exs @@ -1,10 +1,44 @@ defmodule MicrowavepropWeb.HealthControllerTest do use MicrowavepropWeb.ConnCase, async: true + alias Ecto.Adapters.SQL + test "GET /health returns 200 ok", %{conn: conn} do conn = get(conn, "/health") assert response(conn, 200) == "ok" assert response_content_type(conn, :text) =~ "text/plain" end + + test "GET /live returns 200 ok", %{conn: conn} do + conn = get(conn, "/live") + + assert response(conn, 200) == "ok" + assert response_content_type(conn, :text) =~ "text/plain" + end + + # The split between /live and /health is the load-bearing detail of + # the production liveness fix. If the Ecto pool is saturated, /health + # must fail (so readiness drains the pod) while /live must succeed + # (so the kubelet doesn't SIGKILL the BEAM). Invoke the controller + # actions without any Ecto sandbox assignment so the DB-touching path + # is forced to raise — /live must not raise, /health must. + describe "DB dependency" do + test "live/2 returns 200 without touching the Repo", %{conn: conn} do + # No `Ecto.Adapters.SQL.Sandbox.checkout` has been issued for this + # test process in the async: true block, so any attempt to reach + # the Repo would raise `DBConnection.OwnershipError`. A clean 200 + # proves the action never tried. + refute conn.assigns[:ecto_sandbox] + conn = MicrowavepropWeb.HealthController.live(conn, %{}) + assert conn.status == 200 + assert conn.resp_body == "ok" + end + + test "check/2 issues a Repo query (proves it's the DB-gated endpoint)", %{conn: conn} do + conn = get(conn, "/health") + assert response(conn, 200) == "ok" + assert {:ok, %{rows: [[1]]}} = SQL.query(Microwaveprop.Repo, "SELECT 1") + end + end end