From 9ffa49a8f2ac5c99988c59d0684fc85f5ec681b4 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Wed, 20 May 2026 12:07:29 -0500 Subject: [PATCH] fix(oban): use DynamicLifeline in prod; add /health/live + split k8s probes Prod paired the Oban Pro Smart engine + unique workers with core Oban.Plugins.Lifeline, whose rescue has no unique_violation handling. An orphaned executing CheckExecutorWorker job rescued back to available collides with its already-scheduled successor on oban_jobs_unique_index (23505), crashing the plugin every 60s and leaving ~150 orphans stuck. Switch prod to Oban.Pro.Plugins.DynamicLifeline, which repairs the conflict via Smart.clear_uniq_violation. Dev keeps core Lifeline (Basic engine). Add a shallow /health/live endpoint that does not touch db/redis. k8s liveness and startup probes now target it so a transient dependency outage can't kill or block boot of a healthy pod; readiness keeps the deep /health to gate the load balancer. (deployment.yaml probe/replica change pushed separately, after the image carrying /health/live is live.) Also drop unused POSTGRES_* keys from the secrets example. --- CHANGELOG.txt | 23 ++++++++++++++++ config/dev.exs | 5 +++- config/runtime.exs | 9 +++++-- k8s/secrets.example.yaml | 7 +---- .../controllers/health_controller.ex | 11 ++++++++ lib/towerops_web/router.ex | 6 ++++- priv/static/changelog.txt | 4 +++ .../controllers/health_controller_test.exs | 27 +++++++++++++++++++ 8 files changed, 82 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 52873af1..5d1d3bfb 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,26 @@ +2026-05-20 +fix(oban): use Oban Pro DynamicLifeline in prod to stop unique_violation crash loop + CheckExecutorWorker's `unique` constraint covers available/scheduled/retryable + (executing excluded so perform/1 can self-schedule). When a pod is killed + mid-execution the job is orphaned in `executing`; core Oban.Plugins.Lifeline + rescues it back to `available`, where it collides with the already-scheduled + successor (same uniq_key) on oban_jobs_unique_index, raising 23505. Core + Lifeline has no unique_violation handling, so the plugin crashed every 60s and + ~150 orphans never cleared. Oban Pro's DynamicLifeline (the Smart-engine + plugin) catches the 23505 via Smart.clear_uniq_violation and repairs it. + Prod only (runtime.exs, Smart engine); dev keeps core Lifeline (Basic engine). + DynamicLifeline rescues by producer records, so rescue_after was dropped. + Files: config/runtime.exs, config/dev.exs + +feat(health): add shallow /health/live liveness endpoint and split k8s probes + /health/live returns 200 without querying db/redis. k8s liveness and startup + probes now target /health/live so a transient db/redis outage can no longer + kill or block boot of an otherwise-healthy pod; readiness keeps the deep + /health (db+redis) to gate load-balancer membership only. Deployment bumped to + 2 replicas (NFS data volume supports multi-pod mounts; app already clusters). + Files: lib/towerops_web/router.ex, lib/towerops_web/controllers/health_controller.ex, + test/towerops_web/controllers/health_controller_test.exs, k8s/deployment.yaml + 2026-05-14 fix(monitoring): add Oban unique constraint to CheckExecutorWorker to stop duplicate job buildup CheckExecutorWorker had no `unique` constraint. Every insert path — diff --git a/config/dev.exs b/config/dev.exs index db9d17f1..7b7512d8 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -119,7 +119,10 @@ config :towerops, Oban, {Oban.Plugins.Pruner, max_age: 60}, # Rescue orphaned jobs (when node crashes) {Oban.Plugins.Reindexer, schedule: "0 2 * * *"}, - # Rescue jobs stuck in executing state (mark as cancelled after 10 minutes) + # Dev uses the core Basic engine (the Pro Smart engine is prod-only, see + # runtime.exs), so core Lifeline is correct here. Prod uses + # Oban.Pro.Plugins.DynamicLifeline because core Lifeline crashes on the + # unique_violation that the Smart engine's unique index can raise on rescue. {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 10)} ] diff --git a/config/runtime.exs b/config/runtime.exs index 5878bbf6..4407a9b2 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -336,8 +336,13 @@ if config_env() == :prod do "oban_jobs_args_index", "oban_jobs_meta_index" ]}, - # Rescue jobs stuck in executing state (mark as cancelled after 10 minutes) - {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 10)} + # Rescue jobs orphaned in the executing state. DynamicLifeline (not core + # Lifeline) is required with the Pro Smart engine + unique workers: it + # catches the unique_violation that occurs when a rescued orphan collides + # with an already-scheduled successor (same uniq_key) and repairs it, + # whereas core Lifeline crashes on that 23505. It rescues by producer + # records rather than a time threshold, so rescue_after does not apply. + Oban.Pro.Plugins.DynamicLifeline ] # ## SSL Support diff --git a/k8s/secrets.example.yaml b/k8s/secrets.example.yaml index f067281e..989c7eab 100644 --- a/k8s/secrets.example.yaml +++ b/k8s/secrets.example.yaml @@ -40,13 +40,8 @@ metadata: namespace: towerops type: Opaque stringData: - # Full ecto:// URL — what runtime.exs reads. + # Full ecto:// URL — the only DB var the app reads (runtime.exs). DATABASE_URL: "postgresql://towerops:CHANGE_ME@postgres-host:5432/towerops" - POSTGRES_HOST: "CHANGE_ME_postgres_host" - POSTGRES_PORT: "5432" - POSTGRES_DB: "towerops" - POSTGRES_USER: "towerops" - POSTGRES_PASSWORD: "CHANGE_ME" --- apiVersion: v1 kind: Secret diff --git a/lib/towerops_web/controllers/health_controller.ex b/lib/towerops_web/controllers/health_controller.ex index f8ed82d1..551ce1b6 100644 --- a/lib/towerops_web/controllers/health_controller.ex +++ b/lib/towerops_web/controllers/health_controller.ex @@ -6,6 +6,17 @@ defmodule ToweropsWeb.HealthController do alias Ecto.Adapters.SQL alias Towerops.Repo + # Liveness probe: confirms the BEAM/web stack is responsive. Deliberately does + # NOT touch the database or redis — liveness failure causes k8s to kill the + # pod, so it must not react to transient dependency outages. Dependency health + # belongs in readiness (index/2), which only removes the pod from the load + # balancer. + def live(conn, _params) do + conn + |> put_resp_content_type("application/json") + |> send_resp(200, Jason.encode!(%{status: "ok"})) + end + def index(conn, _params) do require Logger diff --git a/lib/towerops_web/router.ex b/lib/towerops_web/router.ex index aa4e61ee..73c7bf27 100644 --- a/lib/towerops_web/router.ex +++ b/lib/towerops_web/router.ex @@ -66,9 +66,13 @@ defmodule ToweropsWeb.Router do plug ToweropsWeb.GraphQL.Context end - # Health check endpoint for Kubernetes probes (no authentication required) + # Health check endpoints for Kubernetes probes (no authentication required). + # /health is a deep check (db + redis) used for readiness/startup; /health/live + # is a shallow liveness check that must not depend on external services, so a + # transient db/redis blip can't make k8s kill an otherwise-healthy pod. scope "/", ToweropsWeb do get "/health", HealthController, :index, log: false + get "/health/live", HealthController, :live, log: false end # Public crawlable endpoints (no pipeline, no CSRF/session required) diff --git a/priv/static/changelog.txt b/priv/static/changelog.txt index f123e2a6..de2d74eb 100644 --- a/priv/static/changelog.txt +++ b/priv/static/changelog.txt @@ -1,3 +1,7 @@ +2026-05-20 — Reliability improvements +* The app now rides through brief database hiccups instead of restarting, reducing momentary unavailability +* Background monitoring jobs recover cleanly after a restart instead of getting stuck + 2026-05-14 — Monitoring check reliability fix * Fixed an issue where monitoring checks could build up a large backlog of duplicate scheduled runs, delaying check execution * Checks now stay on schedule instead of falling behind diff --git a/test/towerops_web/controllers/health_controller_test.exs b/test/towerops_web/controllers/health_controller_test.exs index d82ce41f..00d88b34 100644 --- a/test/towerops_web/controllers/health_controller_test.exs +++ b/test/towerops_web/controllers/health_controller_test.exs @@ -1,6 +1,33 @@ defmodule ToweropsWeb.HealthControllerTest do use ToweropsWeb.ConnCase, async: false + describe "GET /health/live" do + test "returns 200 with status ok", %{conn: conn} do + conn = get(conn, "/health/live") + result = Jason.decode!(response(conn, 200)) + assert result["status"] == "ok" + end + + test "stays 200 even when redis is configured but unreachable", %{conn: conn} do + # Liveness must be decoupled from dependencies: an unreachable redis (or + # db) must NOT fail liveness, otherwise k8s kills a pod that is merely + # waiting on a transient dependency blip. + previous = Application.get_env(:towerops, :redis) + Application.put_env(:towerops, :redis, host: "198.51.100.1", port: 1, timeout: 50) + + try do + conn = get(conn, "/health/live") + assert response(conn, 200) + after + if previous do + Application.put_env(:towerops, :redis, previous) + else + Application.delete_env(:towerops, :redis) + end + end + end + end + describe "GET /health" do test "returns 200 when database is connected", %{conn: conn} do conn = get(conn, "/health")