Graham McIntire
9ffa49a8f2
Prod paired the Oban Pro Smart engine + unique workers with core Oban.Plugins.Lifeline, whose rescue has no unique_violation handling. An orphaned executing CheckExecutorWorker job rescued back to available collides with its already-scheduled successor on oban_jobs_unique_index (23505), crashing the plugin every 60s and leaving ~150 orphans stuck. Switch prod to Oban.Pro.Plugins.DynamicLifeline, which repairs the conflict via Smart.clear_uniq_violation. Dev keeps core Lifeline (Basic engine). Add a shallow /health/live endpoint that does not touch db/redis. k8s liveness and startup probes now target it so a transient dependency outage can't kill or block boot of a healthy pod; readiness keeps the deep /health to gate the load balancer. (deployment.yaml probe/replica change pushed separately, after the image carrying /health/live is live.) Also drop unused POSTGRES_* keys from the secrets example.
93 lines
2.5 KiB
Elixir
93 lines
2.5 KiB
Elixir
defmodule ToweropsWeb.HealthController do
|
|
@moduledoc false
|
|
|
|
use ToweropsWeb, :controller
|
|
|
|
alias Ecto.Adapters.SQL
|
|
alias Towerops.Repo
|
|
|
|
# Liveness probe: confirms the BEAM/web stack is responsive. Deliberately does
|
|
# NOT touch the database or redis — liveness failure causes k8s to kill the
|
|
# pod, so it must not react to transient dependency outages. Dependency health
|
|
# belongs in readiness (index/2), which only removes the pod from the load
|
|
# balancer.
|
|
def live(conn, _params) do
|
|
conn
|
|
|> put_resp_content_type("application/json")
|
|
|> send_resp(200, Jason.encode!(%{status: "ok"}))
|
|
end
|
|
|
|
def index(conn, _params) do
|
|
require Logger
|
|
|
|
db_status = check_database()
|
|
redis_status = check_redis()
|
|
|
|
# Log health check results for debugging
|
|
Logger.debug("Health check: db=#{inspect(db_status)}, redis=#{inspect(redis_status)}")
|
|
|
|
all_healthy = db_status == :ok and redis_status in [:ok, :not_configured]
|
|
|
|
if all_healthy do
|
|
conn
|
|
|> put_resp_content_type("application/json")
|
|
|> send_resp(
|
|
200,
|
|
Jason.encode!(%{
|
|
status: "ok",
|
|
database: "connected",
|
|
redis: redis_status_string(redis_status)
|
|
})
|
|
)
|
|
else
|
|
conn
|
|
|> put_resp_content_type("application/json")
|
|
|> send_resp(
|
|
503,
|
|
Jason.encode!(%{
|
|
status: "error",
|
|
database: db_status_string(db_status),
|
|
redis: redis_status_string(redis_status)
|
|
})
|
|
)
|
|
end
|
|
end
|
|
|
|
defp check_database do
|
|
case SQL.query(Repo, "SELECT 1", []) do
|
|
{:ok, _} -> :ok
|
|
{:error, _} -> :error
|
|
end
|
|
end
|
|
|
|
defp check_redis do
|
|
require Logger
|
|
|
|
redis_config = Application.get_env(:towerops, :redis, [])
|
|
|
|
if Keyword.has_key?(redis_config, :host) do
|
|
# Redis is configured, check connectivity with short timeout for health check
|
|
timeout = Keyword.get(redis_config, :timeout, 2_000)
|
|
|
|
case Towerops.RedisHealthCheck.check_health(redis_config, timeout) do
|
|
:ok ->
|
|
:ok
|
|
|
|
{:error, reason} ->
|
|
Logger.error("Redis health check failed: #{inspect(reason)}")
|
|
:error
|
|
end
|
|
else
|
|
# Redis not configured (e.g., in development without Redis)
|
|
Logger.debug("Redis not configured")
|
|
:not_configured
|
|
end
|
|
end
|
|
|
|
defp db_status_string(:ok), do: "connected"
|
|
defp db_status_string(:error), do: "disconnected"
|
|
|
|
defp redis_status_string(:ok), do: "connected"
|
|
defp redis_status_string(:error), do: "disconnected"
|
|
defp redis_status_string(:not_configured), do: "not_configured"
|
|
end
|