Graham McIntire
9ffa49a8f2
Prod paired the Oban Pro Smart engine + unique workers with core Oban.Plugins.Lifeline, whose rescue has no unique_violation handling. An orphaned executing CheckExecutorWorker job rescued back to available collides with its already-scheduled successor on oban_jobs_unique_index (23505), crashing the plugin every 60s and leaving ~150 orphans stuck. Switch prod to Oban.Pro.Plugins.DynamicLifeline, which repairs the conflict via Smart.clear_uniq_violation. Dev keeps core Lifeline (Basic engine). Add a shallow /health/live endpoint that does not touch db/redis. k8s liveness and startup probes now target it so a transient dependency outage can't kill or block boot of a healthy pod; readiness keeps the deep /health to gate the load balancer. (deployment.yaml probe/replica change pushed separately, after the image carrying /health/live is live.) Also drop unused POSTGRES_* keys from the secrets example.
105 lines
3.8 KiB
Elixir
105 lines
3.8 KiB
Elixir
defmodule ToweropsWeb.HealthControllerTest do
|
|
use ToweropsWeb.ConnCase, async: false
|
|
|
|
describe "GET /health/live" do
|
|
test "returns 200 with status ok", %{conn: conn} do
|
|
conn = get(conn, "/health/live")
|
|
result = Jason.decode!(response(conn, 200))
|
|
assert result["status"] == "ok"
|
|
end
|
|
|
|
test "stays 200 even when redis is configured but unreachable", %{conn: conn} do
|
|
# Liveness must be decoupled from dependencies: an unreachable redis (or
|
|
# db) must NOT fail liveness, otherwise k8s kills a pod that is merely
|
|
# waiting on a transient dependency blip.
|
|
previous = Application.get_env(:towerops, :redis)
|
|
Application.put_env(:towerops, :redis, host: "198.51.100.1", port: 1, timeout: 50)
|
|
|
|
try do
|
|
conn = get(conn, "/health/live")
|
|
assert response(conn, 200)
|
|
after
|
|
if previous do
|
|
Application.put_env(:towerops, :redis, previous)
|
|
else
|
|
Application.delete_env(:towerops, :redis)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "GET /health" do
|
|
test "returns 200 when database is connected", %{conn: conn} do
|
|
conn = get(conn, "/health")
|
|
body = response(conn, 200)
|
|
result = Jason.decode!(body)
|
|
assert result["status"] == "ok"
|
|
assert result["database"] == "connected"
|
|
# `version` deliberately omitted: exposing the app version on a
|
|
# public health endpoint helps attackers fingerprint vulnerable builds.
|
|
refute Map.has_key?(result, "version")
|
|
end
|
|
|
|
test "reports redis status as not_configured when redis is not configured", %{conn: conn} do
|
|
previous = Application.get_env(:towerops, :redis)
|
|
Application.put_env(:towerops, :redis, [])
|
|
|
|
try do
|
|
conn = get(conn, "/health")
|
|
result = Jason.decode!(response(conn, 200))
|
|
assert result["redis"] == "not_configured"
|
|
after
|
|
if previous do
|
|
Application.put_env(:towerops, :redis, previous)
|
|
else
|
|
Application.delete_env(:towerops, :redis)
|
|
end
|
|
end
|
|
end
|
|
|
|
test "reports redis as connected when configured to a real instance", %{conn: conn} do
|
|
previous = Application.get_env(:towerops, :redis)
|
|
Application.put_env(:towerops, :redis, host: "localhost", port: 6379)
|
|
|
|
try do
|
|
conn = get(conn, "/health")
|
|
# Either 200 (redis up) or 503 (redis unreachable in CI) — both branches
|
|
# exercise the configured-redis code path. We just assert the response
|
|
# is well-formed JSON with a redis field.
|
|
body = response(conn, conn.status)
|
|
result = Jason.decode!(body)
|
|
assert result["redis"] in ["connected", "disconnected"]
|
|
after
|
|
if previous do
|
|
Application.put_env(:towerops, :redis, previous)
|
|
else
|
|
Application.delete_env(:towerops, :redis)
|
|
end
|
|
end
|
|
end
|
|
|
|
test "returns 503 when redis is configured but unreachable", %{conn: conn} do
|
|
previous = Application.get_env(:towerops, :redis)
|
|
# Point at a closed port on a routable but unreachable address.
|
|
# 198.51.100.0/24 is TEST-NET-2 (RFC 5737) — guaranteed not routable on the public internet.
|
|
Application.put_env(:towerops, :redis, host: "198.51.100.1", port: 1, timeout: 50)
|
|
|
|
try do
|
|
conn = get(conn, "/health")
|
|
# Expect 503 with redis disconnected
|
|
body = response(conn, 503)
|
|
result = Jason.decode!(body)
|
|
assert result["status"] == "error"
|
|
# Database is still connected in tests — exercises db_status_string(:ok)
|
|
assert result["database"] == "connected"
|
|
assert result["redis"] == "disconnected"
|
|
after
|
|
if previous do
|
|
Application.put_env(:towerops, :redis, previous)
|
|
else
|
|
Application.delete_env(:towerops, :redis)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|