Graham McIntire
b5e3c1f977
Three changes targeting the root cause of production up/down alerting: 1. Skip BruteForceProtection for /health and /health/live paths (removes unnecessary Repo.get_by(IpBlock) DB query from every k8s probe, matching the existing /socket/agent exemption). 2. Relax k8s readiness probe: periodSeconds 5→10, timeoutSeconds 2→5, failureThreshold 2→3, successThreshold 3→2 so transient DB/Redis blips don't cascade into full service unavailability. 3. Use the persistent Towerops.Redix connection for health checks instead of opening a brand-new TCP connection (handshake→AUTH→ PING→close) on every probe. Towerops.Redix.Fake gains a set_ping_response/2 toggle for testing failure paths.
112 lines
3.9 KiB
Elixir
112 lines
3.9 KiB
Elixir
defmodule ToweropsWeb.HealthControllerTest do
|
|
use ToweropsWeb.ConnCase, async: false
|
|
|
|
alias Towerops.Redix.Fake
|
|
|
|
describe "GET /health/live" do
|
|
test "returns 200 with status ok", %{conn: conn} do
|
|
conn = get(conn, "/health/live")
|
|
result = Jason.decode!(response(conn, 200))
|
|
assert result["status"] == "ok"
|
|
end
|
|
|
|
test "stays 200 even when redis is configured but unreachable", %{conn: conn} do
|
|
# Liveness must be decoupled from dependencies: an unreachable redis (or
|
|
# db) must NOT fail liveness, otherwise k8s kills a pod that is merely
|
|
# waiting on a transient dependency blip.
|
|
previous = Application.get_env(:towerops, :redis)
|
|
Application.put_env(:towerops, :redis, host: "198.51.100.1", port: 1, timeout: 50)
|
|
|
|
try do
|
|
conn = get(conn, "/health/live")
|
|
assert response(conn, 200)
|
|
after
|
|
if previous do
|
|
Application.put_env(:towerops, :redis, previous)
|
|
else
|
|
Application.delete_env(:towerops, :redis)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "GET /health" do
|
|
test "returns 200 when database is connected", %{conn: conn} do
|
|
conn = get(conn, "/health")
|
|
body = response(conn, 200)
|
|
result = Jason.decode!(body)
|
|
assert result["status"] == "ok"
|
|
assert result["database"] == "connected"
|
|
# `version` deliberately omitted: exposing the app version on a
|
|
# public health endpoint helps attackers fingerprint vulnerable builds.
|
|
refute Map.has_key?(result, "version")
|
|
end
|
|
|
|
test "reports redis status as not_configured when redis is not configured", %{conn: conn} do
|
|
previous = Application.get_env(:towerops, :redis)
|
|
Application.put_env(:towerops, :redis, [])
|
|
|
|
try do
|
|
conn = get(conn, "/health")
|
|
result = Jason.decode!(response(conn, 200))
|
|
assert result["redis"] == "not_configured"
|
|
after
|
|
if previous do
|
|
Application.put_env(:towerops, :redis, previous)
|
|
else
|
|
Application.delete_env(:towerops, :redis)
|
|
end
|
|
end
|
|
end
|
|
|
|
test "reports redis as connected when configured to a real instance", %{conn: conn} do
|
|
previous = Application.get_env(:towerops, :redis)
|
|
Application.put_env(:towerops, :redis, host: "localhost", port: 6379)
|
|
|
|
try do
|
|
conn = get(conn, "/health")
|
|
# Either 200 (redis up) or 503 (redis unreachable in CI) — both branches
|
|
# exercise the configured-redis code path. We just assert the response
|
|
# is well-formed JSON with a redis field.
|
|
body = response(conn, conn.status)
|
|
result = Jason.decode!(body)
|
|
assert result["redis"] in ["connected", "disconnected"]
|
|
after
|
|
if previous do
|
|
Application.put_env(:towerops, :redis, previous)
|
|
else
|
|
Application.delete_env(:towerops, :redis)
|
|
end
|
|
end
|
|
end
|
|
|
|
test "returns 503 when redis is configured but unreachable", %{conn: conn} do
|
|
previous = Application.get_env(:towerops, :redis)
|
|
Application.put_env(:towerops, :redis, host: "localhost", port: 6379)
|
|
|
|
# Make the Fake return connection errors for PING so we exercise
|
|
# the Redis-down branch without trying to open a real socket.
|
|
# The Fake is registered as `Towerops.Redix`, not by its module name.
|
|
Fake.set_ping_response(Towerops.Redix, :error)
|
|
|
|
try do
|
|
conn = get(conn, "/health")
|
|
# Expect 503 with redis disconnected
|
|
body = response(conn, 503)
|
|
result = Jason.decode!(body)
|
|
assert result["status"] == "error"
|
|
# Database is still connected in tests — exercises db_status_string(:ok)
|
|
assert result["database"] == "connected"
|
|
assert result["redis"] == "disconnected"
|
|
after
|
|
Fake.set_ping_response(Towerops.Redix, :ok)
|
|
|
|
if previous do
|
|
Application.put_env(:towerops, :redis, previous)
|
|
else
|
|
Application.delete_env(:towerops, :redis)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|