aprs.me/test/aprsme/cluster/leader_election_test.exs
Graham McIntire 753fa50463
Fix ConnectionMonitor deadlock and LeaderElection silent leadership loss
ConnectionMonitor.gather_cluster_stats was calling :rpc.call(Node.self())
which GenServer.call'd back into itself while blocked in handle_info,
causing a 5-second deadlock timeout every 30 seconds. Fixed by computing
local stats directly from state.

LeaderElection never verified that a leader still held the :global
registration. After :global conflict resolution (two isolated nodes
merging), the loser was silently unregistered but stayed is_leader: true
forever, causing both pods to connect to APRS-IS with the same callsign
in a reconnect loop. Fixed by adding verify_leadership/1 to
check_leadership which detects lost registrations and steps down.
2026-02-19 18:18:31 -06:00

343 lines
10 KiB
Elixir

defmodule Aprsme.Cluster.LeaderElectionTest do
use ExUnit.Case, async: false
alias Aprsme.Cluster.LeaderElection
@election_key {:aprs_is_leader, LeaderElection}
setup do
# Clean up any existing global registrations
:global.unregister_name(@election_key)
# Stop any running LeaderElection process
if Process.whereis(LeaderElection) do
GenServer.stop(LeaderElection)
end
on_exit(fn ->
# Cleanup after test
:global.unregister_name(@election_key)
if Process.whereis(LeaderElection) do
try do
GenServer.stop(LeaderElection)
catch
:exit, _ -> :ok
end
end
end)
:ok
end
describe "start_link/1" do
test "starts the GenServer process" do
assert {:ok, pid} = LeaderElection.start_link([])
assert Process.alive?(pid)
assert Process.whereis(LeaderElection) == pid
# Clean up after this specific test
GenServer.stop(pid)
end
test "registers with the given name" do
{:ok, pid} = LeaderElection.start_link([])
assert Process.whereis(LeaderElection)
# Clean up after this specific test
GenServer.stop(pid)
end
end
describe "leader election in non-clustered mode" do
setup do
# Ensure clustering is disabled
Application.put_env(:aprsme, :cluster_enabled, false)
{:ok, pid} = LeaderElection.start_link([])
# Give time for election to occur
Process.sleep(200)
{:ok, pid: pid}
end
test "elects itself as leader when clustering is disabled", %{pid: _pid} do
assert LeaderElection.leader?() == true
assert LeaderElection.current_leader() == node()
end
test "registers itself globally", %{pid: pid} do
assert :global.whereis_name(@election_key) == pid
end
end
describe "leader?/0" do
setup do
Application.put_env(:aprsme, :cluster_enabled, false)
{:ok, _pid} = LeaderElection.start_link([])
Process.sleep(200)
:ok
end
test "returns leadership status" do
assert is_boolean(LeaderElection.leader?())
end
test "cached check matches GenServer state" do
# leader_cached?/0 reads from :persistent_term — no GenServer.call
assert LeaderElection.leader_cached?() == LeaderElection.leader?()
end
test "cached check returns false before election" do
# Stop current instance
GenServer.stop(LeaderElection)
:global.unregister_name({:aprs_is_leader, LeaderElection})
# Clear the persistent_term
:persistent_term.put({LeaderElection, :is_leader}, false)
assert LeaderElection.leader_cached?() == false
end
end
describe "current_leader/0" do
setup do
Application.put_env(:aprsme, :cluster_enabled, false)
{:ok, _pid} = LeaderElection.start_link([])
Process.sleep(200)
:ok
end
test "returns the current leader node" do
leader = LeaderElection.current_leader()
assert leader == node() or is_nil(leader)
end
end
describe "get_cluster_aprs_status/0" do
test "returns local status when clustering is disabled" do
Application.put_env(:aprsme, :cluster_enabled, false)
# Since Aprsme.Is.get_status() is called directly, we need to ensure
# the module exists or mock it at a lower level
# For now, just verify the function doesn't crash
result = LeaderElection.get_cluster_aprs_status()
assert is_map(result) or is_nil(result)
end
test "returns cluster-wide status when clustering is enabled" do
Application.put_env(:aprsme, :cluster_enabled, true)
# This will call get_cluster_wide_status which handles the cluster logic
status = LeaderElection.get_cluster_aprs_status()
# The function should return a map with cluster_info when in cluster mode
assert is_map(status) or is_nil(status)
# If we got a valid status back, it should have cluster info
if is_map(status) do
assert Map.has_key?(status, :cluster_info)
if status.cluster_info do
assert status.cluster_info.total_nodes >= 1
assert is_list(status.cluster_info.all_nodes)
end
end
end
end
describe "leadership transitions" do
setup do
Application.put_env(:aprsme, :cluster_enabled, false)
:ok
end
test "handles termination gracefully when leader" do
{:ok, pid} = LeaderElection.start_link([])
Process.sleep(200)
# Verify it's the leader
assert LeaderElection.leader?() == true
# Stop the process
GenServer.stop(pid)
# Verify global registration is cleaned up
assert :global.whereis_name(@election_key) == :undefined
end
@tag :slow
test "periodic leadership check continues running" do
{:ok, _pid} = LeaderElection.start_link([])
# Initial state - might not be leader yet
_initial_leader = LeaderElection.leader?()
# Wait for periodic check
Process.sleep(6000)
# Should still be able to query leadership
current_leader = LeaderElection.leader?()
assert is_boolean(current_leader)
# In non-clustered mode, should become leader
if not Application.get_env(:aprsme, :cluster_enabled, false) do
assert current_leader == true
end
end
end
describe "conflict resolution" do
test "resolve_conflict prefers lexicographically lower node" do
# We can't directly test the private function, but we can test the behavior
# by starting multiple processes and seeing which wins
# This is more of an integration test that would require multiple nodes
# For now, we just verify the module handles conflicts without crashing
{:ok, pid1} = LeaderElection.start_link([])
Process.sleep(100)
# Try to register another process with the same key
# This should fail or trigger conflict resolution
spawn(fn ->
:global.register_name(@election_key, self())
end)
Process.sleep(100)
# Original process should still be alive
assert Process.alive?(pid1)
end
end
describe "stale registration cleanup" do
@tag :slow
test "cleans up registration when process dies" do
# Register a dead process
dead_pid = spawn(fn -> :ok end)
# Ensure it's dead
Process.sleep(50)
# Force register it globally (simulating stale registration)
:global.re_register_name(@election_key, dead_pid)
# Start LeaderElection which should clean it up
Application.put_env(:aprsme, :cluster_enabled, false)
{:ok, _pid} = LeaderElection.start_link([])
# Wait longer for cleanup and election to occur
Process.sleep(500)
# Should have taken over leadership
assert LeaderElection.leader?() == true
end
end
describe "cluster mode behavior" do
setup do
Application.put_env(:aprsme, :cluster_enabled, true)
:ok
end
@tag :slow
test "waits for cluster formation when enabled" do
{:ok, _pid} = LeaderElection.start_link([])
# Immediately after start, might not be leader yet
Process.sleep(100)
# In cluster mode with no other nodes, it will eventually become leader
# but might take longer than non-clustered mode
Process.sleep(3000)
# Should eventually attempt election
assert is_boolean(LeaderElection.leader?())
end
end
describe "leadership verification" do
setup do
Application.put_env(:aprsme, :cluster_enabled, false)
{:ok, pid} = LeaderElection.start_link([])
Process.sleep(200)
assert LeaderElection.leader?() == true
{:ok, pid: pid}
end
test "steps down when global registration is lost", %{pid: pid} do
# Subscribe to leadership change notifications
Phoenix.PubSub.subscribe(Aprsme.PubSub, "cluster:leadership")
# Simulate losing the global registration (as happens during :global conflict resolution)
:global.unregister_name(@election_key)
# Force a leadership check, then use :sys.get_state to synchronously wait
# for the message to be processed (before the 100ms re-election timer fires)
send(pid, :check_leadership)
state = :sys.get_state(pid)
# Should have detected the loss and stepped down
assert state.is_leader == false
assert LeaderElection.leader_cached?() == false
# Should have notified about leadership loss
assert_received {:leadership_change, _, false}
end
test "steps down when another process holds the registration", %{pid: pid} do
Phoenix.PubSub.subscribe(Aprsme.PubSub, "cluster:leadership")
# Simulate another node winning the conflict resolution
imposter = spawn(fn -> Process.sleep(:infinity) end)
:global.re_register_name(@election_key, imposter)
# Force a leadership check
send(pid, :check_leadership)
state = :sys.get_state(pid)
# Should have stepped down
assert state.is_leader == false
assert LeaderElection.leader_cached?() == false
assert_received {:leadership_change, _, false}
# Clean up
Process.exit(imposter, :kill)
end
test "re-elects after stepping down", %{pid: pid} do
# Lose the registration
:global.unregister_name(@election_key)
# Force check — should step down
send(pid, :check_leadership)
state = :sys.get_state(pid)
assert state.is_leader == false
# check_leadership also schedules :attempt_election for non-leaders.
# Wait for re-election.
Process.sleep(300)
assert LeaderElection.leader?() == true
end
end
describe "message handling" do
setup do
Application.put_env(:aprsme, :cluster_enabled, false)
{:ok, pid} = LeaderElection.start_link([])
{:ok, pid: pid}
end
test "handles unknown messages without crashing", %{pid: pid} do
send(pid, :unknown_message)
Process.sleep(100)
assert Process.alive?(pid)
end
test "handles check_leadership message", %{pid: pid} do
send(pid, :check_leadership)
Process.sleep(100)
assert Process.alive?(pid)
end
end
end