ConnectionMonitor.gather_cluster_stats was calling :rpc.call(Node.self()) which GenServer.call'd back into itself while blocked in handle_info, causing a 5-second deadlock timeout every 30 seconds. Fixed by computing local stats directly from state. LeaderElection never verified that a leader still held the :global registration. After :global conflict resolution (two isolated nodes merging), the loser was silently unregistered but stayed is_leader: true forever, causing both pods to connect to APRS-IS with the same callsign in a reconnect loop. Fixed by adding verify_leadership/1 to check_leadership which detects lost registrations and steps down.
343 lines
10 KiB
Elixir
343 lines
10 KiB
Elixir
defmodule Aprsme.Cluster.LeaderElectionTest do
|
|
use ExUnit.Case, async: false
|
|
|
|
alias Aprsme.Cluster.LeaderElection
|
|
|
|
@election_key {:aprs_is_leader, LeaderElection}
|
|
|
|
setup do
|
|
# Clean up any existing global registrations
|
|
:global.unregister_name(@election_key)
|
|
|
|
# Stop any running LeaderElection process
|
|
if Process.whereis(LeaderElection) do
|
|
GenServer.stop(LeaderElection)
|
|
end
|
|
|
|
on_exit(fn ->
|
|
# Cleanup after test
|
|
:global.unregister_name(@election_key)
|
|
|
|
if Process.whereis(LeaderElection) do
|
|
try do
|
|
GenServer.stop(LeaderElection)
|
|
catch
|
|
:exit, _ -> :ok
|
|
end
|
|
end
|
|
end)
|
|
|
|
:ok
|
|
end
|
|
|
|
describe "start_link/1" do
|
|
test "starts the GenServer process" do
|
|
assert {:ok, pid} = LeaderElection.start_link([])
|
|
assert Process.alive?(pid)
|
|
assert Process.whereis(LeaderElection) == pid
|
|
|
|
# Clean up after this specific test
|
|
GenServer.stop(pid)
|
|
end
|
|
|
|
test "registers with the given name" do
|
|
{:ok, pid} = LeaderElection.start_link([])
|
|
assert Process.whereis(LeaderElection)
|
|
|
|
# Clean up after this specific test
|
|
GenServer.stop(pid)
|
|
end
|
|
end
|
|
|
|
describe "leader election in non-clustered mode" do
|
|
setup do
|
|
# Ensure clustering is disabled
|
|
Application.put_env(:aprsme, :cluster_enabled, false)
|
|
{:ok, pid} = LeaderElection.start_link([])
|
|
|
|
# Give time for election to occur
|
|
Process.sleep(200)
|
|
|
|
{:ok, pid: pid}
|
|
end
|
|
|
|
test "elects itself as leader when clustering is disabled", %{pid: _pid} do
|
|
assert LeaderElection.leader?() == true
|
|
assert LeaderElection.current_leader() == node()
|
|
end
|
|
|
|
test "registers itself globally", %{pid: pid} do
|
|
assert :global.whereis_name(@election_key) == pid
|
|
end
|
|
end
|
|
|
|
describe "leader?/0" do
|
|
setup do
|
|
Application.put_env(:aprsme, :cluster_enabled, false)
|
|
{:ok, _pid} = LeaderElection.start_link([])
|
|
Process.sleep(200)
|
|
:ok
|
|
end
|
|
|
|
test "returns leadership status" do
|
|
assert is_boolean(LeaderElection.leader?())
|
|
end
|
|
|
|
test "cached check matches GenServer state" do
|
|
# leader_cached?/0 reads from :persistent_term — no GenServer.call
|
|
assert LeaderElection.leader_cached?() == LeaderElection.leader?()
|
|
end
|
|
|
|
test "cached check returns false before election" do
|
|
# Stop current instance
|
|
GenServer.stop(LeaderElection)
|
|
:global.unregister_name({:aprs_is_leader, LeaderElection})
|
|
|
|
# Clear the persistent_term
|
|
:persistent_term.put({LeaderElection, :is_leader}, false)
|
|
|
|
assert LeaderElection.leader_cached?() == false
|
|
end
|
|
end
|
|
|
|
describe "current_leader/0" do
|
|
setup do
|
|
Application.put_env(:aprsme, :cluster_enabled, false)
|
|
{:ok, _pid} = LeaderElection.start_link([])
|
|
Process.sleep(200)
|
|
:ok
|
|
end
|
|
|
|
test "returns the current leader node" do
|
|
leader = LeaderElection.current_leader()
|
|
assert leader == node() or is_nil(leader)
|
|
end
|
|
end
|
|
|
|
describe "get_cluster_aprs_status/0" do
|
|
test "returns local status when clustering is disabled" do
|
|
Application.put_env(:aprsme, :cluster_enabled, false)
|
|
|
|
# Since Aprsme.Is.get_status() is called directly, we need to ensure
|
|
# the module exists or mock it at a lower level
|
|
# For now, just verify the function doesn't crash
|
|
result = LeaderElection.get_cluster_aprs_status()
|
|
assert is_map(result) or is_nil(result)
|
|
end
|
|
|
|
test "returns cluster-wide status when clustering is enabled" do
|
|
Application.put_env(:aprsme, :cluster_enabled, true)
|
|
|
|
# This will call get_cluster_wide_status which handles the cluster logic
|
|
status = LeaderElection.get_cluster_aprs_status()
|
|
|
|
# The function should return a map with cluster_info when in cluster mode
|
|
assert is_map(status) or is_nil(status)
|
|
|
|
# If we got a valid status back, it should have cluster info
|
|
if is_map(status) do
|
|
assert Map.has_key?(status, :cluster_info)
|
|
|
|
if status.cluster_info do
|
|
assert status.cluster_info.total_nodes >= 1
|
|
assert is_list(status.cluster_info.all_nodes)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "leadership transitions" do
|
|
setup do
|
|
Application.put_env(:aprsme, :cluster_enabled, false)
|
|
:ok
|
|
end
|
|
|
|
test "handles termination gracefully when leader" do
|
|
{:ok, pid} = LeaderElection.start_link([])
|
|
Process.sleep(200)
|
|
|
|
# Verify it's the leader
|
|
assert LeaderElection.leader?() == true
|
|
|
|
# Stop the process
|
|
GenServer.stop(pid)
|
|
|
|
# Verify global registration is cleaned up
|
|
assert :global.whereis_name(@election_key) == :undefined
|
|
end
|
|
|
|
@tag :slow
|
|
test "periodic leadership check continues running" do
|
|
{:ok, _pid} = LeaderElection.start_link([])
|
|
|
|
# Initial state - might not be leader yet
|
|
_initial_leader = LeaderElection.leader?()
|
|
|
|
# Wait for periodic check
|
|
Process.sleep(6000)
|
|
|
|
# Should still be able to query leadership
|
|
current_leader = LeaderElection.leader?()
|
|
assert is_boolean(current_leader)
|
|
|
|
# In non-clustered mode, should become leader
|
|
if not Application.get_env(:aprsme, :cluster_enabled, false) do
|
|
assert current_leader == true
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "conflict resolution" do
|
|
test "resolve_conflict prefers lexicographically lower node" do
|
|
# We can't directly test the private function, but we can test the behavior
|
|
# by starting multiple processes and seeing which wins
|
|
|
|
# This is more of an integration test that would require multiple nodes
|
|
# For now, we just verify the module handles conflicts without crashing
|
|
|
|
{:ok, pid1} = LeaderElection.start_link([])
|
|
Process.sleep(100)
|
|
|
|
# Try to register another process with the same key
|
|
# This should fail or trigger conflict resolution
|
|
spawn(fn ->
|
|
:global.register_name(@election_key, self())
|
|
end)
|
|
|
|
Process.sleep(100)
|
|
|
|
# Original process should still be alive
|
|
assert Process.alive?(pid1)
|
|
end
|
|
end
|
|
|
|
describe "stale registration cleanup" do
|
|
@tag :slow
|
|
test "cleans up registration when process dies" do
|
|
# Register a dead process
|
|
dead_pid = spawn(fn -> :ok end)
|
|
# Ensure it's dead
|
|
Process.sleep(50)
|
|
|
|
# Force register it globally (simulating stale registration)
|
|
:global.re_register_name(@election_key, dead_pid)
|
|
|
|
# Start LeaderElection which should clean it up
|
|
Application.put_env(:aprsme, :cluster_enabled, false)
|
|
{:ok, _pid} = LeaderElection.start_link([])
|
|
|
|
# Wait longer for cleanup and election to occur
|
|
Process.sleep(500)
|
|
|
|
# Should have taken over leadership
|
|
assert LeaderElection.leader?() == true
|
|
end
|
|
end
|
|
|
|
describe "cluster mode behavior" do
|
|
setup do
|
|
Application.put_env(:aprsme, :cluster_enabled, true)
|
|
:ok
|
|
end
|
|
|
|
@tag :slow
|
|
test "waits for cluster formation when enabled" do
|
|
{:ok, _pid} = LeaderElection.start_link([])
|
|
|
|
# Immediately after start, might not be leader yet
|
|
Process.sleep(100)
|
|
|
|
# In cluster mode with no other nodes, it will eventually become leader
|
|
# but might take longer than non-clustered mode
|
|
Process.sleep(3000)
|
|
|
|
# Should eventually attempt election
|
|
assert is_boolean(LeaderElection.leader?())
|
|
end
|
|
end
|
|
|
|
describe "leadership verification" do
|
|
setup do
|
|
Application.put_env(:aprsme, :cluster_enabled, false)
|
|
{:ok, pid} = LeaderElection.start_link([])
|
|
Process.sleep(200)
|
|
assert LeaderElection.leader?() == true
|
|
{:ok, pid: pid}
|
|
end
|
|
|
|
test "steps down when global registration is lost", %{pid: pid} do
|
|
# Subscribe to leadership change notifications
|
|
Phoenix.PubSub.subscribe(Aprsme.PubSub, "cluster:leadership")
|
|
|
|
# Simulate losing the global registration (as happens during :global conflict resolution)
|
|
:global.unregister_name(@election_key)
|
|
|
|
# Force a leadership check, then use :sys.get_state to synchronously wait
|
|
# for the message to be processed (before the 100ms re-election timer fires)
|
|
send(pid, :check_leadership)
|
|
state = :sys.get_state(pid)
|
|
|
|
# Should have detected the loss and stepped down
|
|
assert state.is_leader == false
|
|
assert LeaderElection.leader_cached?() == false
|
|
|
|
# Should have notified about leadership loss
|
|
assert_received {:leadership_change, _, false}
|
|
end
|
|
|
|
test "steps down when another process holds the registration", %{pid: pid} do
|
|
Phoenix.PubSub.subscribe(Aprsme.PubSub, "cluster:leadership")
|
|
|
|
# Simulate another node winning the conflict resolution
|
|
imposter = spawn(fn -> Process.sleep(:infinity) end)
|
|
:global.re_register_name(@election_key, imposter)
|
|
|
|
# Force a leadership check
|
|
send(pid, :check_leadership)
|
|
state = :sys.get_state(pid)
|
|
|
|
# Should have stepped down
|
|
assert state.is_leader == false
|
|
assert LeaderElection.leader_cached?() == false
|
|
assert_received {:leadership_change, _, false}
|
|
|
|
# Clean up
|
|
Process.exit(imposter, :kill)
|
|
end
|
|
|
|
test "re-elects after stepping down", %{pid: pid} do
|
|
# Lose the registration
|
|
:global.unregister_name(@election_key)
|
|
|
|
# Force check — should step down
|
|
send(pid, :check_leadership)
|
|
state = :sys.get_state(pid)
|
|
assert state.is_leader == false
|
|
|
|
# check_leadership also schedules :attempt_election for non-leaders.
|
|
# Wait for re-election.
|
|
Process.sleep(300)
|
|
assert LeaderElection.leader?() == true
|
|
end
|
|
end
|
|
|
|
describe "message handling" do
|
|
setup do
|
|
Application.put_env(:aprsme, :cluster_enabled, false)
|
|
{:ok, pid} = LeaderElection.start_link([])
|
|
{:ok, pid: pid}
|
|
end
|
|
|
|
test "handles unknown messages without crashing", %{pid: pid} do
|
|
send(pid, :unknown_message)
|
|
Process.sleep(100)
|
|
assert Process.alive?(pid)
|
|
end
|
|
|
|
test "handles check_leadership message", %{pid: pid} do
|
|
send(pid, :check_leadership)
|
|
Process.sleep(100)
|
|
assert Process.alive?(pid)
|
|
end
|
|
end
|
|
end
|