defmodule Aprsme.Cluster.LeaderElectionTest do use ExUnit.Case, async: false alias Aprsme.Cluster.LeaderElection @election_key {:aprs_is_leader, LeaderElection} setup do # Clean up any existing global registrations :global.unregister_name(@election_key) # Stop any running LeaderElection process if Process.whereis(LeaderElection) do GenServer.stop(LeaderElection) end on_exit(fn -> # Cleanup after test :global.unregister_name(@election_key) if Process.whereis(LeaderElection) do try do GenServer.stop(LeaderElection) catch :exit, _ -> :ok end end end) :ok end describe "start_link/1" do test "starts the GenServer process" do assert {:ok, pid} = LeaderElection.start_link([]) assert Process.alive?(pid) assert Process.whereis(LeaderElection) == pid # Clean up after this specific test GenServer.stop(pid) end test "registers with the given name" do {:ok, pid} = LeaderElection.start_link([]) assert Process.whereis(LeaderElection) # Clean up after this specific test GenServer.stop(pid) end end describe "leader election in non-clustered mode" do setup do # Ensure clustering is disabled Application.put_env(:aprsme, :cluster_enabled, false) {:ok, pid} = LeaderElection.start_link([]) LeaderElection.leader?() {:ok, pid: pid} end test "elects itself as leader when clustering is disabled", %{pid: _pid} do assert LeaderElection.leader?() == true assert LeaderElection.current_leader() == node() end test "registers itself globally", %{pid: pid} do assert LeaderElection.current_leader() == node() assert :global.whereis_name(@election_key) == pid end end describe "leader?/0" do setup do Application.put_env(:aprsme, :cluster_enabled, false) {:ok, _pid} = LeaderElection.start_link([]) LeaderElection.leader?() :ok end test "returns leadership status" do assert LeaderElection.leader?() in [true, false] end test "cached check matches GenServer state" do # leader_cached?/0 reads from :persistent_term — no GenServer.call assert LeaderElection.leader_cached?() == LeaderElection.leader?() end test "cached check returns false before election" do # Stop current instance GenServer.stop(LeaderElection) :global.unregister_name({:aprs_is_leader, LeaderElection}) # Clear the persistent_term :persistent_term.put({LeaderElection, :is_leader}, false) assert LeaderElection.leader_cached?() == false end end describe "current_leader/0" do setup do Application.put_env(:aprsme, :cluster_enabled, false) {:ok, _pid} = LeaderElection.start_link([]) LeaderElection.leader?() :ok end test "returns the current leader node" do leader = LeaderElection.current_leader() case leader do nil -> :ok _ -> assert leader == node() end end end describe "get_cluster_aprs_status/0" do test "returns local status when clustering is disabled" do Application.put_env(:aprsme, :cluster_enabled, false) # Since Aprsme.Is.get_status() is called directly, we need to ensure # the module exists or mock it at a lower level # For now, just verify the function doesn't crash result = LeaderElection.get_cluster_aprs_status() case result do nil -> :ok %{} -> :ok end end test "returns cluster-wide status when clustering is enabled" do Application.put_env(:aprsme, :cluster_enabled, true) # This will call get_cluster_wide_status which handles the cluster logic status = LeaderElection.get_cluster_aprs_status() # The function should return a map with cluster_info when in cluster mode case status do nil -> :ok %{} -> :ok end # If we got a valid status back, it should have cluster info if is_map(status) do assert Map.has_key?(status, :cluster_info) if status.cluster_info do assert status.cluster_info.total_nodes >= 1 assert status.cluster_info.all_nodes != [] end end end end describe "leadership transitions" do setup do Application.put_env(:aprsme, :cluster_enabled, false) :ok end test "handles termination gracefully when leader" do {:ok, pid} = LeaderElection.start_link([]) # Sync so the 0-delay election fires before we assert leader status. assert LeaderElection.leader?() == true # Stop the process GenServer.stop(pid) # Verify global registration is cleaned up assert :global.whereis_name(@election_key) == :undefined end test "periodic leadership check continues running" do Application.put_env(:aprsme, :election_check_interval_ms, 50) on_exit(fn -> Application.delete_env(:aprsme, :election_check_interval_ms) end) {:ok, _pid} = LeaderElection.start_link([]) # Initial state - might not be leader yet _initial_leader = LeaderElection.leader?() # Wait for periodic check (50ms interval, plus buffer) Process.sleep(150) # Should still be able to query leadership current_leader = LeaderElection.leader?() assert current_leader in [true, false] # In non-clustered mode, should become leader if not Application.get_env(:aprsme, :cluster_enabled, false) do assert current_leader == true end end end describe "conflict resolution" do test "resolve_conflict prefers lexicographically lower node" do # We can't directly test the private function, but we can test the behavior # by starting multiple processes and seeing which wins # This is more of an integration test that would require multiple nodes # For now, we just verify the module handles conflicts without crashing {:ok, pid1} = LeaderElection.start_link([]) LeaderElection.leader?() # Try to register another process with the same key # This should fail or trigger conflict resolution spawn(fn -> :global.register_name(@election_key, self()) end) # Brief pause for the spawned process to attempt registration. Process.sleep(10) # Original process should still be alive assert Process.alive?(pid1) end end describe "stale registration cleanup" do test "cleans up registration when process dies" do # Register a dead process dead_pid = spawn(fn -> :ok end) # Ensure it's dead Process.sleep(10) # Force register it globally (simulating stale registration) :global.re_register_name(@election_key, dead_pid) # Start LeaderElection which should clean it up Application.put_env(:aprsme, :cluster_enabled, false) {:ok, _pid} = LeaderElection.start_link([]) # Wait for cleanup and election to occur Process.sleep(150) # Should have taken over leadership assert LeaderElection.leader?() == true end end describe "cluster mode behavior" do setup do Application.put_env(:aprsme, :cluster_enabled, true) :ok end test "waits for cluster formation when enabled" do Application.put_env(:aprsme, :election_max_cluster_wait_ms, 100) Application.put_env(:aprsme, :election_cluster_check_interval_ms, 50) on_exit(fn -> Application.delete_env(:aprsme, :election_max_cluster_wait_ms) Application.delete_env(:aprsme, :election_cluster_check_interval_ms) end) Phoenix.PubSub.subscribe(Aprsme.PubSub, "cluster:leadership") {:ok, _pid} = LeaderElection.start_link([]) # Wait for force_election_timeout (max_cluster_wait_ms: 100) to fire # and the process to become leader. The :force_election_timeout message # triggers election and publishes {:leadership_change, _, true}. assert_receive {:leadership_change, _, true}, 1000 assert LeaderElection.leader?() == true end end describe "leadership verification" do setup do Application.put_env(:aprsme, :cluster_enabled, false) {:ok, pid} = LeaderElection.start_link([]) assert LeaderElection.leader?() == true {:ok, pid: pid} end test "steps down when global registration is lost", %{pid: pid} do # Subscribe to leadership change notifications Phoenix.PubSub.subscribe(Aprsme.PubSub, "cluster:leadership") # Simulate losing the global registration (as happens during :global conflict resolution) :global.unregister_name(@election_key) # Force a leadership check, then use :sys.get_state to synchronously wait # for the message to be processed (before the 100ms re-election timer fires) send(pid, :check_leadership) state = :sys.get_state(pid) # Should have detected the loss and stepped down assert state.is_leader == false assert LeaderElection.leader_cached?() == false # Should have notified about leadership loss assert_received {:leadership_change, _, false} end test "steps down when another process holds the registration", %{pid: pid} do Phoenix.PubSub.subscribe(Aprsme.PubSub, "cluster:leadership") # Simulate another node winning the conflict resolution imposter = spawn(fn -> Process.sleep(:infinity) end) :global.re_register_name(@election_key, imposter) # Force a leadership check send(pid, :check_leadership) state = :sys.get_state(pid) # Should have stepped down assert state.is_leader == false assert LeaderElection.leader_cached?() == false assert_received {:leadership_change, _, false} # Clean up Process.exit(imposter, :kill) end test "re-elects after stepping down", %{pid: pid} do # Lose the registration :global.unregister_name(@election_key) # Force check — should step down and immediately queue :attempt_election. send(pid, :check_leadership) state = :sys.get_state(pid) assert state.is_leader == false # With election_initial_delay_ms: 0, :attempt_election is already in the # mailbox. leader?/0 (a GenServer.call) processes it first, then answers. assert LeaderElection.leader?() == true end end describe "message handling" do setup do Application.put_env(:aprsme, :cluster_enabled, false) {:ok, pid} = LeaderElection.start_link([]) {:ok, pid: pid} end test "handles unknown messages without crashing", %{pid: pid} do send(pid, :unknown_message) :sys.get_state(pid) assert Process.alive?(pid) assert LeaderElection.leader?() in [true, false] end test "handles check_leadership message", %{pid: pid} do send(pid, :check_leadership) :sys.get_state(pid) assert Process.alive?(pid) assert LeaderElection.leader?() in [true, false] end end describe "get_cluster_aprs_status/0 clustered mode" do setup do original = Application.get_env(:aprsme, :cluster_enabled) on_exit(fn -> Application.put_env(:aprsme, :cluster_enabled, original || false) end) :ok end test "clustered mode aggregates local status with cluster_info" do Application.put_env(:aprsme, :cluster_enabled, true) status = LeaderElection.get_cluster_aprs_status() assert %{connected: _, server: _, port: _} = status # In clustered mode, cluster_info is added. if Map.has_key?(status, :cluster_info) do assert %{total_nodes: _, connected_nodes: _, leader_node: _, all_nodes: _} = status.cluster_info assert Map.has_key?(status.cluster_info, :total_nodes) assert Map.has_key?(status.cluster_info, :connected_nodes) end end test "clustered get_cluster_aprs_status with registered leader pid hits get_leader_node_name pid branch" do Application.put_env(:aprsme, :cluster_enabled, true) # Pre-register a real pid under the @election_key so the get_leader_node_name # branch (line 382: pid when is_pid(pid) -> pid |> node() |> to_string()). :global.unregister_name(@election_key) :global.register_name(@election_key, self()) try do status = LeaderElection.get_cluster_aprs_status() assert %{connected: _, server: _, port: _} = status after :global.unregister_name(@election_key) end end end describe "handle_info(:check_cluster_and_elect) direct callback" do test "no-ops when election already forced" do state = %LeaderElection{ cluster_enabled: true, election_forced: true } assert {:noreply, ^state} = LeaderElection.handle_info(:check_cluster_and_elect, state) end test "reschedules itself when cluster has no peers yet" do state = %LeaderElection{ cluster_enabled: true, election_forced: false } assert {:noreply, new_state} = LeaderElection.handle_info(:check_cluster_and_elect, state) # Still not forced — we're still waiting. refute new_state.election_forced end end describe "handle_info(:force_election_timeout) direct callback" do test "no-ops when election already forced" do state = %LeaderElection{ cluster_enabled: true, election_forced: true, is_leader: false } assert {:noreply, ^state} = LeaderElection.handle_info(:force_election_timeout, state) end test "no-ops when already leader" do state = %LeaderElection{ cluster_enabled: true, election_forced: false, is_leader: true } assert {:noreply, ^state} = LeaderElection.handle_info(:force_election_timeout, state) end test "forces election when not yet forced and not leader" do state = %LeaderElection{ cluster_enabled: true, election_forced: false, is_leader: false } assert {:noreply, new_state} = LeaderElection.handle_info(:force_election_timeout, state) assert new_state.election_forced end end describe "handle_info(:attempt_election) direct callback" do setup do :global.unregister_name(@election_key) on_exit(fn -> :global.unregister_name(@election_key) end) :ok end test "becomes leader and persists leadership when no other registration exists" do state = %LeaderElection{ cluster_enabled: false, is_leader: false, leader_node: nil } assert {:noreply, new_state} = LeaderElection.handle_info(:attempt_election, state) assert new_state.is_leader == true assert new_state.leader_node == node() end test "re-confirms leadership when already registered as self" do # Pre-register ourselves as the leader. :global.register_name(@election_key, self()) state = %LeaderElection{ cluster_enabled: false, is_leader: false, leader_node: nil } assert {:noreply, new_state} = LeaderElection.handle_info(:attempt_election, state) assert new_state.is_leader == true end end describe "handle_call callbacks" do test "leader? returns state.is_leader" do state = %LeaderElection{is_leader: true} assert {:reply, true, ^state} = LeaderElection.handle_call(:leader?, self(), state) state2 = %LeaderElection{is_leader: false} assert {:reply, false, ^state2} = LeaderElection.handle_call(:leader?, self(), state2) end test "current_leader returns state.leader_node" do state = %LeaderElection{leader_node: :node_a} assert {:reply, :node_a, ^state} = LeaderElection.handle_call(:current_leader, self(), state) end end describe "get_cluster_aprs_status/0 detailed branches" do test "returns a map with cluster_info when clustering is enabled and no peers connected" do # Single-node cluster — Node.list() == []; get_cluster_wide_status # falls into the "no connected statuses" branch. Application.put_env(:aprsme, :cluster_enabled, true) status = LeaderElection.get_cluster_aprs_status() assert %{connected: _, server: _, port: _} = status if Map.has_key?(status, :cluster_info) do assert status.cluster_info.total_nodes >= 1 assert status.cluster_info.all_nodes != [] # leader_node is a string (including "none" when no leader). assert byte_size(status.cluster_info.leader_node) > 0 end end end describe "handle_info(:attempt_election) with another pid holding the global key" do test "becomes non-leader when another process holds the registration (:no path)" do # Pre-register a different pid (not the test process) so attempt_election # sees a foreign pid and returns :no via check_own_registration. other = spawn(fn -> Process.sleep(:infinity) end) :global.unregister_name(@election_key) :global.register_name(@election_key, other) try do state = %LeaderElection{ cluster_enabled: false, is_leader: false, leader_node: nil } assert {:noreply, new_state} = LeaderElection.handle_info(:attempt_election, state) # We got the :no path. leader_node should be the foreign pid's node. assert new_state.is_leader == false assert new_state.leader_node == node(other) after Process.exit(other, :kill) :global.unregister_name(@election_key) end end end describe "init/1 in clustered mode (schedule_initial_election(true) path)" do test "schedules :check_cluster_and_elect and :force_election_timeout" do # Capture the test process — init/1 runs in this process and sends # itself the cluster-check messages, so they land in our mailbox. original = Application.get_env(:aprsme, :cluster_enabled) Application.put_env(:aprsme, :cluster_enabled, true) try do assert {:ok, %{cluster_enabled: true}} = LeaderElection.init([]) # Both messages are scheduled with delays — won't arrive immediately # but the function should have run successfully. # The message :check_cluster_and_elect is scheduled 2s out — too long # to wait for in a test. We just verify init succeeded. after Application.put_env(:aprsme, :cluster_enabled, original) end end end describe "schedule_initial_election delay branch" do test "non-zero election_initial_delay_ms uses Process.send_after path" do original = Application.get_env(:aprsme, :election_initial_delay_ms, 100) Application.put_env(:aprsme, :election_initial_delay_ms, 50) try do # init/1 in non-clustered mode calls schedule_initial_election(false) # which goes to the `else: Process.send_after(...)` branch when delay > 0. original_cluster = Application.get_env(:aprsme, :cluster_enabled, false) Application.put_env(:aprsme, :cluster_enabled, false) try do assert {:ok, _state} = LeaderElection.init([]) # The :attempt_election message is scheduled 50ms out — won't arrive within this test. after Application.put_env(:aprsme, :cluster_enabled, original_cluster) end after Application.put_env(:aprsme, :election_initial_delay_ms, original) end end end describe "handle_info(:check_leadership) direct callback" do test "no-op for a non-leader — reschedules and keeps state" do state = %LeaderElection{is_leader: false, leader_node: nil} assert {:noreply, new_state} = LeaderElection.handle_info(:check_leadership, state) assert new_state.is_leader == false end test "leader with intact global registration stays leader" do # Pre-register ourselves globally so verify_leadership/1 sees pid == self(). :global.unregister_name(@election_key) :global.register_name(@election_key, self()) state = %LeaderElection{is_leader: true, leader_node: node()} assert {:noreply, new_state} = LeaderElection.handle_info(:check_leadership, state) assert new_state.is_leader == true :global.unregister_name(@election_key) end test "leader whose registration was lost steps down" do :global.unregister_name(@election_key) state = %LeaderElection{is_leader: true, leader_node: node()} assert {:noreply, new_state} = LeaderElection.handle_info(:check_leadership, state) assert new_state.is_leader == false assert new_state.leader_node == nil end end describe "handle_info(:force_election_timeout, state)" do test "single-node mode warning branch when no other nodes" do # not election_forced, not leader → goes into the if branch. # Node.list() returns [] in this single-node test. state = %LeaderElection{ cluster_enabled: true, election_forced: false, is_leader: false, leader_node: nil } assert {:noreply, new_state} = LeaderElection.handle_info(:force_election_timeout, state) assert new_state.election_forced == true end test "no-op branch when election already forced" do state = %LeaderElection{ cluster_enabled: true, election_forced: true, is_leader: false, leader_node: nil } assert {:noreply, new_state} = LeaderElection.handle_info(:force_election_timeout, state) assert new_state == state end end describe "handle_info(:check_cluster_and_elect, state)" do test "no-op branch when election_forced is true" do state = %LeaderElection{election_forced: true} assert {:noreply, ^state} = LeaderElection.handle_info(:check_cluster_and_elect, state) end test "reschedules itself when no nodes are connected (cluster not formed)" do state = %LeaderElection{election_forced: false} # Node.list() returns [] in single-node test mode → reschedules in 2s. assert {:noreply, new_state} = LeaderElection.handle_info(:check_cluster_and_elect, state) assert new_state.election_forced == false end end describe "handle_info/2 for unrecognized messages" do test "logs and replies :noreply with state unchanged" do state = %LeaderElection{is_leader: false} assert {:noreply, ^state} = LeaderElection.handle_info(:something_unknown, state) end end describe "handle_call(:current_leader, _from, state)" do test "returns the leader_node from state" do state = %LeaderElection{leader_node: :some@node} assert {:reply, :some@node, ^state} = LeaderElection.handle_call(:current_leader, self(), state) end end describe "terminate/2" do test "non-leader terminate returns :ok without unregistering global" do state = %LeaderElection{is_leader: false, leader_node: nil} assert :ok = LeaderElection.terminate(:shutdown, state) end test "leader terminate logs and unregisters global" do :global.unregister_name(@election_key) :global.register_name(@election_key, self()) state = %LeaderElection{is_leader: true, leader_node: node()} assert :ok = LeaderElection.terminate(:shutdown, state) # The global registration should now be cleared. assert :global.whereis_name(@election_key) == :undefined end end end