defmodule Aprsme.Cluster.LeaderElection do @moduledoc """ Manages leader election for APRS-IS connection using distributed Erlang. Only the elected leader will maintain the APRS-IS connection. """ use GenServer require Logger @election_key {:aprs_is_leader, __MODULE__} @check_interval 5_000 # Maximum time to wait for cluster formation before proceeding with election (30 seconds) @max_cluster_wait 30_000 def start_link(opts) do GenServer.start_link(__MODULE__, opts, name: __MODULE__) end def leader? do GenServer.call(__MODULE__, :leader?) end def current_leader do GenServer.call(__MODULE__, :current_leader) end @doc """ Gets APRS-IS status from across the entire cluster. Returns the status from whichever node has an active connection. """ def get_cluster_aprs_status do cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false) if cluster_enabled do get_cluster_wide_status() else # Non-clustered mode - just return local status Aprsme.Is.get_status() end end @impl true def init(_opts) do Logger.info("Starting leader election process") cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false) if cluster_enabled do Logger.info("Clustering enabled - waiting for cluster formation before leader election") # Wait for cluster to form, then check periodically Process.send_after(self(), :check_cluster_and_elect, 2_000) # Set a timeout to force election if cluster doesn't form Process.send_after(self(), :force_election_timeout, @max_cluster_wait) else Logger.info("Clustering disabled - proceeding with immediate leader election") # Non-clustered mode - elect immediately Process.send_after(self(), :attempt_election, 100) end # Schedule periodic checks Process.send_after(self(), :check_leadership, @check_interval) {:ok, %{is_leader: false, leader_node: nil, cluster_enabled: cluster_enabled, election_forced: false}} end @impl true def handle_info(:check_cluster_and_elect, state) do # Don't keep checking if election was already forced if state.election_forced do {:noreply, state} else connected_nodes = Node.list() if connected_nodes == [] do Logger.debug("Cluster not yet formed - waiting...") # Check again in 2 seconds Process.send_after(self(), :check_cluster_and_elect, 2_000) {:noreply, state} else Logger.info("Cluster formed with #{length(connected_nodes)} other nodes: #{inspect(connected_nodes)}") Logger.info("Proceeding with leader election") Process.send_after(self(), :attempt_election, 100) {:noreply, %{state | election_forced: true}} end end end @impl true def handle_info(:force_election_timeout, state) do # Only force election if we haven't already started one if not state.election_forced and not state.is_leader do connected_nodes = Node.list() if connected_nodes == [] do Logger.warning( "Cluster formation timeout reached after #{@max_cluster_wait}ms with no connected nodes. " <> "Proceeding with leader election in single-node mode to ensure APRS-IS connection." ) else Logger.info( "Forcing leader election after #{@max_cluster_wait}ms wait with #{length(connected_nodes)} connected nodes" ) end Process.send_after(self(), :attempt_election, 100) {:noreply, %{state | election_forced: true}} else {:noreply, state} end end @impl true def handle_info(:attempt_election, state) do # First, try to clean up any stale registrations cleanup_stale_registrations() case :global.register_name(@election_key, self(), &resolve_conflict/3) do :yes -> Logger.info("Elected as APRS-IS connection leader on node #{node()}") notify_leadership_change(true) {:noreply, %{state | is_leader: true, leader_node: node()}} :no -> leader_pid = :global.whereis_name(@election_key) leader_node = if leader_pid != :undefined and is_pid(leader_pid), do: node(leader_pid) Logger.info("Not elected as leader. Current leader is on node #{inspect(leader_node)}") {:noreply, %{state | is_leader: false, leader_node: leader_node}} end end @impl true def handle_info(:check_leadership, state) do # Re-attempt election if we're not leader if not state.is_leader do Process.send_after(self(), :attempt_election, 100) end # Schedule next check Process.send_after(self(), :check_leadership, @check_interval) {:noreply, state} end @impl true def handle_info(msg, state) do Logger.debug("LeaderElection received unexpected message: #{inspect(msg)}") {:noreply, state} end @impl true def handle_call(:leader?, _from, state) do {:reply, state.is_leader, state} end @impl true def handle_call(:current_leader, _from, state) do {:reply, state.leader_node, state} end @impl true def terminate(reason, state) do if state.is_leader do Logger.info("Leader stepping down due to: #{inspect(reason)}") :global.unregister_name(@election_key) notify_leadership_change(false) end :ok end # Conflict resolution - prefer the process on the lexicographically lower node defp resolve_conflict(_name, pid1, pid2) do node1 = node(pid1) node2 = node(pid2) Logger.info("Resolving leader conflict between #{node1} and #{node2}") # Choose based on node name ordering for deterministic results if node1 <= node2 do pid1 else pid2 end end defp cleanup_stale_registrations do case :global.whereis_name(@election_key) do :undefined -> :ok pid when is_pid(pid) -> check_and_cleanup_registration(pid) end end defp check_and_cleanup_registration(pid) do pid_node = node(pid) connected_nodes = [node() | Node.list()] if pid_node in connected_nodes do check_pid_liveness(pid, pid_node) else cleanup_registration("disconnected node #{pid_node}") end end defp check_pid_liveness(pid, pid_node) do if is_pid_alive?(pid, pid_node) do :ok else reason = if pid_node == node(), do: "dead local process", else: "dead remote process" cleanup_registration("#{reason} #{inspect(pid)}") end rescue _error -> cleanup_registration("problematic process #{inspect(pid)}") end defp is_pid_alive?(pid, pid_node) when pid_node == node() do Process.alive?(pid) end defp is_pid_alive?(pid, pid_node) do if :rpc.call(pid_node, Process, :alive?, [pid]) do true else false end end defp cleanup_registration(reason) do Logger.info("Cleaning up stale leader registration for #{reason}") :global.unregister_name(@election_key) end defp get_cluster_wide_status do all_nodes = [node() | Node.list()] # Check each node for APRS-IS connection status connected_statuses = all_nodes |> Enum.map(&get_node_status/1) |> Enum.filter(fn status -> status.connected end) case connected_statuses do [status | _] -> # At least one node is connected - return its status # Add cluster info to indicate this is cluster-wide status Map.put(status, :cluster_info, %{ total_nodes: length(all_nodes), connected_nodes: length(connected_statuses), leader_node: get_leader_node_name(), all_nodes: all_nodes |> Enum.map(&to_string/1) |> Enum.sort() }) [] -> # No nodes are connected - return local status but mark as cluster-wide local_status = Aprsme.Is.get_status() Map.put(local_status, :cluster_info, %{ total_nodes: length(all_nodes), connected_nodes: 0, leader_node: get_leader_node_name(), all_nodes: all_nodes |> Enum.map(&to_string/1) |> Enum.sort() }) end end defp get_node_status(node_name) do if node_name == node() do # Local node - call directly Aprsme.Is.get_status() else # Remote node - use RPC case :rpc.call(node_name, Aprsme.Is, :get_status, [], 5000) do {:badrpc, _reason} -> # Node unreachable - return disconnected status %{connected: false, server: "unreachable", port: 0} status when is_map(status) -> status _ -> %{connected: false, server: "error", port: 0} end end rescue _error -> %{connected: false, server: "error", port: 0} end defp get_leader_node_name do case :global.whereis_name(@election_key) do :undefined -> "none" pid when is_pid(pid) -> pid |> node() |> to_string() end end defp notify_leadership_change(became_leader) do Phoenix.PubSub.broadcast( Aprsme.PubSub, "cluster:leadership", {:leadership_change, node(), became_leader} ) end end