defmodule Aprsme.Cluster.LeaderElection do @moduledoc """ Manages leader election for APRS-IS connection using distributed Erlang. Only the elected leader will maintain the APRS-IS connection. """ use GenServer require Logger @election_key {:aprs_is_leader, __MODULE__} @default_check_interval 5_000 # Maximum time to wait for cluster formation before proceeding with election (30 seconds) @default_max_cluster_wait 30_000 @default_cluster_check_interval 2_000 defp check_interval, do: Application.get_env(:aprsme, :election_check_interval_ms, @default_check_interval) defp max_cluster_wait, do: Application.get_env(:aprsme, :election_max_cluster_wait_ms, @default_max_cluster_wait) defp cluster_check_interval, do: Application.get_env(:aprsme, :election_cluster_check_interval_ms, @default_cluster_check_interval) defstruct is_leader: false, leader_node: nil, cluster_enabled: false, election_forced: false @type t :: %__MODULE__{ is_leader: boolean(), leader_node: node() | nil, cluster_enabled: boolean(), election_forced: boolean() } def start_link(opts) do GenServer.start_link(__MODULE__, opts, name: __MODULE__) end def leader? do GenServer.call(__MODULE__, :leader?) end @doc """ Fast cached leadership check using :persistent_term. No GenServer.call overhead — suitable for hot paths like packet distribution. """ @spec leader_cached?() :: boolean() def leader_cached? do :persistent_term.get({__MODULE__, :is_leader}, false) end def current_leader do GenServer.call(__MODULE__, :current_leader) end @doc """ Gets APRS-IS status from across the entire cluster. Returns the status from whichever node has an active connection. """ def get_cluster_aprs_status do fetch_cluster_status(Application.get_env(:aprsme, :cluster_enabled, false)) end defp fetch_cluster_status(true), do: get_cluster_wide_status() defp fetch_cluster_status(false), do: Aprsme.Is.get_status() @impl true def init(_opts) do Logger.info("Starting leader election process") cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false) schedule_initial_election(cluster_enabled) # Schedule periodic checks Process.send_after(self(), :check_leadership, check_interval()) # Initialize cached leadership state :persistent_term.put({__MODULE__, :is_leader}, false) {:ok, %__MODULE__{cluster_enabled: cluster_enabled}} end # Clustered mode: wait for formation, then elect; schedule a backstop timeout. defp schedule_initial_election(true) do Logger.info("Clustering enabled - waiting for cluster formation before leader election") Process.send_after(self(), :check_cluster_and_elect, cluster_check_interval()) Process.send_after(self(), :force_election_timeout, max_cluster_wait()) end # Non-clustered mode: elect immediately. defp schedule_initial_election(false) do Logger.info("Clustering disabled - proceeding with immediate leader election") delay = Application.get_env(:aprsme, :election_initial_delay_ms, 100) if delay == 0, do: send(self(), :attempt_election), else: Process.send_after(self(), :attempt_election, delay) end @impl true def handle_info(:check_cluster_and_elect, state) do # Don't keep checking if election was already forced if state.election_forced do {:noreply, state} else connected_nodes = Node.list() if connected_nodes == [] do Logger.debug("Cluster not yet formed - waiting...") # Check again in 2 seconds Process.send_after(self(), :check_cluster_and_elect, 2_000) {:noreply, state} else Logger.info("Cluster formed with #{length(connected_nodes)} other nodes: #{inspect(connected_nodes)}") Logger.info("Proceeding with leader election") schedule_election_attempt() {:noreply, %{state | election_forced: true}} end end end @impl true def handle_info(:force_election_timeout, state) do # Only force election if we haven't already started one if not state.election_forced and not state.is_leader do connected_nodes = Node.list() if connected_nodes == [] do Logger.warning( "Cluster formation timeout reached after #{max_cluster_wait()}ms with no connected nodes. " <> "Proceeding with leader election in single-node mode to ensure APRS-IS connection." ) else Logger.info( "Forcing leader election after #{max_cluster_wait()}ms wait with #{length(connected_nodes)} connected nodes" ) end schedule_election_attempt() {:noreply, %{state | election_forced: true}} else {:noreply, state} end end @impl true def handle_info(:attempt_election, state) do cleanup_stale_registrations() case :global.whereis_name(@election_key) do pid when pid == self() -> _ = if !state.is_leader do Logger.info("Re-confirming leadership on node #{node()}") :persistent_term.put({__MODULE__, :is_leader}, true) :ok = notify_leadership_change(true) end {:noreply, %{state | is_leader: true, leader_node: node()}} _ -> attempt_registration_atomic(state) end end @impl true def handle_info(:check_leadership, state) do state = verify_leadership(state) # Re-attempt election if we're not leader _ = if not state.is_leader do schedule_election_attempt() end # Schedule next check _ = Process.send_after(self(), :check_leadership, check_interval()) {:noreply, state} end @impl true def handle_info(msg, state) do Logger.debug("LeaderElection received unexpected message: #{inspect(msg)}") {:noreply, state} end @impl true def handle_call(:leader?, _from, state) do {:reply, state.is_leader, state} end @impl true def handle_call(:current_leader, _from, state) do {:reply, state.leader_node, state} end @impl true def terminate(reason, state) do _ = if state.is_leader do Logger.info("Leader stepping down due to: #{inspect(reason)}") :persistent_term.put({__MODULE__, :is_leader}, false) _ = :global.unregister_name(@election_key) notify_leadership_change(false) end :ok end # Verify that a node which thinks it's leader still holds the :global registration. # After :global conflict resolution (e.g. two partitions merging), the losing PID # is silently unregistered — this detects that and steps down. defp schedule_election_attempt do delay = Application.get_env(:aprsme, :election_initial_delay_ms, 100) if delay == 0, do: send(self(), :attempt_election), else: Process.send_after(self(), :attempt_election, delay) end defp verify_leadership(%{is_leader: true} = state) do case :global.whereis_name(@election_key) do pid when pid == self() -> state _ -> Logger.warning("Lost global leadership registration, stepping down") :persistent_term.put({__MODULE__, :is_leader}, false) _ = notify_leadership_change(false) %{state | is_leader: false, leader_node: nil} end end defp verify_leadership(state), do: state defp attempt_registration_atomic(state) do result = :global.trans({@election_key, node()}, fn -> case :global.whereis_name(@election_key) do :undefined -> :global.register_name(@election_key, self(), &resolve_conflict/3) pid -> check_own_registration(pid) end end) case result do :yes -> Logger.info("Elected as APRS-IS connection leader on node #{node()}") :persistent_term.put({__MODULE__, :is_leader}, true) _ = notify_leadership_change(true) {:noreply, %{state | is_leader: true, leader_node: node()}} :no -> leader_pid = :global.whereis_name(@election_key) leader_node = if leader_pid != :undefined and is_pid(leader_pid), do: node(leader_pid) :persistent_term.put({__MODULE__, :is_leader}, false) {:noreply, %{state | is_leader: false, leader_node: leader_node}} end end defp check_own_registration(pid) do if pid == self(), do: :yes, else: :no end # Conflict resolution - prefer the process on the lexicographically lower node defp resolve_conflict(_name, pid1, pid2) do node1 = node(pid1) node2 = node(pid2) Logger.info("Resolving leader conflict between #{node1} and #{node2}") # Choose based on node name ordering for deterministic results if node1 <= node2 do pid1 else pid2 end end defp cleanup_stale_registrations do case :global.whereis_name(@election_key) do :undefined -> :ok pid when is_pid(pid) -> check_and_cleanup_registration(pid) end end defp check_and_cleanup_registration(pid) do pid_node = node(pid) connected_nodes = [node() | Node.list()] if pid_node in connected_nodes do check_pid_liveness(pid, pid_node) else cleanup_registration("disconnected node #{pid_node}") end end defp check_pid_liveness(pid, pid_node) do if pid_alive?(pid, pid_node) do :ok else reason = if pid_node == node(), do: "dead local process", else: "dead remote process" cleanup_registration("#{reason} #{inspect(pid)}") end rescue _error -> cleanup_registration("problematic process #{inspect(pid)}") end defp pid_alive?(pid, pid_node) when pid_node == node() do Process.alive?(pid) end defp pid_alive?(pid, pid_node) do case :rpc.call(pid_node, Process, :alive?, [pid], 5000) do {:badrpc, _} -> false result -> result == true end end defp cleanup_registration(reason) do Logger.info("Cleaning up stale leader registration for #{reason}") :global.unregister_name(@election_key) end defp get_cluster_wide_status do all_nodes = [node() | Node.list()] # Check each node for APRS-IS connection status connected_statuses = all_nodes |> Enum.map(&get_node_status/1) |> Enum.filter(fn status -> status.connected end) case connected_statuses do [status | _] -> # At least one node is connected - return its status # Add cluster info to indicate this is cluster-wide status Map.put(status, :cluster_info, %{ total_nodes: length(all_nodes), connected_nodes: length(connected_statuses), leader_node: get_leader_node_name(), all_nodes: all_nodes |> Enum.map(&to_string/1) |> Enum.sort() }) [] -> # No nodes are connected - return local status but mark as cluster-wide local_status = Aprsme.Is.get_status() Map.put(local_status, :cluster_info, %{ total_nodes: length(all_nodes), connected_nodes: 0, leader_node: get_leader_node_name(), all_nodes: all_nodes |> Enum.map(&to_string/1) |> Enum.sort() }) end end defp get_node_status(node_name) do if node_name == node() do # Local node - call directly Aprsme.Is.get_status() else # Remote node - use RPC case :rpc.call(node_name, Aprsme.Is, :get_status, [], 5000) do {:badrpc, _reason} -> # Node unreachable - return disconnected status %{connected: false, server: "unreachable", port: 0} status when is_map(status) -> status _ -> %{connected: false, server: "error", port: 0} end end rescue _error -> %{connected: false, server: "error", port: 0} end defp get_leader_node_name do case :global.whereis_name(@election_key) do :undefined -> "none" pid when is_pid(pid) -> pid |> node() |> to_string() end end defp notify_leadership_change(became_leader) do Phoenix.PubSub.broadcast( Aprsme.PubSub, "cluster:leadership", {:leadership_change, node(), became_leader} ) end end