Graham McIntire
b8a9b8a465
Turned on :error_handling, :underspecs, and :unmatched_returns in mix.exs dialyzer config. The 97 warnings this surfaced were fixed in place rather than suppressed: - unmatched_return (79): explicit discard with `_ = ...` for fire-and-forget side effects (Process.cancel_timer, :ets.new, send/2), and pattern-matched `:ok = ...` for control-plane Phoenix.PubSub subscribe/unsubscribe/broadcast calls so a future return-shape change fails loud. - contract_supertype (18): tightened @spec arg and return types on data_builder, historical_loader, url_params, packet_utils, encoding_utils, aprs_symbol, weather_controller, packet_replay to match each function's actual success typing. No behavioural change. mix compile clean, 1008 tests pass, dialyzer count is now 0.
372 lines
11 KiB
Elixir
372 lines
11 KiB
Elixir
defmodule Aprsme.Cluster.LeaderElection do
|
|
@moduledoc """
|
|
Manages leader election for APRS-IS connection using distributed Erlang.
|
|
Only the elected leader will maintain the APRS-IS connection.
|
|
"""
|
|
use GenServer
|
|
|
|
require Logger
|
|
|
|
@election_key {:aprs_is_leader, __MODULE__}
|
|
@check_interval 5_000
|
|
# Maximum time to wait for cluster formation before proceeding with election (30 seconds)
|
|
@max_cluster_wait 30_000
|
|
|
|
def start_link(opts) do
|
|
GenServer.start_link(__MODULE__, opts, name: __MODULE__)
|
|
end
|
|
|
|
def leader? do
|
|
GenServer.call(__MODULE__, :leader?)
|
|
end
|
|
|
|
@doc """
|
|
Fast cached leadership check using :persistent_term.
|
|
No GenServer.call overhead — suitable for hot paths like packet distribution.
|
|
"""
|
|
@spec leader_cached?() :: boolean()
|
|
def leader_cached? do
|
|
:persistent_term.get({__MODULE__, :is_leader}, false)
|
|
end
|
|
|
|
def current_leader do
|
|
GenServer.call(__MODULE__, :current_leader)
|
|
end
|
|
|
|
@doc """
|
|
Gets APRS-IS status from across the entire cluster.
|
|
Returns the status from whichever node has an active connection.
|
|
"""
|
|
def get_cluster_aprs_status do
|
|
cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false)
|
|
|
|
if cluster_enabled do
|
|
get_cluster_wide_status()
|
|
else
|
|
# Non-clustered mode - just return local status
|
|
Aprsme.Is.get_status()
|
|
end
|
|
end
|
|
|
|
@impl true
|
|
def init(_opts) do
|
|
Logger.info("Starting leader election process")
|
|
|
|
cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false)
|
|
|
|
if cluster_enabled do
|
|
Logger.info("Clustering enabled - waiting for cluster formation before leader election")
|
|
# Wait for cluster to form, then check periodically
|
|
Process.send_after(self(), :check_cluster_and_elect, 2_000)
|
|
# Set a timeout to force election if cluster doesn't form
|
|
Process.send_after(self(), :force_election_timeout, @max_cluster_wait)
|
|
else
|
|
Logger.info("Clustering disabled - proceeding with immediate leader election")
|
|
# Non-clustered mode - elect immediately
|
|
Process.send_after(self(), :attempt_election, 100)
|
|
end
|
|
|
|
# Schedule periodic checks
|
|
Process.send_after(self(), :check_leadership, @check_interval)
|
|
|
|
# Initialize cached leadership state
|
|
:persistent_term.put({__MODULE__, :is_leader}, false)
|
|
|
|
{:ok, %{is_leader: false, leader_node: nil, cluster_enabled: cluster_enabled, election_forced: false}}
|
|
end
|
|
|
|
@impl true
|
|
def handle_info(:check_cluster_and_elect, state) do
|
|
# Don't keep checking if election was already forced
|
|
if state.election_forced do
|
|
{:noreply, state}
|
|
else
|
|
connected_nodes = Node.list()
|
|
|
|
if connected_nodes == [] do
|
|
Logger.debug("Cluster not yet formed - waiting...")
|
|
# Check again in 2 seconds
|
|
Process.send_after(self(), :check_cluster_and_elect, 2_000)
|
|
{:noreply, state}
|
|
else
|
|
Logger.info("Cluster formed with #{length(connected_nodes)} other nodes: #{inspect(connected_nodes)}")
|
|
Logger.info("Proceeding with leader election")
|
|
Process.send_after(self(), :attempt_election, 100)
|
|
{:noreply, %{state | election_forced: true}}
|
|
end
|
|
end
|
|
end
|
|
|
|
@impl true
|
|
def handle_info(:force_election_timeout, state) do
|
|
# Only force election if we haven't already started one
|
|
if not state.election_forced and not state.is_leader do
|
|
connected_nodes = Node.list()
|
|
|
|
if connected_nodes == [] do
|
|
Logger.warning(
|
|
"Cluster formation timeout reached after #{@max_cluster_wait}ms with no connected nodes. " <>
|
|
"Proceeding with leader election in single-node mode to ensure APRS-IS connection."
|
|
)
|
|
else
|
|
Logger.info(
|
|
"Forcing leader election after #{@max_cluster_wait}ms wait with #{length(connected_nodes)} connected nodes"
|
|
)
|
|
end
|
|
|
|
Process.send_after(self(), :attempt_election, 100)
|
|
{:noreply, %{state | election_forced: true}}
|
|
else
|
|
{:noreply, state}
|
|
end
|
|
end
|
|
|
|
@impl true
|
|
def handle_info(:attempt_election, state) do
|
|
cleanup_stale_registrations()
|
|
|
|
case :global.whereis_name(@election_key) do
|
|
pid when pid == self() ->
|
|
_ =
|
|
if !state.is_leader do
|
|
Logger.info("Re-confirming leadership on node #{node()}")
|
|
:persistent_term.put({__MODULE__, :is_leader}, true)
|
|
:ok = notify_leadership_change(true)
|
|
end
|
|
|
|
{:noreply, %{state | is_leader: true, leader_node: node()}}
|
|
|
|
_ ->
|
|
attempt_registration_atomic(state)
|
|
end
|
|
end
|
|
|
|
@impl true
|
|
def handle_info(:check_leadership, state) do
|
|
state = verify_leadership(state)
|
|
|
|
# Re-attempt election if we're not leader
|
|
_ =
|
|
if not state.is_leader do
|
|
Process.send_after(self(), :attempt_election, 100)
|
|
end
|
|
|
|
# Schedule next check
|
|
_ = Process.send_after(self(), :check_leadership, @check_interval)
|
|
|
|
{:noreply, state}
|
|
end
|
|
|
|
@impl true
|
|
def handle_info(msg, state) do
|
|
Logger.debug("LeaderElection received unexpected message: #{inspect(msg)}")
|
|
{:noreply, state}
|
|
end
|
|
|
|
@impl true
|
|
def handle_call(:leader?, _from, state) do
|
|
{:reply, state.is_leader, state}
|
|
end
|
|
|
|
@impl true
|
|
def handle_call(:current_leader, _from, state) do
|
|
{:reply, state.leader_node, state}
|
|
end
|
|
|
|
@impl true
|
|
def terminate(reason, state) do
|
|
_ =
|
|
if state.is_leader do
|
|
Logger.info("Leader stepping down due to: #{inspect(reason)}")
|
|
:persistent_term.put({__MODULE__, :is_leader}, false)
|
|
_ = :global.unregister_name(@election_key)
|
|
notify_leadership_change(false)
|
|
end
|
|
|
|
:ok
|
|
end
|
|
|
|
# Verify that a node which thinks it's leader still holds the :global registration.
|
|
# After :global conflict resolution (e.g. two partitions merging), the losing PID
|
|
# is silently unregistered — this detects that and steps down.
|
|
defp verify_leadership(%{is_leader: true} = state) do
|
|
case :global.whereis_name(@election_key) do
|
|
pid when pid == self() ->
|
|
state
|
|
|
|
_ ->
|
|
Logger.warning("Lost global leadership registration, stepping down")
|
|
:persistent_term.put({__MODULE__, :is_leader}, false)
|
|
_ = notify_leadership_change(false)
|
|
%{state | is_leader: false, leader_node: nil}
|
|
end
|
|
end
|
|
|
|
defp verify_leadership(state), do: state
|
|
|
|
defp attempt_registration_atomic(state) do
|
|
result =
|
|
:global.trans({@election_key, node()}, fn ->
|
|
case :global.whereis_name(@election_key) do
|
|
:undefined ->
|
|
:global.register_name(@election_key, self(), &resolve_conflict/3)
|
|
|
|
pid ->
|
|
check_own_registration(pid)
|
|
end
|
|
end)
|
|
|
|
case result do
|
|
:yes ->
|
|
Logger.info("Elected as APRS-IS connection leader on node #{node()}")
|
|
:persistent_term.put({__MODULE__, :is_leader}, true)
|
|
_ = notify_leadership_change(true)
|
|
{:noreply, %{state | is_leader: true, leader_node: node()}}
|
|
|
|
:no ->
|
|
leader_pid = :global.whereis_name(@election_key)
|
|
leader_node = if leader_pid != :undefined and is_pid(leader_pid), do: node(leader_pid)
|
|
:persistent_term.put({__MODULE__, :is_leader}, false)
|
|
{:noreply, %{state | is_leader: false, leader_node: leader_node}}
|
|
end
|
|
end
|
|
|
|
defp check_own_registration(pid) do
|
|
if pid == self(), do: :yes, else: :no
|
|
end
|
|
|
|
# Conflict resolution - prefer the process on the lexicographically lower node
|
|
defp resolve_conflict(_name, pid1, pid2) do
|
|
node1 = node(pid1)
|
|
node2 = node(pid2)
|
|
|
|
Logger.info("Resolving leader conflict between #{node1} and #{node2}")
|
|
|
|
# Choose based on node name ordering for deterministic results
|
|
if node1 <= node2 do
|
|
pid1
|
|
else
|
|
pid2
|
|
end
|
|
end
|
|
|
|
defp cleanup_stale_registrations do
|
|
case :global.whereis_name(@election_key) do
|
|
:undefined ->
|
|
:ok
|
|
|
|
pid when is_pid(pid) ->
|
|
check_and_cleanup_registration(pid)
|
|
end
|
|
end
|
|
|
|
defp check_and_cleanup_registration(pid) do
|
|
pid_node = node(pid)
|
|
connected_nodes = [node() | Node.list()]
|
|
|
|
if pid_node in connected_nodes do
|
|
check_pid_liveness(pid, pid_node)
|
|
else
|
|
cleanup_registration("disconnected node #{pid_node}")
|
|
end
|
|
end
|
|
|
|
defp check_pid_liveness(pid, pid_node) do
|
|
if pid_alive?(pid, pid_node) do
|
|
:ok
|
|
else
|
|
reason = if pid_node == node(), do: "dead local process", else: "dead remote process"
|
|
cleanup_registration("#{reason} #{inspect(pid)}")
|
|
end
|
|
rescue
|
|
_error ->
|
|
cleanup_registration("problematic process #{inspect(pid)}")
|
|
end
|
|
|
|
defp pid_alive?(pid, pid_node) when pid_node == node() do
|
|
Process.alive?(pid)
|
|
end
|
|
|
|
defp pid_alive?(pid, pid_node) do
|
|
case :rpc.call(pid_node, Process, :alive?, [pid]) do
|
|
{:badrpc, _} -> false
|
|
result -> result == true
|
|
end
|
|
end
|
|
|
|
defp cleanup_registration(reason) do
|
|
Logger.info("Cleaning up stale leader registration for #{reason}")
|
|
:global.unregister_name(@election_key)
|
|
end
|
|
|
|
defp get_cluster_wide_status do
|
|
all_nodes = [node() | Node.list()]
|
|
|
|
# Check each node for APRS-IS connection status
|
|
connected_statuses =
|
|
all_nodes
|
|
|> Enum.map(&get_node_status/1)
|
|
|> Enum.filter(fn status -> status.connected end)
|
|
|
|
case connected_statuses do
|
|
[status | _] ->
|
|
# At least one node is connected - return its status
|
|
# Add cluster info to indicate this is cluster-wide status
|
|
Map.put(status, :cluster_info, %{
|
|
total_nodes: length(all_nodes),
|
|
connected_nodes: length(connected_statuses),
|
|
leader_node: get_leader_node_name(),
|
|
all_nodes: all_nodes |> Enum.map(&to_string/1) |> Enum.sort()
|
|
})
|
|
|
|
[] ->
|
|
# No nodes are connected - return local status but mark as cluster-wide
|
|
local_status = Aprsme.Is.get_status()
|
|
|
|
Map.put(local_status, :cluster_info, %{
|
|
total_nodes: length(all_nodes),
|
|
connected_nodes: 0,
|
|
leader_node: get_leader_node_name(),
|
|
all_nodes: all_nodes |> Enum.map(&to_string/1) |> Enum.sort()
|
|
})
|
|
end
|
|
end
|
|
|
|
defp get_node_status(node_name) do
|
|
if node_name == node() do
|
|
# Local node - call directly
|
|
Aprsme.Is.get_status()
|
|
else
|
|
# Remote node - use RPC
|
|
case :rpc.call(node_name, Aprsme.Is, :get_status, [], 5000) do
|
|
{:badrpc, _reason} ->
|
|
# Node unreachable - return disconnected status
|
|
%{connected: false, server: "unreachable", port: 0}
|
|
|
|
status when is_map(status) ->
|
|
status
|
|
|
|
_ ->
|
|
%{connected: false, server: "error", port: 0}
|
|
end
|
|
end
|
|
rescue
|
|
_error ->
|
|
%{connected: false, server: "error", port: 0}
|
|
end
|
|
|
|
defp get_leader_node_name do
|
|
case :global.whereis_name(@election_key) do
|
|
:undefined -> "none"
|
|
pid when is_pid(pid) -> pid |> node() |> to_string()
|
|
end
|
|
end
|
|
|
|
defp notify_leadership_change(became_leader) do
|
|
Phoenix.PubSub.broadcast(
|
|
Aprsme.PubSub,
|
|
"cluster:leadership",
|
|
{:leadership_change, node(), became_leader}
|
|
)
|
|
end
|
|
end
|