aprs.me/lib/aprsme/cluster/leader_election.ex

defmodule Aprsme.Cluster.LeaderElection do
  @moduledoc """
  Manages leader election for APRS-IS connection using distributed Erlang.
  Only the elected leader will maintain the APRS-IS connection.
  """
  use GenServer

  require Logger

  @election_key {:aprs_is_leader, __MODULE__}
  @default_check_interval 5_000
  # Maximum time to wait for cluster formation before proceeding with election (30 seconds)
  @default_max_cluster_wait 30_000
  @default_cluster_check_interval 2_000

  defp check_interval, do: Application.get_env(:aprsme, :election_check_interval_ms, @default_check_interval)
  defp max_cluster_wait, do: Application.get_env(:aprsme, :election_max_cluster_wait_ms, @default_max_cluster_wait)

  defp cluster_check_interval,
    do: Application.get_env(:aprsme, :election_cluster_check_interval_ms, @default_cluster_check_interval)

  defstruct is_leader: false,
            leader_node: nil,
            cluster_enabled: false,
            election_forced: false

  @type t :: %__MODULE__{
          is_leader: boolean(),
          leader_node: node() | nil,
          cluster_enabled: boolean(),
          election_forced: boolean()
        }

  def start_link(opts) do
    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
  end

  def leader? do
    GenServer.call(__MODULE__, :leader?)
  end

  @doc """
  Fast cached leadership check using :persistent_term.
  No GenServer.call overhead — suitable for hot paths like packet distribution.
  """
  @spec leader_cached?() :: boolean()
  def leader_cached? do
    :persistent_term.get({__MODULE__, :is_leader}, false)
  end

  def current_leader do
    GenServer.call(__MODULE__, :current_leader)
  end

  @doc """
  Gets APRS-IS status from across the entire cluster.
  Returns the status from whichever node has an active connection.
  """
  def get_cluster_aprs_status do
    fetch_cluster_status(Application.get_env(:aprsme, :cluster_enabled, false))
  end

  defp fetch_cluster_status(true), do: get_cluster_wide_status()
  defp fetch_cluster_status(false), do: Aprsme.Is.get_status()

  @impl true
  def init(_opts) do
    Logger.info("Starting leader election process")
    cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false)
    schedule_initial_election(cluster_enabled)

    # Schedule periodic checks
    Process.send_after(self(), :check_leadership, check_interval())

    # Initialize cached leadership state
    :persistent_term.put({__MODULE__, :is_leader}, false)

    {:ok, %__MODULE__{cluster_enabled: cluster_enabled}}
  end

  # Clustered mode: wait for formation, then elect; schedule a backstop timeout.
  defp schedule_initial_election(true) do
    Logger.info("Clustering enabled - waiting for cluster formation before leader election")
    Process.send_after(self(), :check_cluster_and_elect, cluster_check_interval())
    Process.send_after(self(), :force_election_timeout, max_cluster_wait())
  end

  # Non-clustered mode: elect immediately.
  defp schedule_initial_election(false) do
    Logger.info("Clustering disabled - proceeding with immediate leader election")
    delay = Application.get_env(:aprsme, :election_initial_delay_ms, 100)

    if delay == 0,
      do: send(self(), :attempt_election),
      else: Process.send_after(self(), :attempt_election, delay)
  end

  @impl true
  def handle_info(:check_cluster_and_elect, state) do
    # Don't keep checking if election was already forced
    if state.election_forced do
      {:noreply, state}
    else
      connected_nodes = Node.list()

      if connected_nodes == [] do
        Logger.debug("Cluster not yet formed - waiting...")
        # Check again in 2 seconds
        Process.send_after(self(), :check_cluster_and_elect, 2_000)
        {:noreply, state}
      else
        Logger.info("Cluster formed with #{length(connected_nodes)} other nodes: #{inspect(connected_nodes)}")
        Logger.info("Proceeding with leader election")
        schedule_election_attempt()
        {:noreply, %{state | election_forced: true}}
      end
    end
  end

  @impl true
  def handle_info(:force_election_timeout, state) do
    # Only force election if we haven't already started one
    if not state.election_forced and not state.is_leader do
      connected_nodes = Node.list()

      if connected_nodes == [] do
        Logger.warning(
          "Cluster formation timeout reached after #{max_cluster_wait()}ms with no connected nodes. " <>
            "Proceeding with leader election in single-node mode to ensure APRS-IS connection."
        )
      else
        Logger.info(
          "Forcing leader election after #{max_cluster_wait()}ms wait with #{length(connected_nodes)} connected nodes"
        )
      end

      schedule_election_attempt()
      {:noreply, %{state | election_forced: true}}
    else
      {:noreply, state}
    end
  end

  @impl true
  def handle_info(:attempt_election, state) do
    cleanup_stale_registrations()

    case :global.whereis_name(@election_key) do
      pid when pid == self() ->
        _ =
          if !state.is_leader do
            Logger.info("Re-confirming leadership on node #{node()}")
            :persistent_term.put({__MODULE__, :is_leader}, true)
            :ok = notify_leadership_change(true)
          end

        {:noreply, %{state | is_leader: true, leader_node: node()}}

      _ ->
        attempt_registration_atomic(state)
    end
  end

  @impl true
  def handle_info(:check_leadership, state) do
    state = verify_leadership(state)

    # Re-attempt election if we're not leader
    _ =
      if not state.is_leader do
        schedule_election_attempt()
      end

    # Schedule next check
    _ = Process.send_after(self(), :check_leadership, check_interval())

    {:noreply, state}
  end

  @impl true
  def handle_info(msg, state) do
    Logger.debug("LeaderElection received unexpected message: #{inspect(msg)}")
    {:noreply, state}
  end

  @impl true
  def handle_call(:leader?, _from, state) do
    {:reply, state.is_leader, state}
  end

  @impl true
  def handle_call(:current_leader, _from, state) do
    {:reply, state.leader_node, state}
  end

  @impl true
  def terminate(reason, state) do
    _ =
      if state.is_leader do
        Logger.info("Leader stepping down due to: #{inspect(reason)}")
        :persistent_term.put({__MODULE__, :is_leader}, false)
        _ = :global.unregister_name(@election_key)
        notify_leadership_change(false)
      end

    :ok
  end

  # Verify that a node which thinks it's leader still holds the :global registration.
  # After :global conflict resolution (e.g. two partitions merging), the losing PID
  # is silently unregistered — this detects that and steps down.
  defp schedule_election_attempt do
    delay = Application.get_env(:aprsme, :election_initial_delay_ms, 100)

    if delay == 0,
      do: send(self(), :attempt_election),
      else: Process.send_after(self(), :attempt_election, delay)
  end

  defp verify_leadership(%{is_leader: true} = state) do
    case :global.whereis_name(@election_key) do
      pid when pid == self() ->
        state

      _ ->
        Logger.warning("Lost global leadership registration, stepping down")
        :persistent_term.put({__MODULE__, :is_leader}, false)
        _ = notify_leadership_change(false)
        %{state | is_leader: false, leader_node: nil}
    end
  end

  defp verify_leadership(state), do: state

  defp attempt_registration_atomic(state) do
    result =
      :global.trans({@election_key, node()}, fn ->
        case :global.whereis_name(@election_key) do
          :undefined ->
            :global.register_name(@election_key, self(), &resolve_conflict/3)

          pid ->
            check_own_registration(pid)
        end
      end)

    case result do
      :yes ->
        Logger.info("Elected as APRS-IS connection leader on node #{node()}")
        :persistent_term.put({__MODULE__, :is_leader}, true)
        _ = notify_leadership_change(true)
        {:noreply, %{state | is_leader: true, leader_node: node()}}

      :no ->
        leader_pid = :global.whereis_name(@election_key)
        leader_node = if leader_pid != :undefined and is_pid(leader_pid), do: node(leader_pid)
        :persistent_term.put({__MODULE__, :is_leader}, false)
        {:noreply, %{state | is_leader: false, leader_node: leader_node}}
    end
  end

  defp check_own_registration(pid) do
    if pid == self(), do: :yes, else: :no
  end

  # Conflict resolution - prefer the process on the lexicographically lower node
  defp resolve_conflict(_name, pid1, pid2) do
    node1 = node(pid1)
    node2 = node(pid2)

    Logger.info("Resolving leader conflict between #{node1} and #{node2}")

    # Choose based on node name ordering for deterministic results
    if node1 <= node2 do
      pid1
    else
      pid2
    end
  end

  defp cleanup_stale_registrations do
    case :global.whereis_name(@election_key) do
      :undefined ->
        :ok

      pid when is_pid(pid) ->
        check_and_cleanup_registration(pid)
    end
  end

  defp check_and_cleanup_registration(pid) do
    pid_node = node(pid)
    connected_nodes = [node() | Node.list()]

    if pid_node in connected_nodes do
      check_pid_liveness(pid, pid_node)
    else
      cleanup_registration("disconnected node #{pid_node}")
    end
  end

  defp check_pid_liveness(pid, pid_node) do
    if pid_alive?(pid, pid_node) do
      :ok
    else
      reason = if pid_node == node(), do: "dead local process", else: "dead remote process"
      cleanup_registration("#{reason} #{inspect(pid)}")
    end
  rescue
    _error ->
      cleanup_registration("problematic process #{inspect(pid)}")
  end

  defp pid_alive?(pid, pid_node) when pid_node == node() do
    Process.alive?(pid)
  end

  defp pid_alive?(pid, pid_node) do
    case :rpc.call(pid_node, Process, :alive?, [pid], 5000) do
      {:badrpc, _} -> false
      result -> result == true
    end
  end

  defp cleanup_registration(reason) do
    Logger.info("Cleaning up stale leader registration for #{reason}")
    :global.unregister_name(@election_key)
  end

  defp get_cluster_wide_status do
    all_nodes = [node() | Node.list()]

    # Check each node for APRS-IS connection status
    connected_statuses =
      all_nodes
      |> Enum.map(&get_node_status/1)
      |> Enum.filter(fn status -> status.connected end)

    case connected_statuses do
      [status | _] ->
        # At least one node is connected - return its status
        # Add cluster info to indicate this is cluster-wide status
        Map.put(status, :cluster_info, %{
          total_nodes: length(all_nodes),
          connected_nodes: length(connected_statuses),
          leader_node: get_leader_node_name(),
          all_nodes: all_nodes |> Enum.map(&to_string/1) |> Enum.sort()
        })

      [] ->
        # No nodes are connected - return local status but mark as cluster-wide
        local_status = Aprsme.Is.get_status()

        Map.put(local_status, :cluster_info, %{
          total_nodes: length(all_nodes),
          connected_nodes: 0,
          leader_node: get_leader_node_name(),
          all_nodes: all_nodes |> Enum.map(&to_string/1) |> Enum.sort()
        })
    end
  end

  defp get_node_status(node_name) do
    if node_name == node() do
      # Local node - call directly
      Aprsme.Is.get_status()
    else
      # Remote node - use RPC
      case :rpc.call(node_name, Aprsme.Is, :get_status, [], 5000) do
        {:badrpc, _reason} ->
          # Node unreachable - return disconnected status
          %{connected: false, server: "unreachable", port: 0}

        status when is_map(status) ->
          status

        _ ->
          %{connected: false, server: "error", port: 0}
      end
    end
  rescue
    _error ->
      %{connected: false, server: "error", port: 0}
  end

  defp get_leader_node_name do
    case :global.whereis_name(@election_key) do
      :undefined -> "none"
      pid when is_pid(pid) -> pid |> node() |> to_string()
    end
  end

  defp notify_leadership_change(became_leader) do
    Phoenix.PubSub.broadcast(
      Aprsme.PubSub,
      "cluster:leadership",
      {:leadership_change, node(), became_leader}
    )
  end
end