graham/aprs.me
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: add timeout for cluster formation to ensure APRS-IS connection

Browse source 
Previously, when clustering was enabled, the leader election process
would wait indefinitely for the cluster to form. If cluster formation
failed or took too long, no leader would be elected and the APRS-IS
connection would never start, leaving the system disconnected.

Changes:
- Add 30-second timeout for cluster formation
- Force leader election after timeout, even in single-node mode
- Add election_forced flag to prevent duplicate election attempts
- Improve logging to distinguish between cluster and single-node modes

This ensures that at least one node will always connect to APRS-IS,
even if clustering fails completely.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Graham McIntire 2025-10-24 15:00:43 -05:00
parent 537057ee9a
commit 18d95b9a1f
No known key found for this signature in database
1 changed files with 44 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
lib/aprsme/cluster/leader_election.ex
View file

@ -9,6 +9,8 @@ defmodule Aprsme.Cluster.LeaderElection do
@election_key {:aprs_is_leader, __MODULE__}
@check_interval 5_000
# Maximum time to wait for cluster formation before proceeding with election (30 seconds)
@max_cluster_wait 30_000
def start_link(opts) do
GenServer.start_link(__MODULE__, opts, name: __MODULE__)
@ -45,8 +47,10 @@ defmodule Aprsme.Cluster.LeaderElection do
if cluster_enabled do
Logger.info("Clustering enabled - waiting for cluster formation before leader election")
# Wait longer for cluster to form, then check periodically
# Wait for cluster to form, then check periodically
Process.send_after(self(), :check_cluster_and_elect, 2_000)
# Set a timeout to force election if cluster doesn't form
Process.send_after(self(), :force_election_timeout, @max_cluster_wait)
else
Logger.info("Clustering disabled - proceeding with immediate leader election")
# Non-clustered mode - elect immediately
@ -56,22 +60,51 @@ defmodule Aprsme.Cluster.LeaderElection do
# Schedule periodic checks
Process.send_after(self(), :check_leadership, @check_interval)
{:ok, %{is_leader: false, leader_node: nil, cluster_enabled: cluster_enabled}}
{:ok, %{is_leader: false, leader_node: nil, cluster_enabled: cluster_enabled, election_forced: false}}
end
@impl true
def handle_info(:check_cluster_and_elect, state) do
connected_nodes = Node.list()
if length(connected_nodes) > 0 do
Logger.info("Cluster formed with #{length(connected_nodes)} other nodes: #{inspect(connected_nodes)}")
Logger.info("Proceeding with leader election")
Process.send_after(self(), :attempt_election, 100)
# Don't keep checking if election was already forced
if state.election_forced do
{:noreply, state}
else
Logger.info("Cluster not yet formed - waiting...")
# Check again in 2 seconds
Process.send_after(self(), :check_cluster_and_elect, 2_000)
connected_nodes = Node.list()
if length(connected_nodes) > 0 do
Logger.info("Cluster formed with #{length(connected_nodes)} other nodes: #{inspect(connected_nodes)}")
Logger.info("Proceeding with leader election")
Process.send_after(self(), :attempt_election, 100)
{:noreply, %{state | election_forced: true}}
else
Logger.debug("Cluster not yet formed - waiting...")
# Check again in 2 seconds
Process.send_after(self(), :check_cluster_and_elect, 2_000)
{:noreply, state}
end
end
end
@impl true
def handle_info(:force_election_timeout, state) do
# Only force election if we haven't already started one
if not state.election_forced and not state.is_leader do
connected_nodes = Node.list()
if length(connected_nodes) == 0 do
Logger.warning(
"Cluster formation timeout reached after #{@max_cluster_wait}ms with no connected nodes. " <>
"Proceeding with leader election in single-node mode to ensure APRS-IS connection."
)
else
Logger.info(
"Forcing leader election after #{@max_cluster_wait}ms wait with #{length(connected_nodes)} connected nodes"
)
end
Process.send_after(self(), :attempt_election, 100)
{:noreply, %{state | election_forced: true}}
else
{:noreply, state}
end
end