fix: make etcd watcher optional for graceful degradation
The app was crashing when the etcd watcher couldn't be created, even
though 2/3 etcd nodes were healthy and the connection succeeded.
Changes:
- Made watcher creation optional (nil if it fails)
- App now starts successfully even if watcher is unavailable
- Falls back to periodic rebalancing (60s intervals) instead of instant failover
- Changed error level from error to warning when watcher creation fails
- Added helpful context to warning message
This allows the app to run with degraded performance instead of crashing
completely when there are etcd cluster issues.
🤖 Generated with Claude Code
Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
0888d4eed1
commit
f5ca3f0940
1 changed files with 20 additions and 13 deletions
|
|
@ -34,27 +34,34 @@ defmodule Towerops.Monitoring.EtcdCoordinator do
|
|||
{:ok, conn} ->
|
||||
Logger.info("Connected to etcd cluster")
|
||||
|
||||
# Watch for lock releases (triggers instant failover)
|
||||
# Try to create watcher for lock releases (optimizes instant failover)
|
||||
# If watcher creation fails, we'll rely on periodic rebalancing instead
|
||||
watch_ctx = :eetcd_watch.new()
|
||||
watch_with_key = :eetcd_watch.with_key(watch_ctx, "towerops/locks/")
|
||||
watch_with_prefix = :eetcd_watch.with_prefix(watch_with_key)
|
||||
|
||||
case :eetcd_watch.watch(conn, watch_with_prefix) do
|
||||
{:ok, watcher} ->
|
||||
Logger.info("Watching etcd for lock releases")
|
||||
watcher =
|
||||
case :eetcd_watch.watch(conn, watch_with_prefix) do
|
||||
{:ok, watcher} ->
|
||||
Logger.info("Watching etcd for lock releases (instant failover enabled)")
|
||||
watcher
|
||||
|
||||
# Start acquiring locks for all devices
|
||||
send(self(), :acquire_locks)
|
||||
{:error, reason} ->
|
||||
Logger.warning(
|
||||
"Failed to create etcd watcher: #{inspect(reason)}. " <>
|
||||
"Relying on periodic rebalancing instead (60s intervals)"
|
||||
)
|
||||
|
||||
# Schedule periodic rebalancing (safety net for missed watch events)
|
||||
schedule_rebalance()
|
||||
nil
|
||||
end
|
||||
|
||||
{:ok, %{conn: conn, watcher: watcher, owned_leases: %{}}}
|
||||
# Start acquiring locks for all devices
|
||||
send(self(), :acquire_locks)
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.error("Failed to create etcd watcher: #{inspect(reason)}")
|
||||
{:stop, {:etcd_watcher_failed, reason}}
|
||||
end
|
||||
# Schedule periodic rebalancing (safety net for missed watch events)
|
||||
schedule_rebalance()
|
||||
|
||||
{:ok, %{conn: conn, watcher: watcher, owned_leases: %{}}}
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.error("Failed to connect to etcd: #{inspect(reason)}")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue