From f5ca3f09407d330548b0d33aef98104667588f54 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Sat, 24 Jan 2026 15:27:28 -0600
Subject: [PATCH] fix: make etcd watcher optional for graceful degradation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The app was crashing when the etcd watcher couldn't be created, even
though 2/3 etcd nodes were healthy and the connection succeeded.

Changes:
- Made watcher creation optional (nil if it fails)
- App now starts successfully even if watcher is unavailable
- Falls back to periodic rebalancing (60s intervals) instead of instant failover
- Changed error level from error to warning when watcher creation fails
- Added helpful context to warning message

This allows the app to run with degraded performance instead of crashing
completely when there are etcd cluster issues.

🤖 Generated with Claude Code

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
---
 lib/towerops/monitoring/etcd_coordinator.ex | 33 +++++++++++++--------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/lib/towerops/monitoring/etcd_coordinator.ex b/lib/towerops/monitoring/etcd_coordinator.ex
index a07718ea..73ee7b56 100644
--- a/lib/towerops/monitoring/etcd_coordinator.ex
+++ b/lib/towerops/monitoring/etcd_coordinator.ex
@@ -34,27 +34,34 @@ defmodule Towerops.Monitoring.EtcdCoordinator do
       {:ok, conn} ->
         Logger.info("Connected to etcd cluster")
 
-        # Watch for lock releases (triggers instant failover)
+        # Try to create watcher for lock releases (optimizes instant failover)
+        # If watcher creation fails, we'll rely on periodic rebalancing instead
         watch_ctx = :eetcd_watch.new()
         watch_with_key = :eetcd_watch.with_key(watch_ctx, "towerops/locks/")
         watch_with_prefix = :eetcd_watch.with_prefix(watch_with_key)
 
-        case :eetcd_watch.watch(conn, watch_with_prefix) do
-          {:ok, watcher} ->
-            Logger.info("Watching etcd for lock releases")
+        watcher =
+          case :eetcd_watch.watch(conn, watch_with_prefix) do
+            {:ok, watcher} ->
+              Logger.info("Watching etcd for lock releases (instant failover enabled)")
+              watcher
 
-            # Start acquiring locks for all devices
-            send(self(), :acquire_locks)
+            {:error, reason} ->
+              Logger.warning(
+                "Failed to create etcd watcher: #{inspect(reason)}. " <>
+                  "Relying on periodic rebalancing instead (60s intervals)"
+              )
 
-            # Schedule periodic rebalancing (safety net for missed watch events)
-            schedule_rebalance()
+              nil
+          end
 
-            {:ok, %{conn: conn, watcher: watcher, owned_leases: %{}}}
+        # Start acquiring locks for all devices
+        send(self(), :acquire_locks)
 
-          {:error, reason} ->
-            Logger.error("Failed to create etcd watcher: #{inspect(reason)}")
-            {:stop, {:etcd_watcher_failed, reason}}
-        end
+        # Schedule periodic rebalancing (safety net for missed watch events)
+        schedule_rebalance()
+
+        {:ok, %{conn: conn, watcher: watcher, owned_leases: %{}}}
 
       {:error, reason} ->
         Logger.error("Failed to connect to etcd: #{inspect(reason)}")