From 64ad63d3e3402206a2e5f543976ab564307782e7 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sat, 24 Jan 2026 15:02:46 -0600 Subject: [PATCH] fix: handle etcd watcher creation errors gracefully The pods were crashing with a MatchError when the etcd watcher failed to initialize. The code was pattern matching on {:ok, watcher} but eetcd was returning {:error, :eetcd_conn_unavailable}. Error: ** (MatchError) no match of right hand side value: {:error, :eetcd_conn_unavailable} This was causing CrashLoopBackOff because the application couldn't start the Monitoring.Supervisor due to EtcdCoordinator initialization failure. Fix: - Add case statement to handle both {:ok, watcher} and {:error, reason} - Log the specific error when watcher creation fails - Return proper error tuple to supervisor for clean shutdown --- lib/towerops/monitoring/etcd_coordinator.ex | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/towerops/monitoring/etcd_coordinator.ex b/lib/towerops/monitoring/etcd_coordinator.ex index 344538e5..a07718ea 100644 --- a/lib/towerops/monitoring/etcd_coordinator.ex +++ b/lib/towerops/monitoring/etcd_coordinator.ex @@ -38,16 +38,23 @@ defmodule Towerops.Monitoring.EtcdCoordinator do watch_ctx = :eetcd_watch.new() watch_with_key = :eetcd_watch.with_key(watch_ctx, "towerops/locks/") watch_with_prefix = :eetcd_watch.with_prefix(watch_with_key) - {:ok, watcher} = :eetcd_watch.watch(conn, watch_with_prefix) - Logger.info("Watching etcd for lock releases") - # Start acquiring locks for all devices - send(self(), :acquire_locks) + case :eetcd_watch.watch(conn, watch_with_prefix) do + {:ok, watcher} -> + Logger.info("Watching etcd for lock releases") - # Schedule periodic rebalancing (safety net for missed watch events) - schedule_rebalance() + # Start acquiring locks for all devices + send(self(), :acquire_locks) - {:ok, %{conn: conn, watcher: watcher, owned_leases: %{}}} + # Schedule periodic rebalancing (safety net for missed watch events) + schedule_rebalance() + + {:ok, %{conn: conn, watcher: watcher, owned_leases: %{}}} + + {:error, reason} -> + Logger.error("Failed to create etcd watcher: #{inspect(reason)}") + {:stop, {:etcd_watcher_failed, reason}} + end {:error, reason} -> Logger.error("Failed to connect to etcd: #{inspect(reason)}")