From dab3e2902589263fce201d0040ee1f72e52986fb Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Sun, 18 Jan 2026 11:32:58 -0600
Subject: [PATCH] fix(telemetry): add comprehensive error handling for Exq API
 calls

Wrapped all Exq.Api calls in individual try/rescue/catch blocks to handle
GenServer exits and exceptions when Exq hasn't connected to Redis yet.

The "GenServer Exq terminating" errors in logs are from Exq itself crashing
when it receives calls before it's ready, not from our code. These will still
appear in logs as Exq restarts, but our telemetry code won't crash.

Changes:
- Added try/rescue/catch around each queue_size call in the loop
- Added try/rescue/catch around processes/busy calls
- Kept outer try/rescue/catch as a final safety net
- rescue blocks must come before catch blocks in Elixir
---
 lib/towerops_web/telemetry.ex | 56 ++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 21 deletions(-)

diff --git a/lib/towerops_web/telemetry.ex b/lib/towerops_web/telemetry.ex
index da65428c..55c32714 100644
--- a/lib/towerops_web/telemetry.ex
+++ b/lib/towerops_web/telemetry.ex
@@ -143,35 +143,49 @@ defmodule ToweropsWeb.Telemetry do
         queues = ["default", "discovery", "polling", "monitoring", "maintenance"]
 
         for queue <- queues do
-          case Exq.Api.queue_size(Exq, queue) do
-            {:ok, size} ->
-              :telemetry.execute(
-                [:towerops, :exq, :queue, :size],
-                %{value: size},
-                %{queue: queue}
-              )
+          try do
+            case Exq.Api.queue_size(Exq, queue) do
+              {:ok, size} ->
+                :telemetry.execute(
+                  [:towerops, :exq, :queue, :size],
+                  %{value: size},
+                  %{queue: queue}
+                )
 
-            _error ->
-              :ok
+              _error ->
+                :ok
+            end
+          rescue
+            _ -> :ok
+          catch
+            :exit, _ -> :ok
           end
         end
 
-        with {:ok, processes} <- Exq.Api.processes(Exq),
-             {:ok, busy} <- Exq.Api.busy(Exq) do
-          :telemetry.execute(
-            [:towerops, :exq, :processes, :busy],
-            %{value: length(busy)},
-            %{}
-          )
+        try do
+          with {:ok, processes} <- Exq.Api.processes(Exq),
+               {:ok, busy} <- Exq.Api.busy(Exq) do
+            :telemetry.execute(
+              [:towerops, :exq, :processes, :busy],
+              %{value: length(busy)},
+              %{}
+            )
 
-          :telemetry.execute(
-            [:towerops, :exq, :processes, :total],
-            %{value: length(processes)},
-            %{}
-          )
+            :telemetry.execute(
+              [:towerops, :exq, :processes, :total],
+              %{value: length(processes)},
+              %{}
+            )
+          end
+        rescue
+          _ -> :ok
+        catch
+          :exit, _ -> :ok
         end
 
         :ok
+      rescue
+        _ -> :ok
       catch
         # Catch exits (like :noproc) and exceptions
         _, _ -> :ok