From eb9ba57f2e15c06d2241106aaf36ae0ff3d7ac19 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Sun, 18 Jan 2026 11:25:26 -0600
Subject: [PATCH] fix(telemetry): handle Exq/Redis connection errors gracefully

Changed pattern matching to proper error handling in telemetry functions
to prevent crashes when Exq or Redis connections aren't available.

Issues fixed:
1. Exq.Api.queue_size/2 calls were using pattern matching {:ok, size} =
   which crashed with FunctionClauseError when Exq wasn't connected to Redis

2. Redix.start_link/1 and Redix.command/2 calls were using pattern matching
   which crashed when Redis wasn't available

Changes:
- publish_exq_stats: Use case statements for queue_size calls
- publish_exq_stats: Use with statement for processes/busy calls
- publish_redis_stats: Use with statement for all Redis operations

This allows the app to start and run even when Redis/Exq connections
aren't immediately available, which is especially important during
container startup and rolling deployments.
---
 lib/towerops_web/telemetry.ex | 108 ++++++++++++++++++----------------
 1 file changed, 57 insertions(+), 51 deletions(-)

diff --git a/lib/towerops_web/telemetry.ex b/lib/towerops_web/telemetry.ex
index bc8eb05a..da65428c 100644
--- a/lib/towerops_web/telemetry.ex
+++ b/lib/towerops_web/telemetry.ex
@@ -143,29 +143,33 @@ defmodule ToweropsWeb.Telemetry do
         queues = ["default", "discovery", "polling", "monitoring", "maintenance"]
 
         for queue <- queues do
-          {:ok, size} = Exq.Api.queue_size(Exq, queue)
+          case Exq.Api.queue_size(Exq, queue) do
+            {:ok, size} ->
+              :telemetry.execute(
+                [:towerops, :exq, :queue, :size],
+                %{value: size},
+                %{queue: queue}
+              )
 
-          :telemetry.execute(
-            [:towerops, :exq, :queue, :size],
-            %{value: size},
-            %{queue: queue}
-          )
+            _error ->
+              :ok
+          end
         end
 
-        {:ok, processes} = Exq.Api.processes(Exq)
-        {:ok, busy} = Exq.Api.busy(Exq)
+        with {:ok, processes} <- Exq.Api.processes(Exq),
+             {:ok, busy} <- Exq.Api.busy(Exq) do
+          :telemetry.execute(
+            [:towerops, :exq, :processes, :busy],
+            %{value: length(busy)},
+            %{}
+          )
 
-        :telemetry.execute(
-          [:towerops, :exq, :processes, :busy],
-          %{value: length(busy)},
-          %{}
-        )
-
-        :telemetry.execute(
-          [:towerops, :exq, :processes, :total],
-          %{value: length(processes)},
-          %{}
-        )
+          :telemetry.execute(
+            [:towerops, :exq, :processes, :total],
+            %{value: length(processes)},
+            %{}
+          )
+        end
 
         :ok
       catch
@@ -193,40 +197,42 @@ defmodule ToweropsWeb.Telemetry do
         redis_config = Application.get_env(:towerops, :redis, [])
         host = Keyword.get(redis_config, :host, "localhost")
         port = Keyword.get(redis_config, :port, 6379)
-        {:ok, conn} = Redix.start_link(host: host, port: port)
-        {:ok, info} = Redix.command(conn, ["INFO", "stats"])
-        {:ok, memory_info} = Redix.command(conn, ["INFO", "memory"])
-        {:ok, clients_info} = Redix.command(conn, ["INFO", "clients"])
-        stats = parse_redis_info(info)
-        memory_stats = parse_redis_info(memory_info)
-        client_stats = parse_redis_info(clients_info)
 
-        if total_commands = stats["total_commands_processed"] do
-          :telemetry.execute(
-            [:towerops, :redis, :commands_processed],
-            %{value: String.to_integer(total_commands)},
-            %{}
-          )
+        with {:ok, conn} <- Redix.start_link(host: host, port: port),
+             {:ok, info} <- Redix.command(conn, ["INFO", "stats"]),
+             {:ok, memory_info} <- Redix.command(conn, ["INFO", "memory"]),
+             {:ok, clients_info} <- Redix.command(conn, ["INFO", "clients"]) do
+          stats = parse_redis_info(info)
+          memory_stats = parse_redis_info(memory_info)
+          client_stats = parse_redis_info(clients_info)
+
+          if total_commands = stats["total_commands_processed"] do
+            :telemetry.execute(
+              [:towerops, :redis, :commands_processed],
+              %{value: String.to_integer(total_commands)},
+              %{}
+            )
+          end
+
+          if used_memory = memory_stats["used_memory"] do
+            :telemetry.execute(
+              [:towerops, :redis, :used_memory],
+              %{value: String.to_integer(used_memory)},
+              %{}
+            )
+          end
+
+          if connected_clients = client_stats["connected_clients"] do
+            :telemetry.execute(
+              [:towerops, :redis, :connected_clients],
+              %{value: String.to_integer(connected_clients)},
+              %{}
+            )
+          end
+
+          Redix.stop(conn)
+          :ok
         end
-
-        if used_memory = memory_stats["used_memory"] do
-          :telemetry.execute(
-            [:towerops, :redis, :used_memory],
-            %{value: String.to_integer(used_memory)},
-            %{}
-          )
-        end
-
-        if connected_clients = client_stats["connected_clients"] do
-          :telemetry.execute(
-            [:towerops, :redis, :connected_clients],
-            %{value: String.to_integer(connected_clients)},
-            %{}
-          )
-        end
-
-        Redix.stop(conn)
-        :ok
       rescue
         _ -> :ok
       end