graham/towerops
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(telemetry): handle Exq/Redis connection errors gracefully

Browse source 
Changed pattern matching to proper error handling in telemetry functions
to prevent crashes when Exq or Redis connections aren't available.

Issues fixed:
1. Exq.Api.queue_size/2 calls were using pattern matching {:ok, size} =
   which crashed with FunctionClauseError when Exq wasn't connected to Redis

2. Redix.start_link/1 and Redix.command/2 calls were using pattern matching
   which crashed when Redis wasn't available

Changes:
- publish_exq_stats: Use case statements for queue_size calls
- publish_exq_stats: Use with statement for processes/busy calls
- publish_redis_stats: Use with statement for all Redis operations

This allows the app to start and run even when Redis/Exq connections
aren't immediately available, which is especially important during
container startup and rolling deployments.
This commit is contained in:
Graham McIntire 2026-01-18 11:25:26 -06:00
parent 7e9940c619
commit eb9ba57f2e
No known key found for this signature in database
1 changed files with 57 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
lib/towerops_web/telemetry.ex
View file

@ -143,29 +143,33 @@ defmodule ToweropsWeb.Telemetry do
queues = ["default", "discovery", "polling", "monitoring", "maintenance"]
for queue <- queues do
{:ok, size} = Exq.Api.queue_size(Exq, queue)
case Exq.Api.queue_size(Exq, queue) do
{:ok, size} ->
:telemetry.execute(
[:towerops, :exq, :queue, :size],
%{value: size},
%{queue: queue}
)
:telemetry.execute(
[:towerops, :exq, :queue, :size],
%{value: size},
%{queue: queue}
)
_error ->
:ok
end
end
{:ok, processes} = Exq.Api.processes(Exq)
{:ok, busy} = Exq.Api.busy(Exq)
with {:ok, processes} <- Exq.Api.processes(Exq),
{:ok, busy} <- Exq.Api.busy(Exq) do
:telemetry.execute(
[:towerops, :exq, :processes, :busy],
%{value: length(busy)},
%{}
)
:telemetry.execute(
[:towerops, :exq, :processes, :busy],
%{value: length(busy)},
%{}
)
:telemetry.execute(
[:towerops, :exq, :processes, :total],
%{value: length(processes)},
%{}
)
:telemetry.execute(
[:towerops, :exq, :processes, :total],
%{value: length(processes)},
%{}
)
end
:ok
catch
@ -193,40 +197,42 @@ defmodule ToweropsWeb.Telemetry do
redis_config = Application.get_env(:towerops, :redis, [])
host = Keyword.get(redis_config, :host, "localhost")
port = Keyword.get(redis_config, :port, 6379)
{:ok, conn} = Redix.start_link(host: host, port: port)
{:ok, info} = Redix.command(conn, ["INFO", "stats"])
{:ok, memory_info} = Redix.command(conn, ["INFO", "memory"])
{:ok, clients_info} = Redix.command(conn, ["INFO", "clients"])
stats = parse_redis_info(info)
memory_stats = parse_redis_info(memory_info)
client_stats = parse_redis_info(clients_info)
if total_commands = stats["total_commands_processed"] do
:telemetry.execute(
[:towerops, :redis, :commands_processed],
%{value: String.to_integer(total_commands)},
%{}
)
with {:ok, conn} <- Redix.start_link(host: host, port: port),
{:ok, info} <- Redix.command(conn, ["INFO", "stats"]),
{:ok, memory_info} <- Redix.command(conn, ["INFO", "memory"]),
{:ok, clients_info} <- Redix.command(conn, ["INFO", "clients"]) do
stats = parse_redis_info(info)
memory_stats = parse_redis_info(memory_info)
client_stats = parse_redis_info(clients_info)
if total_commands = stats["total_commands_processed"] do
:telemetry.execute(
[:towerops, :redis, :commands_processed],
%{value: String.to_integer(total_commands)},
%{}
)
end
if used_memory = memory_stats["used_memory"] do
:telemetry.execute(
[:towerops, :redis, :used_memory],
%{value: String.to_integer(used_memory)},
%{}
)
end
if connected_clients = client_stats["connected_clients"] do
:telemetry.execute(
[:towerops, :redis, :connected_clients],
%{value: String.to_integer(connected_clients)},
%{}
)
end
Redix.stop(conn)
:ok
end
if used_memory = memory_stats["used_memory"] do
:telemetry.execute(
[:towerops, :redis, :used_memory],
%{value: String.to_integer(used_memory)},
%{}
)
end
if connected_clients = client_stats["connected_clients"] do
:telemetry.execute(
[:towerops, :redis, :connected_clients],
%{value: String.to_integer(connected_clients)},
%{}
)
end
Redix.stop(conn)
:ok
rescue
_ -> :ok
end