fix(telemetry): add comprehensive error handling for Exq API calls
Wrapped all Exq.Api calls in individual try/rescue/catch blocks to handle GenServer exits and exceptions when Exq hasn't connected to Redis yet. The "GenServer Exq terminating" errors in logs are from Exq itself crashing when it receives calls before it's ready, not from our code. These will still appear in logs as Exq restarts, but our telemetry code won't crash. Changes: - Added try/rescue/catch around each queue_size call in the loop - Added try/rescue/catch around processes/busy calls - Kept outer try/rescue/catch as a final safety net - rescue blocks must come before catch blocks in Elixir
This commit is contained in:
parent
2ecc2082e6
commit
dab3e29025
1 changed files with 35 additions and 21 deletions
|
|
@ -143,35 +143,49 @@ defmodule ToweropsWeb.Telemetry do
|
|||
queues = ["default", "discovery", "polling", "monitoring", "maintenance"]
|
||||
|
||||
for queue <- queues do
|
||||
case Exq.Api.queue_size(Exq, queue) do
|
||||
{:ok, size} ->
|
||||
:telemetry.execute(
|
||||
[:towerops, :exq, :queue, :size],
|
||||
%{value: size},
|
||||
%{queue: queue}
|
||||
)
|
||||
try do
|
||||
case Exq.Api.queue_size(Exq, queue) do
|
||||
{:ok, size} ->
|
||||
:telemetry.execute(
|
||||
[:towerops, :exq, :queue, :size],
|
||||
%{value: size},
|
||||
%{queue: queue}
|
||||
)
|
||||
|
||||
_error ->
|
||||
:ok
|
||||
_error ->
|
||||
:ok
|
||||
end
|
||||
rescue
|
||||
_ -> :ok
|
||||
catch
|
||||
:exit, _ -> :ok
|
||||
end
|
||||
end
|
||||
|
||||
with {:ok, processes} <- Exq.Api.processes(Exq),
|
||||
{:ok, busy} <- Exq.Api.busy(Exq) do
|
||||
:telemetry.execute(
|
||||
[:towerops, :exq, :processes, :busy],
|
||||
%{value: length(busy)},
|
||||
%{}
|
||||
)
|
||||
try do
|
||||
with {:ok, processes} <- Exq.Api.processes(Exq),
|
||||
{:ok, busy} <- Exq.Api.busy(Exq) do
|
||||
:telemetry.execute(
|
||||
[:towerops, :exq, :processes, :busy],
|
||||
%{value: length(busy)},
|
||||
%{}
|
||||
)
|
||||
|
||||
:telemetry.execute(
|
||||
[:towerops, :exq, :processes, :total],
|
||||
%{value: length(processes)},
|
||||
%{}
|
||||
)
|
||||
:telemetry.execute(
|
||||
[:towerops, :exq, :processes, :total],
|
||||
%{value: length(processes)},
|
||||
%{}
|
||||
)
|
||||
end
|
||||
rescue
|
||||
_ -> :ok
|
||||
catch
|
||||
:exit, _ -> :ok
|
||||
end
|
||||
|
||||
:ok
|
||||
rescue
|
||||
_ -> :ok
|
||||
catch
|
||||
# Catch exits (like :noproc) and exceptions
|
||||
_, _ -> :ok
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue