Add Exq and Redis/Valkey metrics to LiveDashboard

Added telemetry metrics for monitoring:

Exq Metrics:
- Queue sizes for all queues (default, discovery, polling, monitoring, maintenance)
- Number of busy worker processes
- Total number of worker processes

Redis/Valkey Metrics:
- Connected clients
- Memory usage
- Total commands processed

Metrics are collected every 10 seconds via telemetry_poller and displayed
in LiveDashboard at /dashboard
This commit is contained in:
Graham McIntire 2026-01-17 17:26:28 -06:00
parent 3b4fd98b21
commit 6ddb74a766
No known key found for this signature in database

View file

@ -97,18 +97,145 @@ defmodule ToweropsWeb.Telemetry do
summary("vm.memory.total", unit: {:byte, :kilobyte}),
summary("vm.total_run_queue_lengths.total"),
summary("vm.total_run_queue_lengths.cpu"),
summary("vm.total_run_queue_lengths.io")
summary("vm.total_run_queue_lengths.io"),
# Exq/Redis Metrics
last_value("towerops.exq.queue.size",
tags: [:queue],
description: "Number of jobs in each Exq queue"
),
last_value("towerops.exq.processes.busy",
description: "Number of busy Exq worker processes"
),
last_value("towerops.exq.processes.total",
description: "Total number of Exq worker processes"
),
last_value("towerops.redis.connected_clients",
description: "Number of Redis/Valkey connected clients"
),
last_value("towerops.redis.used_memory",
unit: {:byte, :megabyte},
description: "Redis/Valkey memory usage"
),
last_value("towerops.redis.commands_processed",
description: "Total commands processed by Redis/Valkey"
)
]
end
defp periodic_measurements do
[
# A module, function and arguments to be invoked periodically.
# This function must call :telemetry.execute/3 and a metric must be added above.
# {ToweropsWeb, :count_users, []}
# Measure Exq and Redis stats every 10 seconds
{__MODULE__, :publish_exq_stats, []},
{__MODULE__, :publish_redis_stats, []}
]
end
@doc """
Publishes Exq queue and process statistics.
"""
def publish_exq_stats do
# Only run if Exq is available (not in test env)
if Application.get_env(:towerops, :env) != :test do
try do
# Get stats for each queue
queues = ["default", "discovery", "polling", "monitoring", "maintenance"]
for queue <- queues do
{:ok, size} = Exq.Api.queue_size(Exq, queue)
:telemetry.execute(
[:towerops, :exq, :queue, :size],
%{value: size},
%{queue: queue}
)
end
# Get process stats
{:ok, processes} = Exq.Api.processes(Exq)
{:ok, busy} = Exq.Api.busy(Exq)
:telemetry.execute(
[:towerops, :exq, :processes, :busy],
%{value: length(busy)},
%{}
)
:telemetry.execute(
[:towerops, :exq, :processes, :total],
%{value: length(processes)},
%{}
)
rescue
_ -> :ok
end
end
end
@doc """
Publishes Redis/Valkey statistics.
"""
def publish_redis_stats do
# Only run if Redis is configured (not in test env)
if Application.get_env(:towerops, :env) != :test do
try do
redis_config = Application.get_env(:towerops, :redis, [])
host = Keyword.get(redis_config, :host, "localhost")
port = Keyword.get(redis_config, :port, 6379)
# Connect to Redis and get INFO
{:ok, conn} = Redix.start_link(host: host, port: port)
{:ok, info} = Redix.command(conn, ["INFO", "stats"])
{:ok, memory_info} = Redix.command(conn, ["INFO", "memory"])
{:ok, clients_info} = Redix.command(conn, ["INFO", "clients"])
# Parse INFO response
stats = parse_redis_info(info)
memory_stats = parse_redis_info(memory_info)
client_stats = parse_redis_info(clients_info)
# Publish metrics
if total_commands = stats["total_commands_processed"] do
:telemetry.execute(
[:towerops, :redis, :commands_processed],
%{value: String.to_integer(total_commands)},
%{}
)
end
if used_memory = memory_stats["used_memory"] do
:telemetry.execute(
[:towerops, :redis, :used_memory],
%{value: String.to_integer(used_memory)},
%{}
)
end
if connected_clients = client_stats["connected_clients"] do
:telemetry.execute(
[:towerops, :redis, :connected_clients],
%{value: String.to_integer(connected_clients)},
%{}
)
end
Redix.stop(conn)
rescue
_ -> :ok
end
end
end
# Parse Redis INFO command output
defp parse_redis_info(info_string) do
info_string
|> String.split("\r\n")
|> Enum.reject(&(String.starts_with?(&1, "#") or &1 == ""))
|> Enum.map(&String.split(&1, ":", parts: 2))
|> Enum.filter(&(length(&1) == 2))
|> Map.new(fn [key, value] -> {key, value} end)
end
# Telemetry handler for router exceptions
def handle_router_exception(_event, _measurements, metadata, _config) do
require Logger