Add Exq and Redis/Valkey metrics to LiveDashboard
Added telemetry metrics for monitoring: Exq Metrics: - Queue sizes for all queues (default, discovery, polling, monitoring, maintenance) - Number of busy worker processes - Total number of worker processes Redis/Valkey Metrics: - Connected clients - Memory usage - Total commands processed Metrics are collected every 10 seconds via telemetry_poller and displayed in LiveDashboard at /dashboard
This commit is contained in:
parent
3b4fd98b21
commit
6ddb74a766
1 changed files with 131 additions and 4 deletions
|
|
@ -97,18 +97,145 @@ defmodule ToweropsWeb.Telemetry do
|
|||
summary("vm.memory.total", unit: {:byte, :kilobyte}),
|
||||
summary("vm.total_run_queue_lengths.total"),
|
||||
summary("vm.total_run_queue_lengths.cpu"),
|
||||
summary("vm.total_run_queue_lengths.io")
|
||||
summary("vm.total_run_queue_lengths.io"),
|
||||
|
||||
# Exq/Redis Metrics
|
||||
last_value("towerops.exq.queue.size",
|
||||
tags: [:queue],
|
||||
description: "Number of jobs in each Exq queue"
|
||||
),
|
||||
last_value("towerops.exq.processes.busy",
|
||||
description: "Number of busy Exq worker processes"
|
||||
),
|
||||
last_value("towerops.exq.processes.total",
|
||||
description: "Total number of Exq worker processes"
|
||||
),
|
||||
last_value("towerops.redis.connected_clients",
|
||||
description: "Number of Redis/Valkey connected clients"
|
||||
),
|
||||
last_value("towerops.redis.used_memory",
|
||||
unit: {:byte, :megabyte},
|
||||
description: "Redis/Valkey memory usage"
|
||||
),
|
||||
last_value("towerops.redis.commands_processed",
|
||||
description: "Total commands processed by Redis/Valkey"
|
||||
)
|
||||
]
|
||||
end
|
||||
|
||||
defp periodic_measurements do
|
||||
[
|
||||
# A module, function and arguments to be invoked periodically.
|
||||
# This function must call :telemetry.execute/3 and a metric must be added above.
|
||||
# {ToweropsWeb, :count_users, []}
|
||||
# Measure Exq and Redis stats every 10 seconds
|
||||
{__MODULE__, :publish_exq_stats, []},
|
||||
{__MODULE__, :publish_redis_stats, []}
|
||||
]
|
||||
end
|
||||
|
||||
@doc """
|
||||
Publishes Exq queue and process statistics.
|
||||
"""
|
||||
def publish_exq_stats do
|
||||
# Only run if Exq is available (not in test env)
|
||||
if Application.get_env(:towerops, :env) != :test do
|
||||
try do
|
||||
# Get stats for each queue
|
||||
queues = ["default", "discovery", "polling", "monitoring", "maintenance"]
|
||||
|
||||
for queue <- queues do
|
||||
{:ok, size} = Exq.Api.queue_size(Exq, queue)
|
||||
|
||||
:telemetry.execute(
|
||||
[:towerops, :exq, :queue, :size],
|
||||
%{value: size},
|
||||
%{queue: queue}
|
||||
)
|
||||
end
|
||||
|
||||
# Get process stats
|
||||
{:ok, processes} = Exq.Api.processes(Exq)
|
||||
{:ok, busy} = Exq.Api.busy(Exq)
|
||||
|
||||
:telemetry.execute(
|
||||
[:towerops, :exq, :processes, :busy],
|
||||
%{value: length(busy)},
|
||||
%{}
|
||||
)
|
||||
|
||||
:telemetry.execute(
|
||||
[:towerops, :exq, :processes, :total],
|
||||
%{value: length(processes)},
|
||||
%{}
|
||||
)
|
||||
rescue
|
||||
_ -> :ok
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Publishes Redis/Valkey statistics.
|
||||
"""
|
||||
def publish_redis_stats do
|
||||
# Only run if Redis is configured (not in test env)
|
||||
if Application.get_env(:towerops, :env) != :test do
|
||||
try do
|
||||
redis_config = Application.get_env(:towerops, :redis, [])
|
||||
host = Keyword.get(redis_config, :host, "localhost")
|
||||
port = Keyword.get(redis_config, :port, 6379)
|
||||
|
||||
# Connect to Redis and get INFO
|
||||
{:ok, conn} = Redix.start_link(host: host, port: port)
|
||||
{:ok, info} = Redix.command(conn, ["INFO", "stats"])
|
||||
{:ok, memory_info} = Redix.command(conn, ["INFO", "memory"])
|
||||
{:ok, clients_info} = Redix.command(conn, ["INFO", "clients"])
|
||||
|
||||
# Parse INFO response
|
||||
stats = parse_redis_info(info)
|
||||
memory_stats = parse_redis_info(memory_info)
|
||||
client_stats = parse_redis_info(clients_info)
|
||||
|
||||
# Publish metrics
|
||||
if total_commands = stats["total_commands_processed"] do
|
||||
:telemetry.execute(
|
||||
[:towerops, :redis, :commands_processed],
|
||||
%{value: String.to_integer(total_commands)},
|
||||
%{}
|
||||
)
|
||||
end
|
||||
|
||||
if used_memory = memory_stats["used_memory"] do
|
||||
:telemetry.execute(
|
||||
[:towerops, :redis, :used_memory],
|
||||
%{value: String.to_integer(used_memory)},
|
||||
%{}
|
||||
)
|
||||
end
|
||||
|
||||
if connected_clients = client_stats["connected_clients"] do
|
||||
:telemetry.execute(
|
||||
[:towerops, :redis, :connected_clients],
|
||||
%{value: String.to_integer(connected_clients)},
|
||||
%{}
|
||||
)
|
||||
end
|
||||
|
||||
Redix.stop(conn)
|
||||
rescue
|
||||
_ -> :ok
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Parse Redis INFO command output
|
||||
defp parse_redis_info(info_string) do
|
||||
info_string
|
||||
|> String.split("\r\n")
|
||||
|> Enum.reject(&(String.starts_with?(&1, "#") or &1 == ""))
|
||||
|> Enum.map(&String.split(&1, ":", parts: 2))
|
||||
|> Enum.filter(&(length(&1) == 2))
|
||||
|> Map.new(fn [key, value] -> {key, value} end)
|
||||
end
|
||||
|
||||
# Telemetry handler for router exceptions
|
||||
def handle_router_exception(_event, _measurements, metadata, _config) do
|
||||
require Logger
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue