diff --git a/lib/towerops_web/telemetry.ex b/lib/towerops_web/telemetry.ex index 04994c91..c03e0611 100644 --- a/lib/towerops_web/telemetry.ex +++ b/lib/towerops_web/telemetry.ex @@ -97,18 +97,145 @@ defmodule ToweropsWeb.Telemetry do summary("vm.memory.total", unit: {:byte, :kilobyte}), summary("vm.total_run_queue_lengths.total"), summary("vm.total_run_queue_lengths.cpu"), - summary("vm.total_run_queue_lengths.io") + summary("vm.total_run_queue_lengths.io"), + + # Exq/Redis Metrics + last_value("towerops.exq.queue.size", + tags: [:queue], + description: "Number of jobs in each Exq queue" + ), + last_value("towerops.exq.processes.busy", + description: "Number of busy Exq worker processes" + ), + last_value("towerops.exq.processes.total", + description: "Total number of Exq worker processes" + ), + last_value("towerops.redis.connected_clients", + description: "Number of Redis/Valkey connected clients" + ), + last_value("towerops.redis.used_memory", + unit: {:byte, :megabyte}, + description: "Redis/Valkey memory usage" + ), + last_value("towerops.redis.commands_processed", + description: "Total commands processed by Redis/Valkey" + ) ] end defp periodic_measurements do [ - # A module, function and arguments to be invoked periodically. - # This function must call :telemetry.execute/3 and a metric must be added above. - # {ToweropsWeb, :count_users, []} + # Measure Exq and Redis stats every 10 seconds + {__MODULE__, :publish_exq_stats, []}, + {__MODULE__, :publish_redis_stats, []} ] end + @doc """ + Publishes Exq queue and process statistics. + """ + def publish_exq_stats do + # Only run if Exq is available (not in test env) + if Application.get_env(:towerops, :env) != :test do + try do + # Get stats for each queue + queues = ["default", "discovery", "polling", "monitoring", "maintenance"] + + for queue <- queues do + {:ok, size} = Exq.Api.queue_size(Exq, queue) + + :telemetry.execute( + [:towerops, :exq, :queue, :size], + %{value: size}, + %{queue: queue} + ) + end + + # Get process stats + {:ok, processes} = Exq.Api.processes(Exq) + {:ok, busy} = Exq.Api.busy(Exq) + + :telemetry.execute( + [:towerops, :exq, :processes, :busy], + %{value: length(busy)}, + %{} + ) + + :telemetry.execute( + [:towerops, :exq, :processes, :total], + %{value: length(processes)}, + %{} + ) + rescue + _ -> :ok + end + end + end + + @doc """ + Publishes Redis/Valkey statistics. + """ + def publish_redis_stats do + # Only run if Redis is configured (not in test env) + if Application.get_env(:towerops, :env) != :test do + try do + redis_config = Application.get_env(:towerops, :redis, []) + host = Keyword.get(redis_config, :host, "localhost") + port = Keyword.get(redis_config, :port, 6379) + + # Connect to Redis and get INFO + {:ok, conn} = Redix.start_link(host: host, port: port) + {:ok, info} = Redix.command(conn, ["INFO", "stats"]) + {:ok, memory_info} = Redix.command(conn, ["INFO", "memory"]) + {:ok, clients_info} = Redix.command(conn, ["INFO", "clients"]) + + # Parse INFO response + stats = parse_redis_info(info) + memory_stats = parse_redis_info(memory_info) + client_stats = parse_redis_info(clients_info) + + # Publish metrics + if total_commands = stats["total_commands_processed"] do + :telemetry.execute( + [:towerops, :redis, :commands_processed], + %{value: String.to_integer(total_commands)}, + %{} + ) + end + + if used_memory = memory_stats["used_memory"] do + :telemetry.execute( + [:towerops, :redis, :used_memory], + %{value: String.to_integer(used_memory)}, + %{} + ) + end + + if connected_clients = client_stats["connected_clients"] do + :telemetry.execute( + [:towerops, :redis, :connected_clients], + %{value: String.to_integer(connected_clients)}, + %{} + ) + end + + Redix.stop(conn) + rescue + _ -> :ok + end + end + end + + # Parse Redis INFO command output + defp parse_redis_info(info_string) do + info_string + |> String.split("\r\n") + |> Enum.reject(&(String.starts_with?(&1, "#") or &1 == "")) + |> Enum.map(&String.split(&1, ":", parts: 2)) + |> Enum.filter(&(length(&1) == 2)) + |> Map.new(fn [key, value] -> {key, value} end) + end + # Telemetry handler for router exceptions def handle_router_exception(_event, _measurements, metadata, _config) do require Logger