towerops/lib/towerops_web/telemetry.ex

defmodule ToweropsWeb.Telemetry do
  @moduledoc false
  use Supervisor

  import Telemetry.Metrics

  def start_link(arg) do
    Supervisor.start_link(__MODULE__, arg, name: __MODULE__)
  end

  @impl true
  def init(_arg) do
    # Attach telemetry handlers for logging request failures
    _ =
      :telemetry.attach(
        "towerops-router-exception",
        [:phoenix, :router_dispatch, :exception],
        &__MODULE__.handle_router_exception/4,
        nil
      )

    _ =
      :telemetry.attach(
        "towerops-endpoint-stop",
        [:phoenix, :endpoint, :stop],
        &__MODULE__.handle_endpoint_stop/4,
        nil
      )

    children = [
      # Telemetry poller will execute the given period measurements
      # every 10_000ms. Learn more here: https://hexdocs.pm/telemetry_metrics
      {:telemetry_poller, measurements: periodic_measurements(), period: 10_000}
      # Add reporters as children of your supervision tree.
      # {Telemetry.Metrics.ConsoleReporter, metrics: metrics()}
    ]

    Supervisor.init(children, strategy: :one_for_one)
  end

  def metrics do
    [
      # Phoenix Metrics
      summary("phoenix.endpoint.start.system_time",
        unit: {:native, :millisecond}
      ),
      summary("phoenix.endpoint.stop.duration",
        unit: {:native, :millisecond}
      ),
      summary("phoenix.router_dispatch.start.system_time",
        tags: [:route],
        unit: {:native, :millisecond}
      ),
      summary("phoenix.router_dispatch.exception.duration",
        tags: [:route],
        unit: {:native, :millisecond}
      ),
      summary("phoenix.router_dispatch.stop.duration",
        tags: [:route],
        unit: {:native, :millisecond}
      ),
      summary("phoenix.socket_connected.duration",
        unit: {:native, :millisecond}
      ),
      sum("phoenix.socket_drain.count"),
      summary("phoenix.channel_joined.duration",
        unit: {:native, :millisecond}
      ),
      summary("phoenix.channel_handled_in.duration",
        tags: [:event],
        unit: {:native, :millisecond}
      ),

      # Database Metrics
      summary("towerops.repo.query.total_time",
        unit: {:native, :millisecond},
        description: "The sum of the other measurements"
      ),
      summary("towerops.repo.query.decode_time",
        unit: {:native, :millisecond},
        description: "The time spent decoding the data received from the database"
      ),
      summary("towerops.repo.query.query_time",
        unit: {:native, :millisecond},
        description: "The time spent executing the query"
      ),
      summary("towerops.repo.query.queue_time",
        unit: {:native, :millisecond},
        description: "The time spent waiting for a database connection"
      ),
      summary("towerops.repo.query.idle_time",
        unit: {:native, :millisecond},
        description: "The time the connection spent waiting before being checked out for the query"
      ),

      # VM Metrics
      summary("vm.memory.total", unit: {:byte, :kilobyte}),
      summary("vm.total_run_queue_lengths.total"),
      summary("vm.total_run_queue_lengths.cpu"),
      summary("vm.total_run_queue_lengths.io"),

      # Exq/Redis Metrics
      last_value("towerops.exq.queue.size",
        tags: [:queue],
        description: "Number of jobs in each Exq queue"
      ),
      last_value("towerops.exq.processes.busy",
        description: "Number of busy Exq worker processes"
      ),
      last_value("towerops.exq.processes.total",
        description: "Total number of Exq worker processes"
      ),
      last_value("towerops.redis.connected_clients",
        description: "Number of Redis/Valkey connected clients"
      ),
      last_value("towerops.redis.used_memory",
        unit: {:byte, :megabyte},
        description: "Redis/Valkey memory usage"
      ),
      last_value("towerops.redis.commands_processed",
        description: "Total commands processed by Redis/Valkey"
      )
    ]
  end

  defp periodic_measurements do
    [
      # Measure Exq and Redis stats every 10 seconds
      {__MODULE__, :publish_exq_stats, []},
      {__MODULE__, :publish_redis_stats, []}
    ]
  end

  @doc """
  Publishes Exq queue and process statistics.
  """
  def publish_exq_stats do
    # Only run if Exq is available (not in test env)
    if Application.get_env(:towerops, :env) == :test do
      :ok
      # Get stats for each queue

      # Get process stats
    else
      try do
        queues = ["default", "discovery", "polling", "monitoring", "maintenance"]

        for queue <- queues do
          {:ok, size} = Exq.Api.queue_size(Exq, queue)

          :telemetry.execute(
            [:towerops, :exq, :queue, :size],
            %{value: size},
            %{queue: queue}
          )
        end

        {:ok, processes} = Exq.Api.processes(Exq)
        {:ok, busy} = Exq.Api.busy(Exq)

        :telemetry.execute(
          [:towerops, :exq, :processes, :busy],
          %{value: length(busy)},
          %{}
        )

        :telemetry.execute(
          [:towerops, :exq, :processes, :total],
          %{value: length(processes)},
          %{}
        )

        :ok
      rescue
        _ -> :ok
      end
    end
  end

  @doc """
  Publishes Redis/Valkey statistics.
  """
  def publish_redis_stats do
    # Only run if Redis is configured (not in test env)
    if Application.get_env(:towerops, :env) == :test do
      :ok

      # Connect to Redis and get INFO

      # Parse INFO response

      # Publish metrics
    else
      try do
        redis_config = Application.get_env(:towerops, :redis, [])
        host = Keyword.get(redis_config, :host, "localhost")
        port = Keyword.get(redis_config, :port, 6379)
        {:ok, conn} = Redix.start_link(host: host, port: port)
        {:ok, info} = Redix.command(conn, ["INFO", "stats"])
        {:ok, memory_info} = Redix.command(conn, ["INFO", "memory"])
        {:ok, clients_info} = Redix.command(conn, ["INFO", "clients"])
        stats = parse_redis_info(info)
        memory_stats = parse_redis_info(memory_info)
        client_stats = parse_redis_info(clients_info)

        if total_commands = stats["total_commands_processed"] do
          :telemetry.execute(
            [:towerops, :redis, :commands_processed],
            %{value: String.to_integer(total_commands)},
            %{}
          )
        end

        if used_memory = memory_stats["used_memory"] do
          :telemetry.execute(
            [:towerops, :redis, :used_memory],
            %{value: String.to_integer(used_memory)},
            %{}
          )
        end

        if connected_clients = client_stats["connected_clients"] do
          :telemetry.execute(
            [:towerops, :redis, :connected_clients],
            %{value: String.to_integer(connected_clients)},
            %{}
          )
        end

        Redix.stop(conn)
        :ok
      rescue
        _ -> :ok
      end
    end
  end

  @doc """
  Parse Redis INFO command output into a map.
  """
  def parse_redis_info(info_string) do
    info_string
    |> String.split(~r/\r?\n/)
    |> Enum.map(&String.trim/1)
    |> Enum.reject(&(String.starts_with?(&1, "#") or &1 == ""))
    |> Enum.map(&String.split(&1, ":", parts: 2))
    |> Enum.filter(&(length(&1) == 2))
    |> Map.new(fn [key, value] -> {key, value} end)
  end

  # Telemetry handler for router exceptions
  def handle_router_exception(_event, _measurements, metadata, _config) do
    require Logger

    Logger.error(
      "Router exception on #{metadata.plug} #{metadata.conn.method} #{metadata.conn.request_path}",
      kind: metadata.kind,
      reason: metadata.reason,
      stacktrace: metadata.stacktrace,
      request_id: metadata.conn.assigns[:request_id]
    )
  end

  # Telemetry handler for endpoint stop events (log slow requests and errors)
  def handle_endpoint_stop(_event, measurements, metadata, _config) do
    require Logger

    duration_ms = System.convert_time_unit(measurements.duration, :native, :millisecond)

    # Log slow requests (over 5 seconds)
    if duration_ms > 5_000 do
      Logger.warning(
        "Slow request: #{metadata.conn.method} #{metadata.conn.request_path} took #{duration_ms}ms",
        request_id: metadata.conn.assigns[:request_id],
        duration_ms: duration_ms
      )
    end

    # Log requests with non-2xx status codes
    status = metadata.conn.status

    if status >= 500 do
      Logger.error(
        "Server error: #{metadata.conn.method} #{metadata.conn.request_path} returned #{status}",
        request_id: metadata.conn.assigns[:request_id],
        status: status,
        duration_ms: duration_ms
      )
    end
  end
end