prop/lib/microwaveprop_web/telemetry.ex

defmodule MicrowavepropWeb.Telemetry do
  @moduledoc false
  use Supervisor

  import Telemetry.Metrics

  @spec start_link(any()) :: Supervisor.on_start()
  def start_link(arg) do
    Supervisor.start_link(__MODULE__, arg, name: __MODULE__)
  end

  @impl true
  def init(_arg) do
    # Oban queue-depth is the only poller measurement and each replica
    # issues the same GROUP-BY over oban_jobs. 30s gives enough fidelity
    # for dashboard trends without burning ~3/min of identical queries
    # per extra replica. Skipped in test (no sandbox owner → connection
    # contention with LiveView async tests; identical pattern to the
    # PromEx Oban plugin we already disable via
    # `:prom_ex_include_oban_plugin`).
    children =
      if Application.get_env(:microwaveprop, :start_telemetry_poller, true) do
        [{:telemetry_poller, measurements: periodic_measurements(), period: 30_000}]
      else
        []
      end

    Supervisor.init(children, strategy: :one_for_one)
  end

  @spec metrics :: [Telemetry.Metrics.t()]
  def metrics do
    phoenix_metrics() ++
      ecto_metrics() ++
      vm_metrics() ++
      oban_metrics() ++
      http_client_metrics() ++
      db_batch_metrics() ++
      worker_phase_metrics()
  end

  defp phoenix_metrics do
    [
      summary("phoenix.endpoint.stop.duration", unit: {:native, :millisecond}),
      summary("phoenix.router_dispatch.start.system_time",
        tags: [:route],
        unit: {:native, :millisecond}
      ),
      summary("phoenix.router_dispatch.exception.duration",
        tags: [:route],
        unit: {:native, :millisecond}
      ),
      summary("phoenix.router_dispatch.stop.duration",
        tags: [:route],
        unit: {:native, :millisecond}
      ),
      summary("phoenix.socket_connected.duration", unit: {:native, :millisecond}),
      sum("phoenix.socket_drain.count"),
      summary("phoenix.channel_joined.duration", unit: {:native, :millisecond}),
      summary("phoenix.channel_handled_in.duration",
        tags: [:event],
        unit: {:native, :millisecond}
      )
    ]
  end

  defp ecto_metrics do
    [
      summary("microwaveprop.repo.query.total_time",
        unit: {:native, :millisecond},
        description: "The sum of the other measurements"
      ),
      summary("microwaveprop.repo.query.decode_time",
        unit: {:native, :millisecond},
        description: "The time spent decoding the data received from the database"
      ),
      summary("microwaveprop.repo.query.query_time",
        unit: {:native, :millisecond},
        description: "The time spent executing the query"
      ),
      summary("microwaveprop.repo.query.queue_time",
        unit: {:native, :millisecond},
        description: "The time spent waiting for a database connection"
      ),
      summary("microwaveprop.repo.query.idle_time",
        unit: {:native, :millisecond},
        description: "The time the connection spent waiting before being checked out for the query"
      )
    ]
  end

  defp vm_metrics do
    [
      summary("vm.memory.total", unit: {:byte, :kilobyte}),
      summary("vm.total_run_queue_lengths.total"),
      summary("vm.total_run_queue_lengths.cpu"),
      summary("vm.total_run_queue_lengths.io")
    ]
  end

  # Oban emits [:oban, :job, :start|:stop|:exception]. `stop` measurements
  # include both `duration` (job runtime) and `queue_time` (available →
  # executing latency). Tagging by worker + queue gives us per-worker
  # distributions without per-worker metric registration.
  defp oban_metrics do
    [
      summary("oban.job.stop.duration",
        event_name: [:oban, :job, :stop],
        measurement: :duration,
        tags: [:worker, :queue, :state],
        tag_values: &oban_tag_values/1,
        unit: {:native, :millisecond}
      ),
      summary("oban.job.stop.queue_time",
        event_name: [:oban, :job, :stop],
        measurement: :queue_time,
        tags: [:worker, :queue],
        tag_values: &oban_tag_values/1,
        unit: {:native, :millisecond}
      ),
      counter("oban.job.stop.count",
        event_name: [:oban, :job, :stop],
        tags: [:worker, :queue, :state],
        tag_values: &oban_tag_values/1
      ),
      counter("oban.job.exception.count",
        event_name: [:oban, :job, :exception],
        tags: [:worker, :queue],
        tag_values: &oban_tag_values/1
      ),
      last_value("microwaveprop.oban.queue.count",
        event_name: [:microwaveprop, :oban, :queue, :depth],
        measurement: :count,
        tags: [:queue, :state],
        description: "Oban job count per (queue, state) sampled every 30s"
      )
    ]
  end

  # HTTP client spans emitted via `Microwaveprop.Instrument`. Each client
  # wraps its own public entry points so we get independent latency
  # histograms per upstream.
  defp http_client_metrics do
    [
      # HRRR pressure-level fetch (batch-of-points via NOMADS/S3 + wgrib2)
      summary("microwaveprop.hrrr.fetch_grid.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),
      summary("microwaveprop.hrrr.fetch_profile.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),
      summary("microwaveprop.hrrr.fetch_idx.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),

      # NEXRAD n0q frame (IEM archive PNG)
      summary("microwaveprop.nexrad.fetch_frame.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),
      summary("microwaveprop.nexrad.decode_png.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),

      # IEM Mesonet ASOS + RAOB
      summary("microwaveprop.iem.fetch_asos.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),
      summary("microwaveprop.iem.fetch_raob.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),

      # GEFS extended-horizon ensemble
      summary("microwaveprop.gefs.fetch_grid_profiles.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),

      # NARR historical backfill
      summary("microwaveprop.narr.fetch_profile.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),

      # University of Wyoming soundings (Canadian RAOB backfill)
      summary("microwaveprop.uwyo.fetch_sounding.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),

      # Terrain elevation API (Open-Meteo or local SRTM)
      summary("microwaveprop.elevation.fetch_profile.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      )
    ]
  end

  defp db_batch_metrics do
    [
      summary("microwaveprop.db.upsert_hrrr_profiles.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),
      summary("microwaveprop.db.upsert_gefs_profiles.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),
      summary("microwaveprop.db.upsert_surface_observations.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),
      summary("microwaveprop.db.upsert_sounding.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),
      summary("microwaveprop.db.replace_scores.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      )
    ]
  end

  defp worker_phase_metrics do
    [
      summary("microwaveprop.propagation_grid.score_band.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),
      summary("microwaveprop.terrain.analyse.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      ),
      summary("microwaveprop.radar.aggregate_stats.stop.duration",
        unit: {:native, :millisecond},
        tags: [:result]
      )
    ]
  end

  defp oban_tag_values(%{job: %Oban.Job{worker: worker, queue: queue}} = meta) do
    %{
      worker: worker || "unknown",
      queue: queue || "unknown",
      state: Map.get(meta, :state, :unknown)
    }
  end

  defp oban_tag_values(_), do: %{worker: "unknown", queue: "unknown", state: :unknown}

  defp periodic_measurements do
    [
      {__MODULE__, :dispatch_oban_queue_depth, []}
    ]
  end

  @doc false
  # Periodic poller: emit `microwaveprop.oban.queue.depth` gauges per
  # (queue, state) tuple so LiveDashboard / Prometheus can chart queue
  # saturation over time. Cheap aggregation query, runs every 30s.
  @spec dispatch_oban_queue_depth :: :ok
  def dispatch_oban_queue_depth do
    import Ecto.Query

    query =
      from(j in "oban_jobs",
        group_by: [j.queue, j.state],
        select: {j.queue, j.state, count(j.id)}
      )

    for {queue, state, count} <- Microwaveprop.Repo.all(query) do
      :telemetry.execute(
        [:microwaveprop, :oban, :queue, :depth],
        %{count: count},
        %{queue: queue, state: state}
      )
    end

    :ok
  rescue
    # Don't crash the poller if the DB blips — telemetry is best-effort.
    e ->
      require Logger

      Logger.warning("Oban queue depth poll failed: #{inspect(e)}")
      :ok
  end
end