defmodule MicrowavepropWeb.Telemetry do @moduledoc false use Supervisor import Telemetry.Metrics def start_link(arg) do Supervisor.start_link(__MODULE__, arg, name: __MODULE__) end @impl true def init(_arg) do # Oban queue-depth is the only poller measurement and each replica # issues the same GROUP-BY over oban_jobs. 30s gives enough fidelity # for dashboard trends without burning ~3/min of identical queries # per extra replica. Skipped in test (no sandbox owner → connection # contention with LiveView async tests; identical pattern to the # PromEx Oban plugin we already disable via # `:prom_ex_include_oban_plugin`). children = if Application.get_env(:microwaveprop, :start_telemetry_poller, true) do [{:telemetry_poller, measurements: periodic_measurements(), period: 30_000}] else [] end Supervisor.init(children, strategy: :one_for_one) end def metrics do phoenix_metrics() ++ ecto_metrics() ++ vm_metrics() ++ oban_metrics() ++ http_client_metrics() ++ db_batch_metrics() ++ worker_phase_metrics() end defp phoenix_metrics do [ summary("phoenix.endpoint.stop.duration", unit: {:native, :millisecond}), summary("phoenix.router_dispatch.start.system_time", tags: [:route], unit: {:native, :millisecond} ), summary("phoenix.router_dispatch.exception.duration", tags: [:route], unit: {:native, :millisecond} ), summary("phoenix.router_dispatch.stop.duration", tags: [:route], unit: {:native, :millisecond} ), summary("phoenix.socket_connected.duration", unit: {:native, :millisecond}), sum("phoenix.socket_drain.count"), summary("phoenix.channel_joined.duration", unit: {:native, :millisecond}), summary("phoenix.channel_handled_in.duration", tags: [:event], unit: {:native, :millisecond} ) ] end defp ecto_metrics do [ summary("microwaveprop.repo.query.total_time", unit: {:native, :millisecond}, description: "The sum of the other measurements" ), summary("microwaveprop.repo.query.decode_time", unit: {:native, :millisecond}, description: "The time spent decoding the data received from the database" ), summary("microwaveprop.repo.query.query_time", unit: {:native, :millisecond}, description: "The time spent executing the query" ), summary("microwaveprop.repo.query.queue_time", unit: {:native, :millisecond}, description: "The time spent waiting for a database connection" ), summary("microwaveprop.repo.query.idle_time", unit: {:native, :millisecond}, description: "The time the connection spent waiting before being checked out for the query" ) ] end defp vm_metrics do [ summary("vm.memory.total", unit: {:byte, :kilobyte}), summary("vm.total_run_queue_lengths.total"), summary("vm.total_run_queue_lengths.cpu"), summary("vm.total_run_queue_lengths.io") ] end # Oban emits [:oban, :job, :start|:stop|:exception]. `stop` measurements # include both `duration` (job runtime) and `queue_time` (available → # executing latency). Tagging by worker + queue gives us per-worker # distributions without per-worker metric registration. defp oban_metrics do [ summary("oban.job.stop.duration", event_name: [:oban, :job, :stop], measurement: :duration, tags: [:worker, :queue, :state], tag_values: &oban_tag_values/1, unit: {:native, :millisecond} ), summary("oban.job.stop.queue_time", event_name: [:oban, :job, :stop], measurement: :queue_time, tags: [:worker, :queue], tag_values: &oban_tag_values/1, unit: {:native, :millisecond} ), counter("oban.job.stop.count", event_name: [:oban, :job, :stop], tags: [:worker, :queue, :state], tag_values: &oban_tag_values/1 ), counter("oban.job.exception.count", event_name: [:oban, :job, :exception], tags: [:worker, :queue], tag_values: &oban_tag_values/1 ), last_value("microwaveprop.oban.queue.count", event_name: [:microwaveprop, :oban, :queue, :depth], measurement: :count, tags: [:queue, :state], description: "Oban job count per (queue, state) sampled every 30s" ) ] end # HTTP client spans emitted via `Microwaveprop.Instrument`. Each client # wraps its own public entry points so we get independent latency # histograms per upstream. defp http_client_metrics do [ # HRRR pressure-level fetch (batch-of-points via NOMADS/S3 + wgrib2) summary("microwaveprop.hrrr.fetch_grid.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), summary("microwaveprop.hrrr.fetch_profile.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), summary("microwaveprop.hrrr.fetch_idx.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), # NEXRAD n0q frame (IEM archive PNG) summary("microwaveprop.nexrad.fetch_frame.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), summary("microwaveprop.nexrad.decode_png.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), # IEM Mesonet ASOS + RAOB summary("microwaveprop.iem.fetch_asos.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), summary("microwaveprop.iem.fetch_raob.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), # GEFS extended-horizon ensemble summary("microwaveprop.gefs.fetch_grid_profiles.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), # NARR historical backfill summary("microwaveprop.narr.fetch_profile.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), # University of Wyoming soundings (Canadian RAOB backfill) summary("microwaveprop.uwyo.fetch_sounding.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), # Terrain elevation API (Open-Meteo or local SRTM) summary("microwaveprop.elevation.fetch_profile.stop.duration", unit: {:native, :millisecond}, tags: [:result] ) ] end defp db_batch_metrics do [ summary("microwaveprop.db.upsert_hrrr_profiles.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), summary("microwaveprop.db.upsert_gefs_profiles.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), summary("microwaveprop.db.upsert_surface_observations.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), summary("microwaveprop.db.upsert_sounding.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), summary("microwaveprop.db.replace_scores.stop.duration", unit: {:native, :millisecond}, tags: [:result] ) ] end defp worker_phase_metrics do [ summary("microwaveprop.propagation_grid.score_band.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), summary("microwaveprop.terrain.analyse.stop.duration", unit: {:native, :millisecond}, tags: [:result] ), summary("microwaveprop.radar.aggregate_stats.stop.duration", unit: {:native, :millisecond}, tags: [:result] ) ] end defp oban_tag_values(%{job: %Oban.Job{worker: worker, queue: queue}} = meta) do %{ worker: worker || "unknown", queue: queue || "unknown", state: Map.get(meta, :state, :unknown) } end defp oban_tag_values(_), do: %{worker: "unknown", queue: "unknown", state: :unknown} defp periodic_measurements do [ {__MODULE__, :dispatch_oban_queue_depth, []} ] end @doc false # Periodic poller: emit `microwaveprop.oban.queue.depth` gauges per # (queue, state) tuple so LiveDashboard / Prometheus can chart queue # saturation over time. Cheap aggregation query, runs every 30s. def dispatch_oban_queue_depth do import Ecto.Query query = from(j in "oban_jobs", group_by: [j.queue, j.state], select: {j.queue, j.state, count(j.id)} ) for {queue, state, count} <- Microwaveprop.Repo.all(query) do :telemetry.execute( [:microwaveprop, :oban, :queue, :depth], %{count: count}, %{queue: queue, state: state} ) end :ok rescue # Don't crash the poller if the DB blips — telemetry is best-effort. _ -> :ok end end