defmodule Microwaveprop.PromEx.InstrumentPlugin do @moduledoc """ PromEx plugin that registers every `Microwaveprop.Instrument` span as a Prometheus histogram. Keep the catalog in sync with the call sites in `HrrrClient`, `NexradClient`, `IemClient`, `GefsClient`, `NarrClient`, `UwyoSoundingClient`, `ElevationClient`, `Weather` batch upserts, `Propagation.replace_scores`, `PropagationGridWorker`, `TerrainAnalysis`, and `CommonVolumeRadarWorker`. """ use PromEx.Plugin @impl true def event_metrics(_opts) do [ http_client_events(), more_http_client_events(), subprocess_events(), db_batch_events(), worker_phase_events(), top_level_worker_events(), liveview_hot_path_events(), cache_hit_events(), oban_queue_depth_event() ] end # PromEx interprets `Telemetry.Metrics` specs the same way the # in-process telemetry_metrics module does, so we re-use the same # event_name / measurement / tags shape from MicrowavepropWeb.Telemetry. defp http_client_events do Event.build( :microwaveprop_http_client_events, [ distribution("microwaveprop.hrrr.fetch_grid.duration.milliseconds", event_name: [:microwaveprop, :hrrr, :fetch_grid, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.hrrr.fetch_profile.duration.milliseconds", event_name: [:microwaveprop, :hrrr, :fetch_profile, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.hrrr.fetch_idx.duration.milliseconds", event_name: [:microwaveprop, :hrrr, :fetch_idx, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.nexrad.fetch_frame.duration.milliseconds", event_name: [:microwaveprop, :nexrad, :fetch_frame, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.nexrad.decode_png.duration.milliseconds", event_name: [:microwaveprop, :nexrad, :decode_png, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.iem.fetch_asos.duration.milliseconds", event_name: [:microwaveprop, :iem, :fetch_asos, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.iem.fetch_raob.duration.milliseconds", event_name: [:microwaveprop, :iem, :fetch_raob, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.gefs.fetch_grid_profiles.duration.milliseconds", event_name: [:microwaveprop, :gefs, :fetch_grid_profiles, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.narr.fetch_profile.duration.milliseconds", event_name: [:microwaveprop, :narr, :fetch_profile, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.uwyo.fetch_sounding.duration.milliseconds", event_name: [:microwaveprop, :uwyo, :fetch_sounding, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.elevation.fetch_profile.duration.milliseconds", event_name: [:microwaveprop, :elevation, :fetch_profile, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ) ] ) end defp more_http_client_events do Event.build( :microwaveprop_more_http_client_events, [ distribution("microwaveprop.iem.fetch_iemre.duration.milliseconds", event_name: [:microwaveprop, :iem, :fetch_iemre, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.mrms.list_latest.duration.milliseconds", event_name: [:microwaveprop, :mrms, :list_latest, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.mrms.download.duration.milliseconds", event_name: [:microwaveprop, :mrms, :download, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.rtma.fetch_observation.duration.milliseconds", event_name: [:microwaveprop, :rtma, :fetch_observation, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.ncei.fetch_metar.duration.milliseconds", event_name: [:microwaveprop, :ncei, :fetch_metar, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.solar.fetch_indices.duration.milliseconds", event_name: [:microwaveprop, :solar, :fetch_indices, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.swpc.fetch.duration.milliseconds", event_name: [:microwaveprop, :swpc, :fetch, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.giro.fetch.duration.milliseconds", event_name: [:microwaveprop, :giro, :fetch, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.qrz.request.duration.milliseconds", event_name: [:microwaveprop, :qrz, :request, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.geocoder.geocode.duration.milliseconds", event_name: [:microwaveprop, :geocoder, :geocode, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.srtm.download_tile.duration.milliseconds", event_name: [:microwaveprop, :srtm, :download_tile, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.hrrr.download_grib_ranges.duration.milliseconds", event_name: [:microwaveprop, :hrrr, :download_grib_ranges, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ) ] ) end defp subprocess_events do Event.build( :microwaveprop_subprocess_events, [ distribution("microwaveprop.wgrib2.extract_grid.duration.milliseconds", event_name: [:microwaveprop, :wgrib2, :extract_grid, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: long_buckets()] ), distribution("microwaveprop.wgrib2.extract_grid_from_file.duration.milliseconds", event_name: [:microwaveprop, :wgrib2, :extract_grid_from_file, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: long_buckets()] ), distribution("microwaveprop.wgrib2.extract_grid_from_file_mapped.duration.milliseconds", event_name: [:microwaveprop, :wgrib2, :extract_grid_from_file_mapped, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: long_buckets()] ) ] ) end defp top_level_worker_events do Event.build( :microwaveprop_top_level_worker_events, [ distribution("microwaveprop.worker.terrain_profile.duration.milliseconds", event_name: [:microwaveprop, :worker, :terrain_profile, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.worker.mechanism_classify.duration.milliseconds", event_name: [:microwaveprop, :worker, :mechanism_classify, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.worker.mrms_fetch.duration.milliseconds", event_name: [:microwaveprop, :worker, :mrms_fetch, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: long_buckets()] ), distribution("microwaveprop.propagation.grid_worker.perform.duration.milliseconds", event_name: [:microwaveprop, :propagation, :grid_worker, :perform, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), last_value("microwaveprop.propagation.grid_worker.seeded.queued_tasks", event_name: [:microwaveprop, :propagation, :grid_worker, :seeded], measurement: :queued_tasks, description: "grid_tasks rows inserted by the hourly seed worker" ), counter("microwaveprop.propagation.grid_worker.exception.count", event_name: [:microwaveprop, :propagation, :grid_worker, :exception], description: "PropagationGridWorker seed failures" ) ] ) end defp liveview_hot_path_events do Event.build( :microwaveprop_liveview_hot_path_events, [ distribution("microwaveprop.propagation.scores_at.duration.milliseconds", event_name: [:microwaveprop, :propagation, :scores_at, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.propagation.point_forecast.duration.milliseconds", event_name: [:microwaveprop, :propagation, :point_forecast, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.propagation.point_detail.duration.milliseconds", event_name: [:microwaveprop, :propagation, :point_detail, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ) ] ) end defp cache_hit_events do Event.build( :microwaveprop_cache_hit_events, [ counter("microwaveprop.propagation.scores_at.cache.count", event_name: [:microwaveprop, :propagation, :scores_at, :cache], tags: [:hit], description: "ScoreCache hit/miss for the map's scores_at fetch" ), counter("microwaveprop.weather.grid_cache.lookup.count", event_name: [:microwaveprop, :weather, :grid_cache, :lookup], tags: [:hit], description: "GridCache hit/miss for the /weather map's grid fetch" ) ] ) end defp db_batch_events do Event.build( :microwaveprop_db_batch_events, [ distribution("microwaveprop.db.upsert_hrrr_profiles.duration.milliseconds", event_name: [:microwaveprop, :db, :upsert_hrrr_profiles, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: db_buckets()] ), distribution("microwaveprop.db.upsert_gefs_profiles.duration.milliseconds", event_name: [:microwaveprop, :db, :upsert_gefs_profiles, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: db_buckets()] ), distribution("microwaveprop.db.replace_scores.duration.milliseconds", event_name: [:microwaveprop, :db, :replace_scores, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: db_buckets()] ) ] ) end defp worker_phase_events do Event.build( :microwaveprop_worker_phase_events, [ distribution("microwaveprop.propagation_grid.score_band.duration.milliseconds", event_name: [:microwaveprop, :propagation_grid, :score_band, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: long_buckets()] ), distribution("microwaveprop.terrain.analyse.duration.milliseconds", event_name: [:microwaveprop, :terrain, :analyse, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ), distribution("microwaveprop.radar.aggregate_stats.duration.milliseconds", event_name: [:microwaveprop, :radar, :aggregate_stats, :stop], measurement: :duration, unit: {:native, :millisecond}, reporter_options: [buckets: default_buckets()] ) ] ) end defp oban_queue_depth_event do Event.build( :microwaveprop_oban_queue_depth, [ last_value("microwaveprop.oban.queue.count", event_name: [:microwaveprop, :oban, :queue, :depth], measurement: :count, tags: [:queue, :state], description: "Oban job count per (queue, state) sampled every 30s" ) ] ) end # Buckets in milliseconds. HTTP + compute phases: up to ~30s. defp default_buckets, do: [10, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000] # DB batch upserts: can take much longer on cold cache or full # forecast-hour writes — up to ~2 min. defp db_buckets, do: [50, 100, 250, 500, 1_000, 2_500, 5_000, 15_000, 60_000, 120_000] # Score computation for a full grid-point batch — up to ~5 min. defp long_buckets, do: [500, 1_000, 5_000, 15_000, 30_000, 60_000, 120_000, 300_000] end