Graham McIntire
70e65ba034
- Remove the 7-day outlook strip and daily_outlook_at/3. It was derived from point_forecast, which is capped at HRRR's 18-hour horizon, so it never had 7 days of data to show. - factors_for: fall back to the nearest persisted analysis profile at or before the requested time. Only f00 hours persist profiles, so clicking any other hour used to yield an empty Analysis and factor table. After the recent cursor fix lands 'Now' on a forecast hour more often, this regressed further. - available_valid_times/1: cap the forward horizon to 18h from now. Leftover valid_times from prior cycles used to pile on the timeline without adding information.
350 lines
14 KiB
Elixir
350 lines
14 KiB
Elixir
defmodule Microwaveprop.PromEx.InstrumentPlugin do
|
|
@moduledoc """
|
|
PromEx plugin that registers every `Microwaveprop.Instrument` span as
|
|
a Prometheus histogram. Keep the catalog in sync with the call sites
|
|
in `HrrrClient`, `NexradClient`, `IemClient`, `GefsClient`,
|
|
`NarrClient`, `UwyoSoundingClient`, `ElevationClient`, `Weather` batch
|
|
upserts, `Propagation.replace_scores`, `PropagationGridWorker`,
|
|
`TerrainAnalysis`, and `CommonVolumeRadarWorker`.
|
|
"""
|
|
use PromEx.Plugin
|
|
|
|
@impl true
|
|
def event_metrics(_opts) do
|
|
[
|
|
http_client_events(),
|
|
more_http_client_events(),
|
|
subprocess_events(),
|
|
db_batch_events(),
|
|
worker_phase_events(),
|
|
top_level_worker_events(),
|
|
liveview_hot_path_events(),
|
|
cache_hit_events(),
|
|
oban_queue_depth_event()
|
|
]
|
|
end
|
|
|
|
# PromEx interprets `Telemetry.Metrics` specs the same way the
|
|
# in-process telemetry_metrics module does, so we re-use the same
|
|
# event_name / measurement / tags shape from MicrowavepropWeb.Telemetry.
|
|
defp http_client_events do
|
|
Event.build(
|
|
:microwaveprop_http_client_events,
|
|
[
|
|
distribution("microwaveprop.hrrr.fetch_grid.duration.milliseconds",
|
|
event_name: [:microwaveprop, :hrrr, :fetch_grid, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.hrrr.fetch_profile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :hrrr, :fetch_profile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.hrrr.fetch_idx.duration.milliseconds",
|
|
event_name: [:microwaveprop, :hrrr, :fetch_idx, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.nexrad.fetch_frame.duration.milliseconds",
|
|
event_name: [:microwaveprop, :nexrad, :fetch_frame, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.nexrad.decode_png.duration.milliseconds",
|
|
event_name: [:microwaveprop, :nexrad, :decode_png, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.iem.fetch_asos.duration.milliseconds",
|
|
event_name: [:microwaveprop, :iem, :fetch_asos, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.iem.fetch_raob.duration.milliseconds",
|
|
event_name: [:microwaveprop, :iem, :fetch_raob, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.gefs.fetch_grid_profiles.duration.milliseconds",
|
|
event_name: [:microwaveprop, :gefs, :fetch_grid_profiles, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.narr.fetch_profile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :narr, :fetch_profile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.uwyo.fetch_sounding.duration.milliseconds",
|
|
event_name: [:microwaveprop, :uwyo, :fetch_sounding, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.elevation.fetch_profile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :elevation, :fetch_profile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp more_http_client_events do
|
|
Event.build(
|
|
:microwaveprop_more_http_client_events,
|
|
[
|
|
distribution("microwaveprop.iem.fetch_iemre.duration.milliseconds",
|
|
event_name: [:microwaveprop, :iem, :fetch_iemre, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.mrms.list_latest.duration.milliseconds",
|
|
event_name: [:microwaveprop, :mrms, :list_latest, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.mrms.download.duration.milliseconds",
|
|
event_name: [:microwaveprop, :mrms, :download, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.rtma.fetch_observation.duration.milliseconds",
|
|
event_name: [:microwaveprop, :rtma, :fetch_observation, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.ncei.fetch_metar.duration.milliseconds",
|
|
event_name: [:microwaveprop, :ncei, :fetch_metar, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.solar.fetch_indices.duration.milliseconds",
|
|
event_name: [:microwaveprop, :solar, :fetch_indices, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.swpc.fetch.duration.milliseconds",
|
|
event_name: [:microwaveprop, :swpc, :fetch, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.giro.fetch.duration.milliseconds",
|
|
event_name: [:microwaveprop, :giro, :fetch, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.qrz.request.duration.milliseconds",
|
|
event_name: [:microwaveprop, :qrz, :request, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.geocoder.geocode.duration.milliseconds",
|
|
event_name: [:microwaveprop, :geocoder, :geocode, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.srtm.download_tile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :srtm, :download_tile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.hrrr.download_grib_ranges.duration.milliseconds",
|
|
event_name: [:microwaveprop, :hrrr, :download_grib_ranges, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp subprocess_events do
|
|
Event.build(
|
|
:microwaveprop_subprocess_events,
|
|
[
|
|
distribution("microwaveprop.wgrib2.extract_grid.duration.milliseconds",
|
|
event_name: [:microwaveprop, :wgrib2, :extract_grid, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
),
|
|
distribution("microwaveprop.wgrib2.extract_grid_from_file.duration.milliseconds",
|
|
event_name: [:microwaveprop, :wgrib2, :extract_grid_from_file, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
),
|
|
distribution("microwaveprop.wgrib2.extract_grid_from_file_mapped.duration.milliseconds",
|
|
event_name: [:microwaveprop, :wgrib2, :extract_grid_from_file_mapped, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp top_level_worker_events do
|
|
Event.build(
|
|
:microwaveprop_top_level_worker_events,
|
|
[
|
|
distribution("microwaveprop.worker.terrain_profile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :worker, :terrain_profile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.worker.mechanism_classify.duration.milliseconds",
|
|
event_name: [:microwaveprop, :worker, :mechanism_classify, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.worker.mrms_fetch.duration.milliseconds",
|
|
event_name: [:microwaveprop, :worker, :mrms_fetch, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp liveview_hot_path_events do
|
|
Event.build(
|
|
:microwaveprop_liveview_hot_path_events,
|
|
[
|
|
distribution("microwaveprop.propagation.scores_at.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation, :scores_at, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.propagation.point_forecast.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation, :point_forecast, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.propagation.point_detail.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation, :point_detail, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp cache_hit_events do
|
|
Event.build(
|
|
:microwaveprop_cache_hit_events,
|
|
[
|
|
counter("microwaveprop.propagation.scores_at.cache.count",
|
|
event_name: [:microwaveprop, :propagation, :scores_at, :cache],
|
|
tags: [:hit],
|
|
description: "ScoreCache hit/miss for the map's scores_at fetch"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp db_batch_events do
|
|
Event.build(
|
|
:microwaveprop_db_batch_events,
|
|
[
|
|
distribution("microwaveprop.db.upsert_hrrr_profiles.duration.milliseconds",
|
|
event_name: [:microwaveprop, :db, :upsert_hrrr_profiles, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: db_buckets()]
|
|
),
|
|
distribution("microwaveprop.db.upsert_gefs_profiles.duration.milliseconds",
|
|
event_name: [:microwaveprop, :db, :upsert_gefs_profiles, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: db_buckets()]
|
|
),
|
|
distribution("microwaveprop.db.replace_scores.duration.milliseconds",
|
|
event_name: [:microwaveprop, :db, :replace_scores, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: db_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp worker_phase_events do
|
|
Event.build(
|
|
:microwaveprop_worker_phase_events,
|
|
[
|
|
distribution("microwaveprop.propagation_grid.score_band.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation_grid, :score_band, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
),
|
|
distribution("microwaveprop.terrain.analyse.duration.milliseconds",
|
|
event_name: [:microwaveprop, :terrain, :analyse, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.radar.aggregate_stats.duration.milliseconds",
|
|
event_name: [:microwaveprop, :radar, :aggregate_stats, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp oban_queue_depth_event do
|
|
Event.build(
|
|
:microwaveprop_oban_queue_depth,
|
|
[
|
|
last_value("microwaveprop.oban.queue.count",
|
|
event_name: [:microwaveprop, :oban, :queue, :depth],
|
|
measurement: :count,
|
|
tags: [:queue, :state],
|
|
description: "Oban job count per (queue, state) sampled every 10s"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
# Buckets in milliseconds. HTTP + compute phases: up to ~30s.
|
|
defp default_buckets, do: [10, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000]
|
|
|
|
# DB batch upserts: can take much longer on cold cache or full
|
|
# forecast-hour writes — up to ~2 min.
|
|
defp db_buckets, do: [50, 100, 250, 500, 1_000, 2_500, 5_000, 15_000, 60_000, 120_000]
|
|
|
|
# Score computation for a full grid-point batch — up to ~5 min.
|
|
defp long_buckets, do: [500, 1_000, 5_000, 15_000, 30_000, 60_000, 120_000, 300_000]
|
|
end
|