Graham McIntire
9bf96349ad
Three small wins from a telemetry audit: 1. Propagation.scores_at/3 wrapped every call in Instrument.span, firing two handler dispatches that dominated the ~10µs ETS lookup on cache hits. The map's LiveView fires this on every pan + point-click, so hot-path latency was mostly telemetry. Span now wraps only the miss branch (where disk IO makes the duration signal meaningful); hit path still emits the cheap hit/miss counter the cache-ratio panel reads. 2. Oban queue-depth poller: 10s → 30s. Every replica independently GROUP-BYs oban_jobs, so each extra replica paid ~3 redundant queries per minute for a gauge that moves on the hour-scale anyway. 3. Dropped summary(phoenix.endpoint.start.system_time): summarizing a wall-clock timestamp produces no useful aggregate — stop.duration below it is the meaningful signal.
350 lines
14 KiB
Elixir
350 lines
14 KiB
Elixir
defmodule Microwaveprop.PromEx.InstrumentPlugin do
|
|
@moduledoc """
|
|
PromEx plugin that registers every `Microwaveprop.Instrument` span as
|
|
a Prometheus histogram. Keep the catalog in sync with the call sites
|
|
in `HrrrClient`, `NexradClient`, `IemClient`, `GefsClient`,
|
|
`NarrClient`, `UwyoSoundingClient`, `ElevationClient`, `Weather` batch
|
|
upserts, `Propagation.replace_scores`, `PropagationGridWorker`,
|
|
`TerrainAnalysis`, and `CommonVolumeRadarWorker`.
|
|
"""
|
|
use PromEx.Plugin
|
|
|
|
@impl true
|
|
def event_metrics(_opts) do
|
|
[
|
|
http_client_events(),
|
|
more_http_client_events(),
|
|
subprocess_events(),
|
|
db_batch_events(),
|
|
worker_phase_events(),
|
|
top_level_worker_events(),
|
|
liveview_hot_path_events(),
|
|
cache_hit_events(),
|
|
oban_queue_depth_event()
|
|
]
|
|
end
|
|
|
|
# PromEx interprets `Telemetry.Metrics` specs the same way the
|
|
# in-process telemetry_metrics module does, so we re-use the same
|
|
# event_name / measurement / tags shape from MicrowavepropWeb.Telemetry.
|
|
defp http_client_events do
|
|
Event.build(
|
|
:microwaveprop_http_client_events,
|
|
[
|
|
distribution("microwaveprop.hrrr.fetch_grid.duration.milliseconds",
|
|
event_name: [:microwaveprop, :hrrr, :fetch_grid, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.hrrr.fetch_profile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :hrrr, :fetch_profile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.hrrr.fetch_idx.duration.milliseconds",
|
|
event_name: [:microwaveprop, :hrrr, :fetch_idx, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.nexrad.fetch_frame.duration.milliseconds",
|
|
event_name: [:microwaveprop, :nexrad, :fetch_frame, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.nexrad.decode_png.duration.milliseconds",
|
|
event_name: [:microwaveprop, :nexrad, :decode_png, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.iem.fetch_asos.duration.milliseconds",
|
|
event_name: [:microwaveprop, :iem, :fetch_asos, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.iem.fetch_raob.duration.milliseconds",
|
|
event_name: [:microwaveprop, :iem, :fetch_raob, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.gefs.fetch_grid_profiles.duration.milliseconds",
|
|
event_name: [:microwaveprop, :gefs, :fetch_grid_profiles, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.narr.fetch_profile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :narr, :fetch_profile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.uwyo.fetch_sounding.duration.milliseconds",
|
|
event_name: [:microwaveprop, :uwyo, :fetch_sounding, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.elevation.fetch_profile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :elevation, :fetch_profile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp more_http_client_events do
|
|
Event.build(
|
|
:microwaveprop_more_http_client_events,
|
|
[
|
|
distribution("microwaveprop.iem.fetch_iemre.duration.milliseconds",
|
|
event_name: [:microwaveprop, :iem, :fetch_iemre, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.mrms.list_latest.duration.milliseconds",
|
|
event_name: [:microwaveprop, :mrms, :list_latest, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.mrms.download.duration.milliseconds",
|
|
event_name: [:microwaveprop, :mrms, :download, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.rtma.fetch_observation.duration.milliseconds",
|
|
event_name: [:microwaveprop, :rtma, :fetch_observation, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.ncei.fetch_metar.duration.milliseconds",
|
|
event_name: [:microwaveprop, :ncei, :fetch_metar, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.solar.fetch_indices.duration.milliseconds",
|
|
event_name: [:microwaveprop, :solar, :fetch_indices, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.swpc.fetch.duration.milliseconds",
|
|
event_name: [:microwaveprop, :swpc, :fetch, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.giro.fetch.duration.milliseconds",
|
|
event_name: [:microwaveprop, :giro, :fetch, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.qrz.request.duration.milliseconds",
|
|
event_name: [:microwaveprop, :qrz, :request, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.geocoder.geocode.duration.milliseconds",
|
|
event_name: [:microwaveprop, :geocoder, :geocode, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.srtm.download_tile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :srtm, :download_tile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.hrrr.download_grib_ranges.duration.milliseconds",
|
|
event_name: [:microwaveprop, :hrrr, :download_grib_ranges, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp subprocess_events do
|
|
Event.build(
|
|
:microwaveprop_subprocess_events,
|
|
[
|
|
distribution("microwaveprop.wgrib2.extract_grid.duration.milliseconds",
|
|
event_name: [:microwaveprop, :wgrib2, :extract_grid, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
),
|
|
distribution("microwaveprop.wgrib2.extract_grid_from_file.duration.milliseconds",
|
|
event_name: [:microwaveprop, :wgrib2, :extract_grid_from_file, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
),
|
|
distribution("microwaveprop.wgrib2.extract_grid_from_file_mapped.duration.milliseconds",
|
|
event_name: [:microwaveprop, :wgrib2, :extract_grid_from_file_mapped, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp top_level_worker_events do
|
|
Event.build(
|
|
:microwaveprop_top_level_worker_events,
|
|
[
|
|
distribution("microwaveprop.worker.terrain_profile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :worker, :terrain_profile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.worker.mechanism_classify.duration.milliseconds",
|
|
event_name: [:microwaveprop, :worker, :mechanism_classify, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.worker.mrms_fetch.duration.milliseconds",
|
|
event_name: [:microwaveprop, :worker, :mrms_fetch, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp liveview_hot_path_events do
|
|
Event.build(
|
|
:microwaveprop_liveview_hot_path_events,
|
|
[
|
|
distribution("microwaveprop.propagation.scores_at.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation, :scores_at, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.propagation.point_forecast.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation, :point_forecast, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.propagation.point_detail.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation, :point_detail, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp cache_hit_events do
|
|
Event.build(
|
|
:microwaveprop_cache_hit_events,
|
|
[
|
|
counter("microwaveprop.propagation.scores_at.cache.count",
|
|
event_name: [:microwaveprop, :propagation, :scores_at, :cache],
|
|
tags: [:hit],
|
|
description: "ScoreCache hit/miss for the map's scores_at fetch"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp db_batch_events do
|
|
Event.build(
|
|
:microwaveprop_db_batch_events,
|
|
[
|
|
distribution("microwaveprop.db.upsert_hrrr_profiles.duration.milliseconds",
|
|
event_name: [:microwaveprop, :db, :upsert_hrrr_profiles, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: db_buckets()]
|
|
),
|
|
distribution("microwaveprop.db.upsert_gefs_profiles.duration.milliseconds",
|
|
event_name: [:microwaveprop, :db, :upsert_gefs_profiles, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: db_buckets()]
|
|
),
|
|
distribution("microwaveprop.db.replace_scores.duration.milliseconds",
|
|
event_name: [:microwaveprop, :db, :replace_scores, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: db_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp worker_phase_events do
|
|
Event.build(
|
|
:microwaveprop_worker_phase_events,
|
|
[
|
|
distribution("microwaveprop.propagation_grid.score_band.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation_grid, :score_band, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
),
|
|
distribution("microwaveprop.terrain.analyse.duration.milliseconds",
|
|
event_name: [:microwaveprop, :terrain, :analyse, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.radar.aggregate_stats.duration.milliseconds",
|
|
event_name: [:microwaveprop, :radar, :aggregate_stats, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp oban_queue_depth_event do
|
|
Event.build(
|
|
:microwaveprop_oban_queue_depth,
|
|
[
|
|
last_value("microwaveprop.oban.queue.count",
|
|
event_name: [:microwaveprop, :oban, :queue, :depth],
|
|
measurement: :count,
|
|
tags: [:queue, :state],
|
|
description: "Oban job count per (queue, state) sampled every 30s"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
# Buckets in milliseconds. HTTP + compute phases: up to ~30s.
|
|
defp default_buckets, do: [10, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000]
|
|
|
|
# DB batch upserts: can take much longer on cold cache or full
|
|
# forecast-hour writes — up to ~2 min.
|
|
defp db_buckets, do: [50, 100, 250, 500, 1_000, 2_500, 5_000, 15_000, 60_000, 120_000]
|
|
|
|
# Score computation for a full grid-point batch — up to ~5 min.
|
|
defp long_buckets, do: [500, 1_000, 5_000, 15_000, 30_000, 60_000, 120_000, 300_000]
|
|
end
|