Graham McIntire
4f82bd691e
ScoreCache held 437 entries × ~1.94 MiB each (~850 MiB per pod) of
{band_mhz, valid_time} grids — data already on NFS as compact .prop
files. NotifyListener and ScoreCacheReconciler both eagerly materialised
every band into ETS on each Rust completion / 60s sweep, so 5 hot
replicas wasted ~4.2 GiB of redundant cache and OOMed at 6 GiB.
Bound the cache to 32 entries with eviction by oldest valid_time, drop
the eager warm loops, and let LiveView callers lazy-fill via the
existing ScoresFile read path. ScoreCacheReconciler had no remaining
purpose and is deleted; runbook + prom_ex counters updated to match.
Steady-state cache footprint ~60 MiB per pod instead of ~850 MiB.
370 lines
15 KiB
Elixir
370 lines
15 KiB
Elixir
defmodule Microwaveprop.PromEx.InstrumentPlugin do
|
|
@moduledoc """
|
|
PromEx plugin that registers every `Microwaveprop.Instrument` span as
|
|
a Prometheus histogram. Keep the catalog in sync with the call sites
|
|
in `HrrrClient`, `NexradClient`, `IemClient`, `GefsClient`,
|
|
`NarrClient`, `UwyoSoundingClient`, `ElevationClient`, `Weather` batch
|
|
upserts, `Propagation.replace_scores`, `PropagationGridWorker`,
|
|
`TerrainAnalysis`, and `CommonVolumeRadarWorker`.
|
|
"""
|
|
use PromEx.Plugin
|
|
|
|
@impl true
|
|
def event_metrics(_opts) do
|
|
[
|
|
http_client_events(),
|
|
more_http_client_events(),
|
|
subprocess_events(),
|
|
db_batch_events(),
|
|
worker_phase_events(),
|
|
top_level_worker_events(),
|
|
liveview_hot_path_events(),
|
|
cache_hit_events(),
|
|
oban_queue_depth_event()
|
|
]
|
|
end
|
|
|
|
# PromEx interprets `Telemetry.Metrics` specs the same way the
|
|
# in-process telemetry_metrics module does, so we re-use the same
|
|
# event_name / measurement / tags shape from MicrowavepropWeb.Telemetry.
|
|
defp http_client_events do
|
|
Event.build(
|
|
:microwaveprop_http_client_events,
|
|
[
|
|
distribution("microwaveprop.hrrr.fetch_grid.duration.milliseconds",
|
|
event_name: [:microwaveprop, :hrrr, :fetch_grid, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.hrrr.fetch_profile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :hrrr, :fetch_profile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.hrrr.fetch_idx.duration.milliseconds",
|
|
event_name: [:microwaveprop, :hrrr, :fetch_idx, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.nexrad.fetch_frame.duration.milliseconds",
|
|
event_name: [:microwaveprop, :nexrad, :fetch_frame, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.nexrad.decode_png.duration.milliseconds",
|
|
event_name: [:microwaveprop, :nexrad, :decode_png, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.iem.fetch_asos.duration.milliseconds",
|
|
event_name: [:microwaveprop, :iem, :fetch_asos, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.iem.fetch_raob.duration.milliseconds",
|
|
event_name: [:microwaveprop, :iem, :fetch_raob, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.gefs.fetch_grid_profiles.duration.milliseconds",
|
|
event_name: [:microwaveprop, :gefs, :fetch_grid_profiles, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.narr.fetch_profile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :narr, :fetch_profile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.uwyo.fetch_sounding.duration.milliseconds",
|
|
event_name: [:microwaveprop, :uwyo, :fetch_sounding, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.elevation.fetch_profile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :elevation, :fetch_profile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp more_http_client_events do
|
|
Event.build(
|
|
:microwaveprop_more_http_client_events,
|
|
[
|
|
distribution("microwaveprop.iem.fetch_iemre.duration.milliseconds",
|
|
event_name: [:microwaveprop, :iem, :fetch_iemre, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.mrms.list_latest.duration.milliseconds",
|
|
event_name: [:microwaveprop, :mrms, :list_latest, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.mrms.download.duration.milliseconds",
|
|
event_name: [:microwaveprop, :mrms, :download, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.rtma.fetch_observation.duration.milliseconds",
|
|
event_name: [:microwaveprop, :rtma, :fetch_observation, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.ncei.fetch_metar.duration.milliseconds",
|
|
event_name: [:microwaveprop, :ncei, :fetch_metar, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.solar.fetch_indices.duration.milliseconds",
|
|
event_name: [:microwaveprop, :solar, :fetch_indices, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.swpc.fetch.duration.milliseconds",
|
|
event_name: [:microwaveprop, :swpc, :fetch, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.giro.fetch.duration.milliseconds",
|
|
event_name: [:microwaveprop, :giro, :fetch, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.qrz.request.duration.milliseconds",
|
|
event_name: [:microwaveprop, :qrz, :request, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.geocoder.geocode.duration.milliseconds",
|
|
event_name: [:microwaveprop, :geocoder, :geocode, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.srtm.download_tile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :srtm, :download_tile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.hrrr.download_grib_ranges.duration.milliseconds",
|
|
event_name: [:microwaveprop, :hrrr, :download_grib_ranges, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp subprocess_events do
|
|
Event.build(
|
|
:microwaveprop_subprocess_events,
|
|
[
|
|
distribution("microwaveprop.wgrib2.extract_grid.duration.milliseconds",
|
|
event_name: [:microwaveprop, :wgrib2, :extract_grid, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
),
|
|
distribution("microwaveprop.wgrib2.extract_grid_from_file.duration.milliseconds",
|
|
event_name: [:microwaveprop, :wgrib2, :extract_grid_from_file, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
),
|
|
distribution("microwaveprop.wgrib2.extract_grid_from_file_mapped.duration.milliseconds",
|
|
event_name: [:microwaveprop, :wgrib2, :extract_grid_from_file_mapped, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp top_level_worker_events do
|
|
Event.build(
|
|
:microwaveprop_top_level_worker_events,
|
|
[
|
|
distribution("microwaveprop.worker.terrain_profile.duration.milliseconds",
|
|
event_name: [:microwaveprop, :worker, :terrain_profile, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.worker.mechanism_classify.duration.milliseconds",
|
|
event_name: [:microwaveprop, :worker, :mechanism_classify, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.worker.mrms_fetch.duration.milliseconds",
|
|
event_name: [:microwaveprop, :worker, :mrms_fetch, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
),
|
|
distribution("microwaveprop.propagation.grid_worker.perform.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation, :grid_worker, :perform, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
last_value("microwaveprop.propagation.grid_worker.seeded.queued_tasks",
|
|
event_name: [:microwaveprop, :propagation, :grid_worker, :seeded],
|
|
measurement: :queued_tasks,
|
|
description: "grid_tasks rows inserted by the hourly seed worker"
|
|
),
|
|
counter("microwaveprop.propagation.grid_worker.exception.count",
|
|
event_name: [:microwaveprop, :propagation, :grid_worker, :exception],
|
|
description: "PropagationGridWorker seed failures"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp liveview_hot_path_events do
|
|
Event.build(
|
|
:microwaveprop_liveview_hot_path_events,
|
|
[
|
|
distribution("microwaveprop.propagation.scores_at.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation, :scores_at, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.propagation.point_forecast.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation, :point_forecast, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.propagation.point_detail.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation, :point_detail, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp cache_hit_events do
|
|
Event.build(
|
|
:microwaveprop_cache_hit_events,
|
|
[
|
|
counter("microwaveprop.propagation.scores_at.cache.count",
|
|
event_name: [:microwaveprop, :propagation, :scores_at, :cache],
|
|
tags: [:hit],
|
|
description: "ScoreCache hit/miss for the map's scores_at fetch"
|
|
),
|
|
counter("microwaveprop.weather.grid_cache.lookup.count",
|
|
event_name: [:microwaveprop, :weather, :grid_cache, :lookup],
|
|
tags: [:hit],
|
|
description: "GridCache hit/miss for the /weather map's grid fetch"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp db_batch_events do
|
|
Event.build(
|
|
:microwaveprop_db_batch_events,
|
|
[
|
|
distribution("microwaveprop.db.upsert_hrrr_profiles.duration.milliseconds",
|
|
event_name: [:microwaveprop, :db, :upsert_hrrr_profiles, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: db_buckets()]
|
|
),
|
|
distribution("microwaveprop.db.upsert_gefs_profiles.duration.milliseconds",
|
|
event_name: [:microwaveprop, :db, :upsert_gefs_profiles, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: db_buckets()]
|
|
),
|
|
distribution("microwaveprop.db.replace_scores.duration.milliseconds",
|
|
event_name: [:microwaveprop, :db, :replace_scores, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: db_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp worker_phase_events do
|
|
Event.build(
|
|
:microwaveprop_worker_phase_events,
|
|
[
|
|
distribution("microwaveprop.propagation_grid.score_band.duration.milliseconds",
|
|
event_name: [:microwaveprop, :propagation_grid, :score_band, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: long_buckets()]
|
|
),
|
|
distribution("microwaveprop.terrain.analyse.duration.milliseconds",
|
|
event_name: [:microwaveprop, :terrain, :analyse, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
),
|
|
distribution("microwaveprop.radar.aggregate_stats.duration.milliseconds",
|
|
event_name: [:microwaveprop, :radar, :aggregate_stats, :stop],
|
|
measurement: :duration,
|
|
unit: {:native, :millisecond},
|
|
reporter_options: [buckets: default_buckets()]
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
defp oban_queue_depth_event do
|
|
Event.build(
|
|
:microwaveprop_oban_queue_depth,
|
|
[
|
|
last_value("microwaveprop.oban.queue.count",
|
|
event_name: [:microwaveprop, :oban, :queue, :depth],
|
|
measurement: :count,
|
|
tags: [:queue, :state],
|
|
description: "Oban job count per (queue, state) sampled every 30s"
|
|
)
|
|
]
|
|
)
|
|
end
|
|
|
|
# Buckets in milliseconds. HTTP + compute phases: up to ~30s.
|
|
defp default_buckets, do: [10, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000]
|
|
|
|
# DB batch upserts: can take much longer on cold cache or full
|
|
# forecast-hour writes — up to ~2 min.
|
|
defp db_buckets, do: [50, 100, 250, 500, 1_000, 2_500, 5_000, 15_000, 60_000, 120_000]
|
|
|
|
# Score computation for a full grid-point batch — up to ~5 min.
|
|
defp long_buckets, do: [500, 1_000, 5_000, 15_000, 30_000, 60_000, 120_000, 300_000]
|
|
end
|