diff --git a/lib/microwaveprop/propagation.ex b/lib/microwaveprop/propagation.ex index 8b1bd346..03aca25c 100644 --- a/lib/microwaveprop/propagation.ex +++ b/lib/microwaveprop/propagation.ex @@ -149,17 +149,24 @@ defmodule Microwaveprop.Propagation do Used by `PropagationGridWorker` on the hot path. Scores are written as binary files on disk via `ScoresFile.write!/3`, one file per - band. An empty `scores` list deletes the pre-existing files for - that valid_time so a partial-failure run doesn't leave stale data - visible to the map. + band. + + Consumes `scores` in a single streaming pass that folds each score + straight into a per-band accumulator. Previously this function ran + `Enum.to_list/1` followed by `Enum.group_by/2`, which held two full + copies of the ~460k-entry grid (list + grouped list) in memory at + once — the hot path's largest transient spike after native-duct + merge. The single-pass reduce keeps only one copy and buys back + ~100 MB of headroom per forecast-hour step. """ @spec replace_scores(Enumerable.t(), DateTime.t()) :: {:ok, non_neg_integer()} | {:error, term()} def replace_scores(scores, %DateTime{} = valid_time) do - scores_list = Enum.to_list(scores) + {per_band, total} = + Enum.reduce(scores, {%{}, 0}, fn score, {acc, count} -> + {Map.update(acc, score.band_mhz, [score], &[score | &1]), count + 1} + end) - scores_list - |> Enum.group_by(& &1.band_mhz) - |> Enum.each(fn {band_mhz, band_scores} -> + Enum.each(per_band, fn {band_mhz, band_scores} -> try do ScoresFile.write!(band_mhz, valid_time, band_scores) rescue @@ -168,16 +175,7 @@ defmodule Microwaveprop.Propagation do end end) - # Clear any stale files for bands that received no scores this - # run — otherwise a partial-failure forecast hour could leave a - # prior run's data visible for the missing bands. - if scores_list == [] do - :ok - else - :ok - end - - {:ok, length(scores_list)} + {:ok, total} end @doc """ diff --git a/lib/microwaveprop/workers/propagation_grid_worker.ex b/lib/microwaveprop/workers/propagation_grid_worker.ex index 87f92ab9..737b093c 100644 --- a/lib/microwaveprop/workers/propagation_grid_worker.ex +++ b/lib/microwaveprop/workers/propagation_grid_worker.ex @@ -209,16 +209,24 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do # re-running the scorer against the stored profile. persist_profiles(grid_data, valid_time) - # Build weather cache rows from in-memory grid_data — avoids the ~20s - # JSONB round trip that was crashing the worker before f01–f18 could run. - rows = Weather.build_grid_cache_rows(grid_data, valid_time) - GridCache.broadcast_put(valid_time, rows) + # Weather map only shows the analysis hour — f01..f18 are + # forecast data that /weather doesn't render. Building and + # broadcasting a 92k-row GridCache payload for every one of + # them added a ~90 MB/pod transient spike (×3 replicas via + # PubSub) per forecast hour without any consumer. Skip both + # the cache broadcast and the weather:updated fan-out on + # forecast hours; the ProfilesFile on disk remains the source + # of truth for per-point lookups through `weather_point_detail_from_profiles/3`. + if forecast_hour == 0 do + rows = Weather.build_grid_cache_rows(grid_data, valid_time) + GridCache.broadcast_put(valid_time, rows) - Phoenix.PubSub.broadcast( - Microwaveprop.PubSub, - "weather:updated", - {:weather_updated, valid_time} - ) + Phoenix.PubSub.broadcast( + Microwaveprop.PubSub, + "weather:updated", + {:weather_updated, valid_time} + ) + end scores = compute_scores(grid_data, valid_time, forecast_hour)