Telemetry showed the application-master process holding ~830 MiB of terms from warm_grid_cache_from_latest_profile — the data lives in the app master's heap and never GCs because the process is idle. Running it in a Task.start lets the terms die with the task. Mark GridCache, MrmsCache, NexradCache, and ScoreCache ETS tables :compressed. The scored-band-map and HRRR grid data are map-heavy; compression trims hundreds of MiB at a few percent CPU cost. Memoise HRRR .idx responses in Microwaveprop.Cache. Published idx files are immutable for a model run, but the hourly chain re-fetches the same URL dozens of times across forecast hours. Cuts ~10s per repeat out of hrrr_fetch_idx. Force a garbage collect at the end of HrrrFetchWorker.perform to reclaim the refc binary heap held from GRIB2 ranges before the Oban producer hands the process its next job.
160 lines
4.9 KiB
Elixir
160 lines
4.9 KiB
Elixir
defmodule Microwaveprop.Propagation.ScoreCache do
|
|
@moduledoc """
|
|
Node-local ETS cache of propagation scores keyed by `{band_mhz, valid_time}`.
|
|
|
|
Populated eagerly by `Microwaveprop.Workers.PropagationGridWorker` after each
|
|
hourly compute (via `broadcast_put/3`) and lazily by `Microwaveprop.Propagation.scores_at/3`
|
|
on a cache miss. `broadcast_put/3` fans the payload out across the cluster on
|
|
the `"propagation:cache"` PubSub topic so every BEAM node stays in sync with
|
|
a single compute.
|
|
|
|
Internally each cache entry stores a `%{{lat, lon} => score}` map so that
|
|
per-point lookups (used by `point_forecast/3` and `point_detail/4`) are O(1).
|
|
"""
|
|
use GenServer
|
|
|
|
alias Phoenix.PubSub
|
|
|
|
@table :propagation_score_cache
|
|
@topic "propagation:cache"
|
|
@pubsub Microwaveprop.PubSub
|
|
|
|
@type score :: %{lat: float(), lon: float(), score: non_neg_integer()}
|
|
@type bounds :: %{optional(String.t()) => float()}
|
|
|
|
# ---------- Public API ----------
|
|
|
|
@spec start_link(keyword()) :: GenServer.on_start()
|
|
def start_link(opts) do
|
|
GenServer.start_link(__MODULE__, opts, name: __MODULE__)
|
|
end
|
|
|
|
@spec fetch(non_neg_integer(), DateTime.t()) :: {:ok, [score()]} | :miss
|
|
def fetch(band_mhz, valid_time) do
|
|
case :ets.lookup(@table, {band_mhz, valid_time}) do
|
|
[{_, grid}] -> {:ok, grid_to_list(grid)}
|
|
[] -> :miss
|
|
end
|
|
end
|
|
|
|
@spec fetch_bounds(non_neg_integer(), DateTime.t(), bounds() | nil) ::
|
|
{:ok, [score()]} | :miss
|
|
def fetch_bounds(band_mhz, valid_time, bounds) do
|
|
case :ets.lookup(@table, {band_mhz, valid_time}) do
|
|
[{_, grid}] -> {:ok, grid_to_filtered_list(grid, bounds)}
|
|
[] -> :miss
|
|
end
|
|
end
|
|
|
|
@doc """
|
|
Look up the score for a single `{lat, lon}` point at `{band_mhz, valid_time}`.
|
|
Returns `{:ok, score}` on hit or `:miss` on cache miss. The coordinates are
|
|
used as-is (no snap-to-grid) — callers that need the nearest grid cell should
|
|
snap first.
|
|
"""
|
|
@spec fetch_point(non_neg_integer(), DateTime.t(), float(), float()) ::
|
|
{:ok, non_neg_integer()} | :miss
|
|
def fetch_point(band_mhz, valid_time, lat, lon) do
|
|
case :ets.lookup(@table, {band_mhz, valid_time}) do
|
|
[{_, grid}] ->
|
|
case Map.get(grid, {lat, lon}) do
|
|
nil -> :miss
|
|
score -> {:ok, score}
|
|
end
|
|
|
|
[] ->
|
|
:miss
|
|
end
|
|
end
|
|
|
|
@spec put(non_neg_integer(), DateTime.t(), [score()]) :: :ok
|
|
def put(band_mhz, valid_time, scores) do
|
|
grid = list_to_grid(scores)
|
|
:ets.insert(@table, {{band_mhz, valid_time}, grid})
|
|
:ok
|
|
end
|
|
|
|
@doc """
|
|
Insert locally AND broadcast to peer nodes via PubSub.
|
|
|
|
Call this from the single pod that computed the scores; every cluster member
|
|
(including the caller) will receive the PubSub message and populate its local
|
|
ETS. Use `put/3` directly if you already have a cluster-wide fan-out.
|
|
"""
|
|
@spec broadcast_put(non_neg_integer(), DateTime.t(), [score()]) :: :ok
|
|
def broadcast_put(band_mhz, valid_time, scores) do
|
|
PubSub.broadcast(@pubsub, @topic, {:cache_refresh, band_mhz, valid_time, scores})
|
|
:ok
|
|
end
|
|
|
|
@doc "Returns the sorted list of valid_times cached for `band_mhz`."
|
|
@spec valid_times(non_neg_integer()) :: [DateTime.t()]
|
|
def valid_times(band_mhz) do
|
|
match_spec = [{{{band_mhz, :"$1"}, :_}, [], [:"$1"]}]
|
|
|
|
@table
|
|
|> :ets.select(match_spec)
|
|
|> Enum.sort({:asc, DateTime})
|
|
end
|
|
|
|
@spec prune_older_than(DateTime.t()) :: non_neg_integer()
|
|
def prune_older_than(cutoff) do
|
|
match_spec = [
|
|
{{{:_, :"$1"}, :_}, [{:<, :"$1", {:const, cutoff}}], [true]}
|
|
]
|
|
|
|
:ets.select_delete(@table, match_spec)
|
|
end
|
|
|
|
@spec clear() :: :ok
|
|
def clear do
|
|
:ets.delete_all_objects(@table)
|
|
:ok
|
|
end
|
|
|
|
@doc "Flush the GenServer mailbox so any in-flight cache_refresh messages are applied."
|
|
@spec sync() :: :ok
|
|
def sync do
|
|
GenServer.call(__MODULE__, :sync)
|
|
end
|
|
|
|
# ---------- GenServer callbacks ----------
|
|
|
|
@impl true
|
|
def init(_opts) do
|
|
:ets.new(@table, [:set, :named_table, :public, :compressed, read_concurrency: true])
|
|
PubSub.subscribe(@pubsub, @topic)
|
|
{:ok, %{}}
|
|
end
|
|
|
|
@impl true
|
|
def handle_call(:sync, _from, state), do: {:reply, :ok, state}
|
|
|
|
@impl true
|
|
def handle_info({:cache_refresh, band_mhz, valid_time, scores}, state) do
|
|
put(band_mhz, valid_time, scores)
|
|
{:noreply, state}
|
|
end
|
|
|
|
def handle_info(_msg, state), do: {:noreply, state}
|
|
|
|
# ---------- Internal ----------
|
|
|
|
defp list_to_grid(scores) do
|
|
Map.new(scores, fn %{lat: lat, lon: lon, score: score} -> {{lat, lon}, score} end)
|
|
end
|
|
|
|
defp grid_to_list(grid) do
|
|
Enum.map(grid, fn {{lat, lon}, score} -> %{lat: lat, lon: lon, score: score} end)
|
|
end
|
|
|
|
defp grid_to_filtered_list(grid, nil), do: grid_to_list(grid)
|
|
|
|
defp grid_to_filtered_list(grid, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
|
|
grid
|
|
|> Enum.filter(fn {{lat, lon}, _score} ->
|
|
lat >= s and lat <= n and lon >= w and lon <= e
|
|
end)
|
|
|> Enum.map(fn {{lat, lon}, score} -> %{lat: lat, lon: lon, score: score} end)
|
|
end
|
|
end
|