prop/lib/microwaveprop/weather/grid_cache.ex
Graham McIntire 253adaf89b Cache /weather grid, defer contact show mount, cache stats
- Add Weather.GridCache: ETS cache of derived HRRR grid rows keyed by
  valid_time, cluster-synced via PubSub. Eagerly warmed from
  PropagationGridWorker after each upsert so /weather map pan/zoom and
  weather_point_detail hit zero DB on warm cache.
- Replace latest_weather_grid DB query path with cache-first lookup +
  DB fallback. hrrr_profiles is 42M rows partitioned; pulling 3-10k
  rows per viewport on every pan was the main cost.
- ContactLive.Show: defer the heavy enrichment loads (weather, solar,
  HRRR, terrain, IEMRE, elevation profile, ITU-R propagation analysis,
  data_sources) into a handle_info(:hydrate) that runs after the shell
  renders. Initial mount now returns nil placeholders; template already
  had :if guards for all of them. Shell-to-first-paint goes from
  ~500ms-2s down to ~20ms.
- Cache fetch_queue_counts for 5s in ContactLive.Show — oban_jobs group
  by query was running on every contact page view.
- Backfill stats: wrap count_unprocessed, fetch_stats, fetch_db_stats
  in Microwaveprop.Cache with 2-5s TTLs; bump refresh debounce from 1s
  to 2s so bulk enrichment events don't thrash the DB.
2026-04-12 12:55:50 -05:00

137 lines
4 KiB
Elixir

defmodule Microwaveprop.Weather.GridCache do
@moduledoc """
Node-local ETS cache of derived HRRR grid rows keyed by `valid_time`. Mirrors
`Microwaveprop.Propagation.ScoreCache` but for the `/weather` map.
The Weather map LiveView calls `latest_weather_grid/1` on mount and every
pan/zoom. Each call otherwise hits the 42M-row partitioned `hrrr_profiles`
table, runs per-row `derive_and_clean` transforms, and returns 3-10k rows.
With this cache those calls become in-memory map iterations.
Each cache entry stores `%{{lat, lon} => derived_row}` so per-point lookups
(used by `weather_point_detail/3`) are O(1). Populated by
`Microwaveprop.Weather.warm_grid_cache/1` after the hourly worker upserts
new HRRR data, fanned out across the cluster via the `"weather:cache"`
PubSub topic so every node stays in sync.
"""
use GenServer
alias Phoenix.PubSub
@table :weather_grid_cache
@topic "weather:cache"
@pubsub Microwaveprop.PubSub
@type row :: %{required(:lat) => float(), required(:lon) => float(), optional(atom()) => any()}
@type bounds :: %{optional(String.t()) => float()}
@spec start_link(keyword()) :: GenServer.on_start()
def start_link(opts), do: GenServer.start_link(__MODULE__, opts, name: __MODULE__)
@spec fetch(DateTime.t()) :: {:ok, [row()]} | :miss
def fetch(valid_time) do
case :ets.lookup(@table, valid_time) do
[{_, grid}] -> {:ok, grid_to_list(grid)}
[] -> :miss
end
end
@spec fetch_bounds(DateTime.t(), bounds() | nil) :: {:ok, [row()]} | :miss
def fetch_bounds(valid_time, bounds) do
case :ets.lookup(@table, valid_time) do
[{_, grid}] -> {:ok, grid_to_filtered_list(grid, bounds)}
[] -> :miss
end
end
@spec fetch_point(DateTime.t(), float(), float()) :: {:ok, row()} | :miss
def fetch_point(valid_time, lat, lon) do
case :ets.lookup(@table, valid_time) do
[{_, grid}] ->
case Map.get(grid, {lat, lon}) do
nil -> :miss
row -> {:ok, row}
end
[] ->
:miss
end
end
@spec put(DateTime.t(), [row()]) :: :ok
def put(valid_time, rows) do
grid = list_to_grid(rows)
:ets.insert(@table, {valid_time, grid})
:ok
end
@doc "Insert locally AND broadcast to peer nodes via PubSub."
@spec broadcast_put(DateTime.t(), [row()]) :: :ok
def broadcast_put(valid_time, rows) do
PubSub.broadcast(@pubsub, @topic, {:weather_cache_refresh, valid_time, rows})
:ok
end
@spec latest_valid_time() :: DateTime.t() | nil
def latest_valid_time do
match_spec = [{{:"$1", :_}, [], [:"$1"]}]
case :ets.select(@table, match_spec) do
[] -> nil
times -> Enum.max(times, DateTime)
end
end
@spec prune_older_than(DateTime.t()) :: non_neg_integer()
def prune_older_than(cutoff) do
match_spec = [{{:"$1", :_}, [{:<, :"$1", {:const, cutoff}}], [true]}]
:ets.select_delete(@table, match_spec)
end
@spec clear() :: :ok
def clear do
:ets.delete_all_objects(@table)
:ok
end
@spec sync() :: :ok
def sync do
GenServer.call(__MODULE__, :sync)
end
@impl true
def init(_opts) do
:ets.new(@table, [:set, :named_table, :public, read_concurrency: true])
PubSub.subscribe(@pubsub, @topic)
{:ok, %{}}
end
@impl true
def handle_call(:sync, _from, state), do: {:reply, :ok, state}
@impl true
def handle_info({:weather_cache_refresh, valid_time, rows}, state) do
put(valid_time, rows)
{:noreply, state}
end
def handle_info(_msg, state), do: {:noreply, state}
# ---------- Internal ----------
defp list_to_grid(rows) do
Map.new(rows, fn %{lat: lat, lon: lon} = row -> {{lat, lon}, row} end)
end
defp grid_to_list(grid), do: Enum.map(grid, fn {_, row} -> row end)
defp grid_to_filtered_list(grid, nil), do: grid_to_list(grid)
defp grid_to_filtered_list(grid, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
grid
|> Enum.filter(fn {{lat, lon}, _} ->
lat >= s and lat <= n and lon >= w and lon <= e
end)
|> Enum.map(fn {_, row} -> row end)
end
end