- Add Weather.GridCache: ETS cache of derived HRRR grid rows keyed by valid_time, cluster-synced via PubSub. Eagerly warmed from PropagationGridWorker after each upsert so /weather map pan/zoom and weather_point_detail hit zero DB on warm cache. - Replace latest_weather_grid DB query path with cache-first lookup + DB fallback. hrrr_profiles is 42M rows partitioned; pulling 3-10k rows per viewport on every pan was the main cost. - ContactLive.Show: defer the heavy enrichment loads (weather, solar, HRRR, terrain, IEMRE, elevation profile, ITU-R propagation analysis, data_sources) into a handle_info(:hydrate) that runs after the shell renders. Initial mount now returns nil placeholders; template already had :if guards for all of them. Shell-to-first-paint goes from ~500ms-2s down to ~20ms. - Cache fetch_queue_counts for 5s in ContactLive.Show — oban_jobs group by query was running on every contact page view. - Backfill stats: wrap count_unprocessed, fetch_stats, fetch_db_stats in Microwaveprop.Cache with 2-5s TTLs; bump refresh debounce from 1s to 2s so bulk enrichment events don't thrash the DB.
137 lines
4 KiB
Elixir
137 lines
4 KiB
Elixir
defmodule Microwaveprop.Weather.GridCache do
|
|
@moduledoc """
|
|
Node-local ETS cache of derived HRRR grid rows keyed by `valid_time`. Mirrors
|
|
`Microwaveprop.Propagation.ScoreCache` but for the `/weather` map.
|
|
|
|
The Weather map LiveView calls `latest_weather_grid/1` on mount and every
|
|
pan/zoom. Each call otherwise hits the 42M-row partitioned `hrrr_profiles`
|
|
table, runs per-row `derive_and_clean` transforms, and returns 3-10k rows.
|
|
With this cache those calls become in-memory map iterations.
|
|
|
|
Each cache entry stores `%{{lat, lon} => derived_row}` so per-point lookups
|
|
(used by `weather_point_detail/3`) are O(1). Populated by
|
|
`Microwaveprop.Weather.warm_grid_cache/1` after the hourly worker upserts
|
|
new HRRR data, fanned out across the cluster via the `"weather:cache"`
|
|
PubSub topic so every node stays in sync.
|
|
"""
|
|
use GenServer
|
|
|
|
alias Phoenix.PubSub
|
|
|
|
@table :weather_grid_cache
|
|
@topic "weather:cache"
|
|
@pubsub Microwaveprop.PubSub
|
|
|
|
@type row :: %{required(:lat) => float(), required(:lon) => float(), optional(atom()) => any()}
|
|
@type bounds :: %{optional(String.t()) => float()}
|
|
|
|
@spec start_link(keyword()) :: GenServer.on_start()
|
|
def start_link(opts), do: GenServer.start_link(__MODULE__, opts, name: __MODULE__)
|
|
|
|
@spec fetch(DateTime.t()) :: {:ok, [row()]} | :miss
|
|
def fetch(valid_time) do
|
|
case :ets.lookup(@table, valid_time) do
|
|
[{_, grid}] -> {:ok, grid_to_list(grid)}
|
|
[] -> :miss
|
|
end
|
|
end
|
|
|
|
@spec fetch_bounds(DateTime.t(), bounds() | nil) :: {:ok, [row()]} | :miss
|
|
def fetch_bounds(valid_time, bounds) do
|
|
case :ets.lookup(@table, valid_time) do
|
|
[{_, grid}] -> {:ok, grid_to_filtered_list(grid, bounds)}
|
|
[] -> :miss
|
|
end
|
|
end
|
|
|
|
@spec fetch_point(DateTime.t(), float(), float()) :: {:ok, row()} | :miss
|
|
def fetch_point(valid_time, lat, lon) do
|
|
case :ets.lookup(@table, valid_time) do
|
|
[{_, grid}] ->
|
|
case Map.get(grid, {lat, lon}) do
|
|
nil -> :miss
|
|
row -> {:ok, row}
|
|
end
|
|
|
|
[] ->
|
|
:miss
|
|
end
|
|
end
|
|
|
|
@spec put(DateTime.t(), [row()]) :: :ok
|
|
def put(valid_time, rows) do
|
|
grid = list_to_grid(rows)
|
|
:ets.insert(@table, {valid_time, grid})
|
|
:ok
|
|
end
|
|
|
|
@doc "Insert locally AND broadcast to peer nodes via PubSub."
|
|
@spec broadcast_put(DateTime.t(), [row()]) :: :ok
|
|
def broadcast_put(valid_time, rows) do
|
|
PubSub.broadcast(@pubsub, @topic, {:weather_cache_refresh, valid_time, rows})
|
|
:ok
|
|
end
|
|
|
|
@spec latest_valid_time() :: DateTime.t() | nil
|
|
def latest_valid_time do
|
|
match_spec = [{{:"$1", :_}, [], [:"$1"]}]
|
|
|
|
case :ets.select(@table, match_spec) do
|
|
[] -> nil
|
|
times -> Enum.max(times, DateTime)
|
|
end
|
|
end
|
|
|
|
@spec prune_older_than(DateTime.t()) :: non_neg_integer()
|
|
def prune_older_than(cutoff) do
|
|
match_spec = [{{:"$1", :_}, [{:<, :"$1", {:const, cutoff}}], [true]}]
|
|
:ets.select_delete(@table, match_spec)
|
|
end
|
|
|
|
@spec clear() :: :ok
|
|
def clear do
|
|
:ets.delete_all_objects(@table)
|
|
:ok
|
|
end
|
|
|
|
@spec sync() :: :ok
|
|
def sync do
|
|
GenServer.call(__MODULE__, :sync)
|
|
end
|
|
|
|
@impl true
|
|
def init(_opts) do
|
|
:ets.new(@table, [:set, :named_table, :public, read_concurrency: true])
|
|
PubSub.subscribe(@pubsub, @topic)
|
|
{:ok, %{}}
|
|
end
|
|
|
|
@impl true
|
|
def handle_call(:sync, _from, state), do: {:reply, :ok, state}
|
|
|
|
@impl true
|
|
def handle_info({:weather_cache_refresh, valid_time, rows}, state) do
|
|
put(valid_time, rows)
|
|
{:noreply, state}
|
|
end
|
|
|
|
def handle_info(_msg, state), do: {:noreply, state}
|
|
|
|
# ---------- Internal ----------
|
|
|
|
defp list_to_grid(rows) do
|
|
Map.new(rows, fn %{lat: lat, lon: lon} = row -> {{lat, lon}, row} end)
|
|
end
|
|
|
|
defp grid_to_list(grid), do: Enum.map(grid, fn {_, row} -> row end)
|
|
|
|
defp grid_to_filtered_list(grid, nil), do: grid_to_list(grid)
|
|
|
|
defp grid_to_filtered_list(grid, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
|
|
grid
|
|
|> Enum.filter(fn {{lat, lon}, _} ->
|
|
lat >= s and lat <= n and lon >= w and lon <= e
|
|
end)
|
|
|> Enum.map(fn {_, row} -> row end)
|
|
end
|
|
end
|