perf(map): chunk-keyed ScoreCache for viewport reads
Bucket each cache entry into 5°×5° spatial chunks matching the layout used by Weather.GridCache. fetch_bounds/3 now walks only the chunks intersecting the viewport instead of the full 92k-cell CONUS map; a typical zoomed viewport hits ~1500 cells per chunk × the chunks the viewport overlaps instead of scanning all 92k. Public API and observable behavior unchanged. Add cross-chunk-boundary regression tests so future refactors can't silently flatten the layout.
This commit is contained in:
parent
6a90d153ca
commit
c20aceba95
2 changed files with 105 additions and 35 deletions
|
|
@ -14,8 +14,14 @@ defmodule Microwaveprop.Propagation.ScoreCache do
|
||||||
lighter-weight `"propagation:updated"` PubSub so LiveView clients refresh
|
lighter-weight `"propagation:updated"` PubSub so LiveView clients refresh
|
||||||
while the actual scores load on demand from `/data/scores/.../*.prop`.
|
while the actual scores load on demand from `/data/scores/.../*.prop`.
|
||||||
|
|
||||||
Each cache entry stores a `%{{lat, lon} => score}` map so per-point lookups
|
## Storage layout
|
||||||
(used by `point_forecast/3` and `point_detail/4`) stay O(1) on hits.
|
|
||||||
|
Each cache entry is bucketed into 5°×5° spatial chunks matching the layout
|
||||||
|
used by `Microwaveprop.Weather.GridCache`. The ETS value for
|
||||||
|
`{band_mhz, valid_time}` is `%{{lat_band, lon_band} => %{{lat, lon} => score}}`.
|
||||||
|
That way `fetch_bounds/3` only walks the chunks that intersect the
|
||||||
|
requested viewport instead of the full 92k-cell CONUS map; `fetch_point/4`
|
||||||
|
reads exactly one chunk.
|
||||||
"""
|
"""
|
||||||
use GenServer
|
use GenServer
|
||||||
|
|
||||||
|
|
@ -24,14 +30,16 @@ defmodule Microwaveprop.Propagation.ScoreCache do
|
||||||
@table :propagation_score_cache
|
@table :propagation_score_cache
|
||||||
@topic "propagation:cache"
|
@topic "propagation:cache"
|
||||||
@pubsub Microwaveprop.PubSub
|
@pubsub Microwaveprop.PubSub
|
||||||
|
@chunk_step 5
|
||||||
|
|
||||||
# Hard cap on cached `{band_mhz, valid_time}` entries. Each value is a
|
# Hard cap on cached `{band_mhz, valid_time}` entries. Each value is a
|
||||||
# 92k-cell `Map.new` weighing ~1.94 MiB; without a cap the eager warm
|
# 92k-cell map (now bucketed into 5°×5° chunks) on the order of ~2 MiB;
|
||||||
# path materialises 23 bands × 19 forecast hours = 437 entries per pod
|
# without a cap the eager warm path materialises 23 bands × 19 forecast
|
||||||
# (~850 MiB) and OOMs the BEAM. The cap keeps the LiveView hot path
|
# hours = 437 entries per pod (~850 MiB) and OOMs the BEAM. The cap
|
||||||
# warm (one band × the hour or two the user is actively scrubbing)
|
# keeps the LiveView hot path warm (one band × the hour or two the
|
||||||
# and lets the rest fall back to direct `.prop` reads via
|
# user is actively scrubbing) and lets the rest fall back to direct
|
||||||
# `read_from_disk_and_cache/3`, which is sub-100 ms per file.
|
# `.prop` reads via `read_from_disk_and_cache/3`, which is sub-100 ms
|
||||||
|
# per file.
|
||||||
@max_entries 32
|
@max_entries 32
|
||||||
|
|
||||||
@type score :: %{lat: float(), lon: float(), score: non_neg_integer()}
|
@type score :: %{lat: float(), lon: float(), score: non_neg_integer()}
|
||||||
|
|
@ -47,7 +55,7 @@ defmodule Microwaveprop.Propagation.ScoreCache do
|
||||||
@spec fetch(non_neg_integer(), DateTime.t()) :: {:ok, [score()]} | :miss
|
@spec fetch(non_neg_integer(), DateTime.t()) :: {:ok, [score()]} | :miss
|
||||||
def fetch(band_mhz, valid_time) do
|
def fetch(band_mhz, valid_time) do
|
||||||
case :ets.lookup(@table, {band_mhz, valid_time}) do
|
case :ets.lookup(@table, {band_mhz, valid_time}) do
|
||||||
[{_, grid}] -> {:ok, grid_to_list(grid)}
|
[{_, chunked}] -> {:ok, chunks_to_list(chunked)}
|
||||||
[] -> :miss
|
[] -> :miss
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
@ -56,7 +64,7 @@ defmodule Microwaveprop.Propagation.ScoreCache do
|
||||||
{:ok, [score()]} | :miss
|
{:ok, [score()]} | :miss
|
||||||
def fetch_bounds(band_mhz, valid_time, bounds) do
|
def fetch_bounds(band_mhz, valid_time, bounds) do
|
||||||
case :ets.lookup(@table, {band_mhz, valid_time}) do
|
case :ets.lookup(@table, {band_mhz, valid_time}) do
|
||||||
[{_, grid}] -> {:ok, grid_to_filtered_list(grid, bounds)}
|
[{_, chunked}] -> {:ok, chunks_filtered_to_list(chunked, bounds)}
|
||||||
[] -> :miss
|
[] -> :miss
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
@ -71,8 +79,10 @@ defmodule Microwaveprop.Propagation.ScoreCache do
|
||||||
{:ok, non_neg_integer()} | :miss
|
{:ok, non_neg_integer()} | :miss
|
||||||
def fetch_point(band_mhz, valid_time, lat, lon) do
|
def fetch_point(band_mhz, valid_time, lat, lon) do
|
||||||
case :ets.lookup(@table, {band_mhz, valid_time}) do
|
case :ets.lookup(@table, {band_mhz, valid_time}) do
|
||||||
[{_, grid}] ->
|
[{_, chunked}] ->
|
||||||
case Map.get(grid, {lat, lon}) do
|
chunk_key = chunk_key_for(lat, lon)
|
||||||
|
|
||||||
|
case chunked |> Map.get(chunk_key, %{}) |> Map.get({lat, lon}) do
|
||||||
nil -> :miss
|
nil -> :miss
|
||||||
score -> {:ok, score}
|
score -> {:ok, score}
|
||||||
end
|
end
|
||||||
|
|
@ -88,8 +98,8 @@ defmodule Microwaveprop.Propagation.ScoreCache do
|
||||||
|
|
||||||
@spec put(non_neg_integer(), DateTime.t(), [score()]) :: :ok
|
@spec put(non_neg_integer(), DateTime.t(), [score()]) :: :ok
|
||||||
def put(band_mhz, valid_time, scores) do
|
def put(band_mhz, valid_time, scores) do
|
||||||
grid = list_to_grid(scores)
|
chunked = list_to_chunks(scores)
|
||||||
:ets.insert(@table, {{band_mhz, valid_time}, grid})
|
:ets.insert(@table, {{band_mhz, valid_time}, chunked})
|
||||||
enforce_capacity()
|
enforce_capacity()
|
||||||
:ok
|
:ok
|
||||||
end
|
end
|
||||||
|
|
@ -207,27 +217,51 @@ defmodule Microwaveprop.Propagation.ScoreCache do
|
||||||
|
|
||||||
# ---------- Internal ----------
|
# ---------- Internal ----------
|
||||||
|
|
||||||
defp list_to_grid(scores) do
|
defp list_to_chunks(scores) do
|
||||||
Map.new(scores, fn %{lat: lat, lon: lon, score: score} -> {{lat, lon}, score} end)
|
Enum.reduce(scores, %{}, fn %{lat: lat, lon: lon, score: score}, acc ->
|
||||||
end
|
key = chunk_key_for(lat, lon)
|
||||||
|
chunk = Map.get(acc, key, %{})
|
||||||
defp grid_to_list(grid) do
|
Map.put(acc, key, Map.put(chunk, {lat, lon}, score))
|
||||||
Enum.map(grid, fn {{lat, lon}, score} -> %{lat: lat, lon: lon, score: score} end)
|
|
||||||
end
|
|
||||||
|
|
||||||
defp grid_to_filtered_list(grid, nil), do: grid_to_list(grid)
|
|
||||||
|
|
||||||
defp grid_to_filtered_list(grid, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
|
|
||||||
# Single pass over 92k cells: the former Enum.filter |> Enum.map
|
|
||||||
# walked the map twice and allocated an intermediate list of
|
|
||||||
# {key, value} tuples between them. Enum.reduce emits only the
|
|
||||||
# in-bounds result maps directly.
|
|
||||||
Enum.reduce(grid, [], fn {{lat, lon}, score}, acc ->
|
|
||||||
if lat >= s and lat <= n and lon >= w and lon <= e do
|
|
||||||
[%{lat: lat, lon: lon, score: score} | acc]
|
|
||||||
else
|
|
||||||
acc
|
|
||||||
end
|
|
||||||
end)
|
end)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
defp chunks_to_list(chunked) do
|
||||||
|
Enum.flat_map(chunked, fn {_chunk_key, cells} ->
|
||||||
|
Enum.map(cells, fn {{lat, lon}, score} ->
|
||||||
|
%{lat: lat, lon: lon, score: score}
|
||||||
|
end)
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp chunks_filtered_to_list(chunked, nil), do: chunks_to_list(chunked)
|
||||||
|
|
||||||
|
defp chunks_filtered_to_list(chunked, %{"south" => s, "north" => n, "west" => w, "east" => e} = bounds) do
|
||||||
|
chunked
|
||||||
|
|> Enum.filter(fn {chunk_key, _} -> chunk_intersects_bounds?(chunk_key, bounds) end)
|
||||||
|
|> Enum.flat_map(fn {_, cells} ->
|
||||||
|
for {{lat, lon}, score} <- cells,
|
||||||
|
lat >= s,
|
||||||
|
lat <= n,
|
||||||
|
lon >= w,
|
||||||
|
lon <= e,
|
||||||
|
do: %{lat: lat, lon: lon, score: score}
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp chunk_key_for(lat, lon) do
|
||||||
|
{chunk_band(lat * 1.0), chunk_band(lon * 1.0)}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp chunk_band(value) when is_float(value) do
|
||||||
|
(value / @chunk_step) |> Float.floor() |> trunc()
|
||||||
|
end
|
||||||
|
|
||||||
|
defp chunk_intersects_bounds?({lat_band, lon_band}, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
|
||||||
|
chunk_south = lat_band * @chunk_step
|
||||||
|
chunk_north = chunk_south + @chunk_step
|
||||||
|
chunk_west = lon_band * @chunk_step
|
||||||
|
chunk_east = chunk_west + @chunk_step
|
||||||
|
|
||||||
|
chunk_north >= s and chunk_south <= n and chunk_east >= w and chunk_west <= e
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
|
||||||
|
|
@ -76,6 +76,42 @@ defmodule Microwaveprop.Propagation.ScoreCacheTest do
|
||||||
assert {:ok, [%{lat: 32.0, lon: -97.0, score: 75}]} =
|
assert {:ok, [%{lat: 32.0, lon: -97.0, score: 75}]} =
|
||||||
ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 12:00:00Z], bounds)
|
ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 12:00:00Z], bounds)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
test "returns cells across multiple 5°×5° chunks when viewport spans them" do
|
||||||
|
# Three points spanning three distinct 5°×5° chunks on the lat axis
|
||||||
|
# (~30, ~35, ~40) and three on the lon axis (~-100, ~-95, ~-75).
|
||||||
|
# A viewport covering all three must return them all even though
|
||||||
|
# each lives in its own chunk under the bucketed storage layout.
|
||||||
|
scores = [
|
||||||
|
%{lat: 31.0, lon: -99.0, score: 10},
|
||||||
|
%{lat: 36.0, lon: -94.0, score: 20},
|
||||||
|
%{lat: 41.0, lon: -74.0, score: 30}
|
||||||
|
]
|
||||||
|
|
||||||
|
ScoreCache.put(10_000, ~U[2026-04-12 13:00:00Z], scores)
|
||||||
|
|
||||||
|
bounds = %{"south" => 30.0, "north" => 45.0, "west" => -100.0, "east" => -70.0}
|
||||||
|
|
||||||
|
assert {:ok, returned} =
|
||||||
|
ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 13:00:00Z], bounds)
|
||||||
|
|
||||||
|
assert Enum.sort_by(returned, & &1.score) == Enum.sort_by(scores, & &1.score)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "narrow viewport excludes cells in non-overlapping chunks" do
|
||||||
|
scores = [
|
||||||
|
%{lat: 31.0, lon: -99.0, score: 10},
|
||||||
|
%{lat: 41.0, lon: -74.0, score: 30}
|
||||||
|
]
|
||||||
|
|
||||||
|
ScoreCache.put(10_000, ~U[2026-04-12 13:00:00Z], scores)
|
||||||
|
|
||||||
|
# Viewport entirely within the (lat 30-35, lon -100..-95) chunk.
|
||||||
|
bounds = %{"south" => 30.0, "north" => 33.0, "west" => -100.0, "east" => -97.0}
|
||||||
|
|
||||||
|
assert {:ok, [%{score: 10}]} =
|
||||||
|
ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 13:00:00Z], bounds)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
describe "prune_older_than/1" do
|
describe "prune_older_than/1" do
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue