graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

perf(map): chunk-keyed ScoreCache for viewport reads

Browse source 
Bucket each cache entry into 5°×5° spatial chunks matching the layout
used by Weather.GridCache. fetch_bounds/3 now walks only the chunks
intersecting the viewport instead of the full 92k-cell CONUS map; a
typical zoomed viewport hits ~1500 cells per chunk × the chunks the
viewport overlaps instead of scanning all 92k.

Public API and observable behavior unchanged. Add cross-chunk-boundary
regression tests so future refactors can't silently flatten the layout.
This commit is contained in:
Graham McIntire 2026-04-29 16:47:03 -05:00
parent 6a90d153ca
commit c20aceba95
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 105 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

104
lib/microwaveprop/propagation/score_cache.ex
View file

@ -14,8 +14,14 @@ defmodule Microwaveprop.Propagation.ScoreCache do
lighter-weight `"propagation:updated"` PubSub so LiveView clients refresh
while the actual scores load on demand from `/data/scores/.../*.prop`.
Each cache entry stores a `%{{lat, lon} => score}` map so per-point lookups
(used by `point_forecast/3` and `point_detail/4`) stay O(1) on hits.
## Storage layout
Each cache entry is bucketed into 5°×5° spatial chunks matching the layout
used by `Microwaveprop.Weather.GridCache`. The ETS value for
`{band_mhz, valid_time}` is `%{{lat_band, lon_band} => %{{lat, lon} => score}}`.
That way `fetch_bounds/3` only walks the chunks that intersect the
requested viewport instead of the full 92k-cell CONUS map; `fetch_point/4`
reads exactly one chunk.
"""
use GenServer
@ -24,14 +30,16 @@ defmodule Microwaveprop.Propagation.ScoreCache do
@table :propagation_score_cache
@topic "propagation:cache"
@pubsub Microwaveprop.PubSub
@chunk_step 5
# Hard cap on cached `{band_mhz, valid_time}` entries. Each value is a
# 92k-cell `Map.new` weighing ~1.94 MiB; without a cap the eager warm
# path materialises 23 bands × 19 forecast hours = 437 entries per pod
# (~850 MiB) and OOMs the BEAM. The cap keeps the LiveView hot path
# warm (one band × the hour or two the user is actively scrubbing)
# and lets the rest fall back to direct `.prop` reads via
# `read_from_disk_and_cache/3`, which is sub-100 ms per file.
# 92k-cell map (now bucketed into 5°×5° chunks) on the order of ~2 MiB;
# without a cap the eager warm path materialises 23 bands × 19 forecast
# hours = 437 entries per pod (~850 MiB) and OOMs the BEAM. The cap
# keeps the LiveView hot path warm (one band × the hour or two the
# user is actively scrubbing) and lets the rest fall back to direct
# `.prop` reads via `read_from_disk_and_cache/3`, which is sub-100 ms
# per file.
@max_entries 32
@type score :: %{lat: float(), lon: float(), score: non_neg_integer()}
@ -47,7 +55,7 @@ defmodule Microwaveprop.Propagation.ScoreCache do
@spec fetch(non_neg_integer(), DateTime.t()) :: {:ok, [score()]} | :miss
def fetch(band_mhz, valid_time) do
case :ets.lookup(@table, {band_mhz, valid_time}) do
[{_, grid}] -> {:ok, grid_to_list(grid)}
[{_, chunked}] -> {:ok, chunks_to_list(chunked)}
[] -> :miss
end
end
@ -56,7 +64,7 @@ defmodule Microwaveprop.Propagation.ScoreCache do
{:ok, [score()]} | :miss
def fetch_bounds(band_mhz, valid_time, bounds) do
case :ets.lookup(@table, {band_mhz, valid_time}) do
[{_, grid}] -> {:ok, grid_to_filtered_list(grid, bounds)}
[{_, chunked}] -> {:ok, chunks_filtered_to_list(chunked, bounds)}
[] -> :miss
end
end
@ -71,8 +79,10 @@ defmodule Microwaveprop.Propagation.ScoreCache do
{:ok, non_neg_integer()} | :miss
def fetch_point(band_mhz, valid_time, lat, lon) do
case :ets.lookup(@table, {band_mhz, valid_time}) do
[{_, grid}] ->
case Map.get(grid, {lat, lon}) do
[{_, chunked}] ->
chunk_key = chunk_key_for(lat, lon)
case chunked |> Map.get(chunk_key, %{}) |> Map.get({lat, lon}) do
nil -> :miss
score -> {:ok, score}
end
@ -88,8 +98,8 @@ defmodule Microwaveprop.Propagation.ScoreCache do
@spec put(non_neg_integer(), DateTime.t(), [score()]) :: :ok
def put(band_mhz, valid_time, scores) do
grid = list_to_grid(scores)
:ets.insert(@table, {{band_mhz, valid_time}, grid})
chunked = list_to_chunks(scores)
:ets.insert(@table, {{band_mhz, valid_time}, chunked})
enforce_capacity()
:ok
end
@ -207,27 +217,51 @@ defmodule Microwaveprop.Propagation.ScoreCache do
# ---------- Internal ----------
defp list_to_grid(scores) do
Map.new(scores, fn %{lat: lat, lon: lon, score: score} -> {{lat, lon}, score} end)
end
defp grid_to_list(grid) do
Enum.map(grid, fn {{lat, lon}, score} -> %{lat: lat, lon: lon, score: score} end)
end
defp grid_to_filtered_list(grid, nil), do: grid_to_list(grid)
defp grid_to_filtered_list(grid, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
# Single pass over 92k cells: the former Enum.filter |> Enum.map
# walked the map twice and allocated an intermediate list of
# {key, value} tuples between them. Enum.reduce emits only the
# in-bounds result maps directly.
Enum.reduce(grid, [], fn {{lat, lon}, score}, acc ->
if lat >= s and lat <= n and lon >= w and lon <= e do
[%{lat: lat, lon: lon, score: score} | acc]
else
acc
end
defp list_to_chunks(scores) do
Enum.reduce(scores, %{}, fn %{lat: lat, lon: lon, score: score}, acc ->
key = chunk_key_for(lat, lon)
chunk = Map.get(acc, key, %{})
Map.put(acc, key, Map.put(chunk, {lat, lon}, score))
end)
end
defp chunks_to_list(chunked) do
Enum.flat_map(chunked, fn {_chunk_key, cells} ->
Enum.map(cells, fn {{lat, lon}, score} ->
%{lat: lat, lon: lon, score: score}
end)
end)
end
defp chunks_filtered_to_list(chunked, nil), do: chunks_to_list(chunked)
defp chunks_filtered_to_list(chunked, %{"south" => s, "north" => n, "west" => w, "east" => e} = bounds) do
chunked
|> Enum.filter(fn {chunk_key, _} -> chunk_intersects_bounds?(chunk_key, bounds) end)
|> Enum.flat_map(fn {_, cells} ->
for {{lat, lon}, score} <- cells,
lat >= s,
lat <= n,
lon >= w,
lon <= e,
do: %{lat: lat, lon: lon, score: score}
end)
end
defp chunk_key_for(lat, lon) do
{chunk_band(lat * 1.0), chunk_band(lon * 1.0)}
end
defp chunk_band(value) when is_float(value) do
(value / @chunk_step) |> Float.floor() |> trunc()
end
defp chunk_intersects_bounds?({lat_band, lon_band}, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
chunk_south = lat_band * @chunk_step
chunk_north = chunk_south + @chunk_step
chunk_west = lon_band * @chunk_step
chunk_east = chunk_west + @chunk_step
chunk_north >= s and chunk_south <= n and chunk_east >= w and chunk_west <= e
end
end

36
test/microwaveprop/propagation/score_cache_test.exs
View file

@ -76,6 +76,42 @@ defmodule Microwaveprop.Propagation.ScoreCacheTest do
assert {:ok, [%{lat: 32.0, lon: -97.0, score: 75}]} =
ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 12:00:00Z], bounds)
end
test "returns cells across multiple 5°×5° chunks when viewport spans them" do
# Three points spanning three distinct 5°×5° chunks on the lat axis
# (~30, ~35, ~40) and three on the lon axis (~-100, ~-95, ~-75).
# A viewport covering all three must return them all even though
# each lives in its own chunk under the bucketed storage layout.
scores = [
%{lat: 31.0, lon: -99.0, score: 10},
%{lat: 36.0, lon: -94.0, score: 20},
%{lat: 41.0, lon: -74.0, score: 30}
]
ScoreCache.put(10_000, ~U[2026-04-12 13:00:00Z], scores)
bounds = %{"south" => 30.0, "north" => 45.0, "west" => -100.0, "east" => -70.0}
assert {:ok, returned} =
ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 13:00:00Z], bounds)
assert Enum.sort_by(returned, & &1.score) == Enum.sort_by(scores, & &1.score)
end
test "narrow viewport excludes cells in non-overlapping chunks" do
scores = [
%{lat: 31.0, lon: -99.0, score: 10},
%{lat: 41.0, lon: -74.0, score: 30}
]
ScoreCache.put(10_000, ~U[2026-04-12 13:00:00Z], scores)
# Viewport entirely within the (lat 30-35, lon -100..-95) chunk.
bounds = %{"south" => 30.0, "north" => 33.0, "west" => -100.0, "east" => -97.0}
assert {:ok, [%{score: 10}]} =
ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 13:00:00Z], bounds)
end
end
describe "prune_older_than/1" do