prop/test/microwaveprop/propagation/score_cache_test.exs
Graham McIntire 4f82bd691e
perf(prop): cap ScoreCache LRU + drop eager warmers to fix hot-pod OOMs
ScoreCache held 437 entries × ~1.94 MiB each (~850 MiB per pod) of
{band_mhz, valid_time} grids — data already on NFS as compact .prop
files. NotifyListener and ScoreCacheReconciler both eagerly materialised
every band into ETS on each Rust completion / 60s sweep, so 5 hot
replicas wasted ~4.2 GiB of redundant cache and OOMed at 6 GiB.

Bound the cache to 32 entries with eviction by oldest valid_time, drop
the eager warm loops, and let LiveView callers lazy-fill via the
existing ScoresFile read path. ScoreCacheReconciler had no remaining
purpose and is deleted; runbook + prom_ex counters updated to match.

Steady-state cache footprint ~60 MiB per pod instead of ~850 MiB.
2026-04-24 18:49:36 -05:00

243 lines
9 KiB
Elixir

defmodule Microwaveprop.Propagation.ScoreCacheTest do
use ExUnit.Case, async: false
alias Microwaveprop.Propagation.ScoreCache
setup do
ScoreCache.clear()
:ok
end
describe "fetch/2" do
test "returns :miss when nothing is cached" do
assert ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z]) == :miss
end
test "returns cached scores after put" do
scores = [%{lat: 32.0, lon: -97.0, score: 75}]
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], scores)
assert {:ok, [%{lat: 32.0, lon: -97.0, score: 75}]} =
ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
end
test "isolates entries by band" do
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [%{lat: 1.0, lon: 1.0, score: 10}])
ScoreCache.put(24_000, ~U[2026-04-12 12:00:00Z], [%{lat: 2.0, lon: 2.0, score: 20}])
assert {:ok, [%{score: 10}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
assert {:ok, [%{score: 20}]} = ScoreCache.fetch(24_000, ~U[2026-04-12 12:00:00Z])
end
test "isolates entries by valid_time" do
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [%{lat: 1.0, lon: 1.0, score: 10}])
ScoreCache.put(10_000, ~U[2026-04-12 13:00:00Z], [%{lat: 2.0, lon: 2.0, score: 20}])
assert {:ok, [%{score: 10}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
assert {:ok, [%{score: 20}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 13:00:00Z])
end
end
describe "fetch_bounds/3" do
setup do
scores = [
%{lat: 32.0, lon: -97.0, score: 75},
%{lat: 40.0, lon: -74.0, score: 50},
%{lat: 34.0, lon: -98.0, score: 60}
]
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], scores)
:ok
end
test "returns :miss when no entry for band/time" do
bounds = %{"south" => 30.0, "north" => 50.0, "west" => -100.0, "east" => -70.0}
assert ScoreCache.fetch_bounds(24_000, ~U[2026-04-12 12:00:00Z], bounds) == :miss
end
test "returns all scores when bounds is nil" do
assert {:ok, all} = ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 12:00:00Z], nil)
assert length(all) == 3
end
test "returns only scores inside the bounds" do
bounds = %{"south" => 31.0, "north" => 35.0, "west" => -100.0, "east" => -95.0}
assert {:ok, filtered} =
ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 12:00:00Z], bounds)
assert length(filtered) == 2
assert Enum.all?(filtered, fn %{lat: lat} -> lat >= 31.0 and lat <= 35.0 end)
end
test "includes scores exactly on the boundary" do
bounds = %{"south" => 32.0, "north" => 32.0, "west" => -97.0, "east" => -97.0}
assert {:ok, [%{lat: 32.0, lon: -97.0, score: 75}]} =
ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 12:00:00Z], bounds)
end
end
describe "prune_older_than/1" do
test "removes entries whose valid_time is strictly before the cutoff" do
ScoreCache.put(10_000, ~U[2026-04-12 10:00:00Z], [%{lat: 1.0, lon: 1.0, score: 1}])
ScoreCache.put(10_000, ~U[2026-04-12 14:00:00Z], [%{lat: 2.0, lon: 2.0, score: 2}])
ScoreCache.prune_older_than(~U[2026-04-12 12:00:00Z])
assert ScoreCache.fetch(10_000, ~U[2026-04-12 10:00:00Z]) == :miss
assert {:ok, [%{score: 2}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 14:00:00Z])
end
test "keeps entries whose valid_time equals the cutoff" do
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [%{lat: 1.0, lon: 1.0, score: 1}])
ScoreCache.prune_older_than(~U[2026-04-12 12:00:00Z])
assert {:ok, [%{score: 1}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
end
end
describe "prune_outside_window/2" do
test "removes entries strictly outside [past_cutoff, future_cutoff]" do
ScoreCache.put(10_000, ~U[2026-04-12 08:00:00Z], [%{lat: 1.0, lon: 1.0, score: 1}])
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [%{lat: 2.0, lon: 2.0, score: 2}])
ScoreCache.put(10_000, ~U[2026-04-12 20:00:00Z], [%{lat: 3.0, lon: 3.0, score: 3}])
removed =
ScoreCache.prune_outside_window(~U[2026-04-12 10:00:00Z], ~U[2026-04-12 18:00:00Z])
assert removed == 2
assert ScoreCache.fetch(10_000, ~U[2026-04-12 08:00:00Z]) == :miss
assert {:ok, [%{score: 2}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
assert ScoreCache.fetch(10_000, ~U[2026-04-12 20:00:00Z]) == :miss
end
test "keeps entries equal to either cutoff (inclusive bounds)" do
ScoreCache.put(10_000, ~U[2026-04-12 10:00:00Z], [%{lat: 1.0, lon: 1.0, score: 1}])
ScoreCache.put(10_000, ~U[2026-04-12 18:00:00Z], [%{lat: 2.0, lon: 2.0, score: 2}])
removed =
ScoreCache.prune_outside_window(~U[2026-04-12 10:00:00Z], ~U[2026-04-12 18:00:00Z])
assert removed == 0
assert {:ok, _} = ScoreCache.fetch(10_000, ~U[2026-04-12 10:00:00Z])
assert {:ok, _} = ScoreCache.fetch(10_000, ~U[2026-04-12 18:00:00Z])
end
end
describe "fetch_point/4" do
test "returns :miss when nothing cached" do
assert ScoreCache.fetch_point(10_000, ~U[2026-04-12 12:00:00Z], 32.0, -97.0) == :miss
end
test "returns :miss when the point is not in the cached grid" do
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [
%{lat: 32.0, lon: -97.0, score: 75}
])
assert ScoreCache.fetch_point(10_000, ~U[2026-04-12 12:00:00Z], 40.0, -74.0) == :miss
end
test "returns the score for a cached point" do
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [
%{lat: 32.0, lon: -97.0, score: 75},
%{lat: 33.0, lon: -97.0, score: 80}
])
assert {:ok, 75} = ScoreCache.fetch_point(10_000, ~U[2026-04-12 12:00:00Z], 32.0, -97.0)
assert {:ok, 80} = ScoreCache.fetch_point(10_000, ~U[2026-04-12 12:00:00Z], 33.0, -97.0)
end
end
describe "valid_times/1" do
test "returns empty list when nothing cached" do
assert ScoreCache.valid_times(10_000) == []
end
test "returns sorted valid_times for a band" do
ScoreCache.put(10_000, ~U[2026-04-12 14:00:00Z], [])
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [])
ScoreCache.put(10_000, ~U[2026-04-12 13:00:00Z], [])
assert ScoreCache.valid_times(10_000) == [
~U[2026-04-12 12:00:00Z],
~U[2026-04-12 13:00:00Z],
~U[2026-04-12 14:00:00Z]
]
end
test "only returns times for the requested band" do
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [])
ScoreCache.put(24_000, ~U[2026-04-12 13:00:00Z], [])
assert ScoreCache.valid_times(10_000) == [~U[2026-04-12 12:00:00Z]]
assert ScoreCache.valid_times(24_000) == [~U[2026-04-12 13:00:00Z]]
end
end
describe "broadcast_put/3" do
test "populates the cache on the local node" do
scores = [%{lat: 32.0, lon: -97.0, score: 80}]
ScoreCache.broadcast_put(10_000, ~U[2026-04-12 12:00:00Z], scores)
ScoreCache.sync()
assert {:ok, [%{lat: 32.0, lon: -97.0, score: 80}]} =
ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
end
end
describe "max-entries eviction" do
test "caps entries at max_entries, evicting the oldest valid_time" do
cap = ScoreCache.max_entries()
base = ~U[2026-04-12 00:00:00Z]
score = [%{lat: 1.0, lon: 1.0, score: 1}]
# Insert cap+1 entries, each at a distinct valid_time, oldest first.
for hour <- 0..cap do
ScoreCache.put(10_000, DateTime.add(base, hour * 3600, :second), score)
end
# Size never exceeds the cap.
assert :ets.info(:propagation_score_cache, :size) == cap
# The oldest valid_time was evicted.
assert ScoreCache.fetch(10_000, base) == :miss
# The newest is retained.
newest = DateTime.add(base, cap * 3600, :second)
assert {:ok, [%{score: 1}]} = ScoreCache.fetch(10_000, newest)
end
test "evicts across bands by valid_time when at capacity" do
cap = ScoreCache.max_entries()
base = ~U[2026-04-12 00:00:00Z]
score = [%{lat: 1.0, lon: 1.0, score: 1}]
# Fill cap entries on band A at older valid_times.
for hour <- 0..(cap - 1) do
ScoreCache.put(10_000, DateTime.add(base, hour * 3600, :second), score)
end
assert :ets.info(:propagation_score_cache, :size) == cap
# One more on band B at a newer valid_time should evict the oldest
# band-A entry, not the new band-B one.
newer = DateTime.add(base, cap * 3600, :second)
ScoreCache.put(24_000, newer, score)
assert :ets.info(:propagation_score_cache, :size) == cap
assert ScoreCache.fetch(10_000, base) == :miss
assert {:ok, _} = ScoreCache.fetch(24_000, newer)
end
test "replacing an existing key does not change the cache size" do
base = ~U[2026-04-12 00:00:00Z]
ScoreCache.put(10_000, base, [%{lat: 1.0, lon: 1.0, score: 1}])
size_before = :ets.info(:propagation_score_cache, :size)
ScoreCache.put(10_000, base, [%{lat: 1.0, lon: 1.0, score: 99}])
assert :ets.info(:propagation_score_cache, :size) == size_before
assert {:ok, [%{score: 99}]} = ScoreCache.fetch(10_000, base)
end
end
end