Graham McIntire
c192d6fd9e
Hot pods were restart-looping every ~25 minutes on liveness probe timeouts. Root cause: ScoreCacheReconciler mirrored every .prop file from NFS into ETS, including 7 days of GEFS Day 2-7 forecasts. With 23 bands × 43 valid_times × ~2 MB per grid = 1.95 GB in the propagation_score_cache table alone; GC sweeps starved the scheduler enough that /live dropped its 3 s budget. The /map UI only ever requests the [now-1h, now+18h] window. Share that bound as Propagation.hot_cache_window/0 and apply it in the reconciler's disk-scan path plus NotifyListener's post-warm prune. Long-horizon GEFS files stay on disk and are still served via the lazy read_from_disk_and_cache path when requested directly. Adds ScoreCache.prune_outside_window/2 (inclusive bounds) and updates the reconciler tests to use relative-to-now timestamps since hardcoded fixture dates now drift out of window.
187 lines
7 KiB
Elixir
187 lines
7 KiB
Elixir
defmodule Microwaveprop.Propagation.ScoreCacheTest do
|
|
use ExUnit.Case, async: false
|
|
|
|
alias Microwaveprop.Propagation.ScoreCache
|
|
|
|
setup do
|
|
ScoreCache.clear()
|
|
:ok
|
|
end
|
|
|
|
describe "fetch/2" do
|
|
test "returns :miss when nothing is cached" do
|
|
assert ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z]) == :miss
|
|
end
|
|
|
|
test "returns cached scores after put" do
|
|
scores = [%{lat: 32.0, lon: -97.0, score: 75}]
|
|
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], scores)
|
|
|
|
assert {:ok, [%{lat: 32.0, lon: -97.0, score: 75}]} =
|
|
ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
|
|
end
|
|
|
|
test "isolates entries by band" do
|
|
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [%{lat: 1.0, lon: 1.0, score: 10}])
|
|
ScoreCache.put(24_000, ~U[2026-04-12 12:00:00Z], [%{lat: 2.0, lon: 2.0, score: 20}])
|
|
|
|
assert {:ok, [%{score: 10}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
|
|
assert {:ok, [%{score: 20}]} = ScoreCache.fetch(24_000, ~U[2026-04-12 12:00:00Z])
|
|
end
|
|
|
|
test "isolates entries by valid_time" do
|
|
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [%{lat: 1.0, lon: 1.0, score: 10}])
|
|
ScoreCache.put(10_000, ~U[2026-04-12 13:00:00Z], [%{lat: 2.0, lon: 2.0, score: 20}])
|
|
|
|
assert {:ok, [%{score: 10}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
|
|
assert {:ok, [%{score: 20}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 13:00:00Z])
|
|
end
|
|
end
|
|
|
|
describe "fetch_bounds/3" do
|
|
setup do
|
|
scores = [
|
|
%{lat: 32.0, lon: -97.0, score: 75},
|
|
%{lat: 40.0, lon: -74.0, score: 50},
|
|
%{lat: 34.0, lon: -98.0, score: 60}
|
|
]
|
|
|
|
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], scores)
|
|
:ok
|
|
end
|
|
|
|
test "returns :miss when no entry for band/time" do
|
|
bounds = %{"south" => 30.0, "north" => 50.0, "west" => -100.0, "east" => -70.0}
|
|
assert ScoreCache.fetch_bounds(24_000, ~U[2026-04-12 12:00:00Z], bounds) == :miss
|
|
end
|
|
|
|
test "returns all scores when bounds is nil" do
|
|
assert {:ok, all} = ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 12:00:00Z], nil)
|
|
assert length(all) == 3
|
|
end
|
|
|
|
test "returns only scores inside the bounds" do
|
|
bounds = %{"south" => 31.0, "north" => 35.0, "west" => -100.0, "east" => -95.0}
|
|
|
|
assert {:ok, filtered} =
|
|
ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 12:00:00Z], bounds)
|
|
|
|
assert length(filtered) == 2
|
|
assert Enum.all?(filtered, fn %{lat: lat} -> lat >= 31.0 and lat <= 35.0 end)
|
|
end
|
|
|
|
test "includes scores exactly on the boundary" do
|
|
bounds = %{"south" => 32.0, "north" => 32.0, "west" => -97.0, "east" => -97.0}
|
|
|
|
assert {:ok, [%{lat: 32.0, lon: -97.0, score: 75}]} =
|
|
ScoreCache.fetch_bounds(10_000, ~U[2026-04-12 12:00:00Z], bounds)
|
|
end
|
|
end
|
|
|
|
describe "prune_older_than/1" do
|
|
test "removes entries whose valid_time is strictly before the cutoff" do
|
|
ScoreCache.put(10_000, ~U[2026-04-12 10:00:00Z], [%{lat: 1.0, lon: 1.0, score: 1}])
|
|
ScoreCache.put(10_000, ~U[2026-04-12 14:00:00Z], [%{lat: 2.0, lon: 2.0, score: 2}])
|
|
|
|
ScoreCache.prune_older_than(~U[2026-04-12 12:00:00Z])
|
|
|
|
assert ScoreCache.fetch(10_000, ~U[2026-04-12 10:00:00Z]) == :miss
|
|
assert {:ok, [%{score: 2}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 14:00:00Z])
|
|
end
|
|
|
|
test "keeps entries whose valid_time equals the cutoff" do
|
|
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [%{lat: 1.0, lon: 1.0, score: 1}])
|
|
ScoreCache.prune_older_than(~U[2026-04-12 12:00:00Z])
|
|
assert {:ok, [%{score: 1}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
|
|
end
|
|
end
|
|
|
|
describe "prune_outside_window/2" do
|
|
test "removes entries strictly outside [past_cutoff, future_cutoff]" do
|
|
ScoreCache.put(10_000, ~U[2026-04-12 08:00:00Z], [%{lat: 1.0, lon: 1.0, score: 1}])
|
|
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [%{lat: 2.0, lon: 2.0, score: 2}])
|
|
ScoreCache.put(10_000, ~U[2026-04-12 20:00:00Z], [%{lat: 3.0, lon: 3.0, score: 3}])
|
|
|
|
removed =
|
|
ScoreCache.prune_outside_window(~U[2026-04-12 10:00:00Z], ~U[2026-04-12 18:00:00Z])
|
|
|
|
assert removed == 2
|
|
assert ScoreCache.fetch(10_000, ~U[2026-04-12 08:00:00Z]) == :miss
|
|
assert {:ok, [%{score: 2}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
|
|
assert ScoreCache.fetch(10_000, ~U[2026-04-12 20:00:00Z]) == :miss
|
|
end
|
|
|
|
test "keeps entries equal to either cutoff (inclusive bounds)" do
|
|
ScoreCache.put(10_000, ~U[2026-04-12 10:00:00Z], [%{lat: 1.0, lon: 1.0, score: 1}])
|
|
ScoreCache.put(10_000, ~U[2026-04-12 18:00:00Z], [%{lat: 2.0, lon: 2.0, score: 2}])
|
|
|
|
removed =
|
|
ScoreCache.prune_outside_window(~U[2026-04-12 10:00:00Z], ~U[2026-04-12 18:00:00Z])
|
|
|
|
assert removed == 0
|
|
assert {:ok, _} = ScoreCache.fetch(10_000, ~U[2026-04-12 10:00:00Z])
|
|
assert {:ok, _} = ScoreCache.fetch(10_000, ~U[2026-04-12 18:00:00Z])
|
|
end
|
|
end
|
|
|
|
describe "fetch_point/4" do
|
|
test "returns :miss when nothing cached" do
|
|
assert ScoreCache.fetch_point(10_000, ~U[2026-04-12 12:00:00Z], 32.0, -97.0) == :miss
|
|
end
|
|
|
|
test "returns :miss when the point is not in the cached grid" do
|
|
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [
|
|
%{lat: 32.0, lon: -97.0, score: 75}
|
|
])
|
|
|
|
assert ScoreCache.fetch_point(10_000, ~U[2026-04-12 12:00:00Z], 40.0, -74.0) == :miss
|
|
end
|
|
|
|
test "returns the score for a cached point" do
|
|
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [
|
|
%{lat: 32.0, lon: -97.0, score: 75},
|
|
%{lat: 33.0, lon: -97.0, score: 80}
|
|
])
|
|
|
|
assert {:ok, 75} = ScoreCache.fetch_point(10_000, ~U[2026-04-12 12:00:00Z], 32.0, -97.0)
|
|
assert {:ok, 80} = ScoreCache.fetch_point(10_000, ~U[2026-04-12 12:00:00Z], 33.0, -97.0)
|
|
end
|
|
end
|
|
|
|
describe "valid_times/1" do
|
|
test "returns empty list when nothing cached" do
|
|
assert ScoreCache.valid_times(10_000) == []
|
|
end
|
|
|
|
test "returns sorted valid_times for a band" do
|
|
ScoreCache.put(10_000, ~U[2026-04-12 14:00:00Z], [])
|
|
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [])
|
|
ScoreCache.put(10_000, ~U[2026-04-12 13:00:00Z], [])
|
|
|
|
assert ScoreCache.valid_times(10_000) == [
|
|
~U[2026-04-12 12:00:00Z],
|
|
~U[2026-04-12 13:00:00Z],
|
|
~U[2026-04-12 14:00:00Z]
|
|
]
|
|
end
|
|
|
|
test "only returns times for the requested band" do
|
|
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], [])
|
|
ScoreCache.put(24_000, ~U[2026-04-12 13:00:00Z], [])
|
|
|
|
assert ScoreCache.valid_times(10_000) == [~U[2026-04-12 12:00:00Z]]
|
|
assert ScoreCache.valid_times(24_000) == [~U[2026-04-12 13:00:00Z]]
|
|
end
|
|
end
|
|
|
|
describe "broadcast_put/3" do
|
|
test "populates the cache on the local node" do
|
|
scores = [%{lat: 32.0, lon: -97.0, score: 80}]
|
|
ScoreCache.broadcast_put(10_000, ~U[2026-04-12 12:00:00Z], scores)
|
|
ScoreCache.sync()
|
|
|
|
assert {:ok, [%{lat: 32.0, lon: -97.0, score: 80}]} =
|
|
ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
|
|
end
|
|
end
|
|
end
|