Graham McIntire
b67c0e88c0
Each /weather/tiles request was running the full ScalarFile decode path (File.read → gunzip → Msgpax.unpack → normalize) for any non-analysis valid_time, because GridCache deliberately skipped forecast hours to keep memory low. With 21 simultaneous viewport tiles all hitting the same few chunk files, each tile took 1.5-3.2s of contended IO + CPU. On a GridCache miss when a ScalarFile exists, hydrate GridCache from the file once (deduped via the existing claim_fill primitive) and serve all subsequent reads from ETS. Bound memory with prune_keep_latest/1 — keep the 24 most-recently-touched valid_times, which covers a full HRRR run (analysis + 18 forecasts) plus a few stragglers from the previous run. In the steady state, the first viewport read for a given forecast hour takes one full ScalarFile decode (~50-100 ms for 92k cells); every subsequent tile, point lookup, and viewport read for the same hour serves from ETS in <1 ms.
224 lines
7.4 KiB
Elixir
224 lines
7.4 KiB
Elixir
defmodule Microwaveprop.Weather.GridCacheTest do
|
|
use ExUnit.Case, async: false
|
|
|
|
alias Microwaveprop.Weather.GridCache
|
|
|
|
setup do
|
|
GridCache.clear()
|
|
:ok
|
|
end
|
|
|
|
describe "fetch/1" do
|
|
test "returns :miss when nothing is cached" do
|
|
assert GridCache.fetch(~U[2026-04-12 12:00:00Z]) == :miss
|
|
end
|
|
|
|
test "returns cached rows after put" do
|
|
rows = [%{lat: 32.0, lon: -97.0, temperature: 25.0}]
|
|
GridCache.put(~U[2026-04-12 12:00:00Z], rows)
|
|
assert {:ok, [%{lat: 32.0, lon: -97.0}]} = GridCache.fetch(~U[2026-04-12 12:00:00Z])
|
|
end
|
|
end
|
|
|
|
describe "fetch_bounds/2" do
|
|
setup do
|
|
rows = [
|
|
%{lat: 32.0, lon: -97.0, temperature: 25.0},
|
|
%{lat: 40.0, lon: -74.0, temperature: 20.0},
|
|
%{lat: 34.0, lon: -98.0, temperature: 28.0}
|
|
]
|
|
|
|
GridCache.put(~U[2026-04-12 12:00:00Z], rows)
|
|
:ok
|
|
end
|
|
|
|
test "returns all rows when bounds are nil" do
|
|
assert {:ok, list} = GridCache.fetch_bounds(~U[2026-04-12 12:00:00Z], nil)
|
|
assert length(list) == 3
|
|
end
|
|
|
|
test "returns only rows inside the bounds" do
|
|
bounds = %{"south" => 31.0, "north" => 35.0, "west" => -100.0, "east" => -95.0}
|
|
assert {:ok, list} = GridCache.fetch_bounds(~U[2026-04-12 12:00:00Z], bounds)
|
|
assert length(list) == 2
|
|
end
|
|
|
|
test "returns :miss when the valid_time is not cached" do
|
|
bounds = %{"south" => 0.0, "north" => 90.0, "west" => -180.0, "east" => 0.0}
|
|
assert GridCache.fetch_bounds(~U[2099-01-01 00:00:00Z], bounds) == :miss
|
|
end
|
|
end
|
|
|
|
describe "fetch_point/3" do
|
|
setup do
|
|
rows = [
|
|
%{lat: 32.0, lon: -97.0, temperature: 25.0},
|
|
%{lat: 33.0, lon: -97.0, temperature: 27.0}
|
|
]
|
|
|
|
GridCache.put(~U[2026-04-12 12:00:00Z], rows)
|
|
:ok
|
|
end
|
|
|
|
test "returns the cached row for a known point" do
|
|
assert {:ok, %{temperature: 25.0}} =
|
|
GridCache.fetch_point(~U[2026-04-12 12:00:00Z], 32.0, -97.0)
|
|
end
|
|
|
|
test "returns :miss for an unknown point" do
|
|
assert GridCache.fetch_point(~U[2026-04-12 12:00:00Z], 40.0, -74.0) == :miss
|
|
end
|
|
end
|
|
|
|
describe "claim_fill/1 and release_fill/1" do
|
|
@valid_time ~U[2026-04-21 12:00:00Z]
|
|
|
|
test "first claimer wins, subsequent claimers see :in_progress" do
|
|
assert GridCache.claim_fill(@valid_time) == true
|
|
assert GridCache.claim_fill(@valid_time) == false
|
|
end
|
|
|
|
test "release_fill allows a re-claim" do
|
|
assert GridCache.claim_fill(@valid_time) == true
|
|
:ok = GridCache.release_fill(@valid_time)
|
|
assert GridCache.claim_fill(@valid_time) == true
|
|
end
|
|
|
|
test "lock is auto-released when the claimer process crashes" do
|
|
parent = self()
|
|
|
|
{:ok, claimer} =
|
|
Task.start(fn ->
|
|
true = GridCache.claim_fill(@valid_time)
|
|
send(parent, :claimed)
|
|
# Block until we're killed
|
|
Process.sleep(:infinity)
|
|
end)
|
|
|
|
receive do
|
|
:claimed -> :ok
|
|
after
|
|
1000 -> flunk("claimer never signaled")
|
|
end
|
|
|
|
# Lock is held by the crashed process
|
|
assert GridCache.claim_fill(@valid_time) == false
|
|
|
|
# Kill the claimer
|
|
ref = Process.monitor(claimer)
|
|
Process.exit(claimer, :kill)
|
|
|
|
receive do
|
|
{:DOWN, ^ref, :process, ^claimer, _} -> :ok
|
|
after
|
|
1000 -> flunk("claimer didn't die")
|
|
end
|
|
|
|
# Flush the GenServer so the :DOWN handler runs before we re-probe ETS.
|
|
:ok = GridCache.sync()
|
|
|
|
assert GridCache.claim_fill(@valid_time) == true
|
|
end
|
|
end
|
|
|
|
describe "lookup telemetry" do
|
|
@event [:microwaveprop, :weather, :grid_cache, :lookup]
|
|
|
|
setup do
|
|
test_pid = self()
|
|
handler_id = {__MODULE__, :grid_cache_lookup, System.unique_integer([:positive])}
|
|
|
|
:telemetry.attach(
|
|
handler_id,
|
|
@event,
|
|
fn event, measurements, metadata, _config ->
|
|
send(test_pid, {:telemetry, event, measurements, metadata})
|
|
end,
|
|
nil
|
|
)
|
|
|
|
on_exit(fn -> :telemetry.detach(handler_id) end)
|
|
:ok
|
|
end
|
|
|
|
test "fetch/1 emits hit: true when the valid_time is cached" do
|
|
GridCache.put(~U[2026-04-12 12:00:00Z], [%{lat: 32.0, lon: -97.0}])
|
|
{:ok, _} = GridCache.fetch(~U[2026-04-12 12:00:00Z])
|
|
assert_receive {:telemetry, @event, %{}, %{hit: true}}
|
|
end
|
|
|
|
test "fetch/1 emits hit: false when the valid_time is absent" do
|
|
:miss = GridCache.fetch(~U[2099-01-01 00:00:00Z])
|
|
assert_receive {:telemetry, @event, %{}, %{hit: false}}
|
|
end
|
|
|
|
test "fetch_bounds/2 emits hit: true on a cached valid_time" do
|
|
GridCache.put(~U[2026-04-12 12:00:00Z], [%{lat: 32.0, lon: -97.0}])
|
|
bounds = %{"south" => 0.0, "north" => 90.0, "west" => -180.0, "east" => 0.0}
|
|
{:ok, _} = GridCache.fetch_bounds(~U[2026-04-12 12:00:00Z], bounds)
|
|
assert_receive {:telemetry, @event, %{}, %{hit: true}}
|
|
end
|
|
|
|
test "fetch_bounds/2 emits hit: false when the valid_time is absent" do
|
|
bounds = %{"south" => 0.0, "north" => 90.0, "west" => -180.0, "east" => 0.0}
|
|
:miss = GridCache.fetch_bounds(~U[2099-01-01 00:00:00Z], bounds)
|
|
assert_receive {:telemetry, @event, %{}, %{hit: false}}
|
|
end
|
|
|
|
test "fetch_point/3 emits hit: true when the point exists" do
|
|
GridCache.put(~U[2026-04-12 12:00:00Z], [%{lat: 32.0, lon: -97.0}])
|
|
{:ok, _} = GridCache.fetch_point(~U[2026-04-12 12:00:00Z], 32.0, -97.0)
|
|
assert_receive {:telemetry, @event, %{}, %{hit: true}}
|
|
end
|
|
|
|
test "fetch_point/3 emits hit: false when the valid_time is missing" do
|
|
:miss = GridCache.fetch_point(~U[2099-01-01 00:00:00Z], 1.0, 2.0)
|
|
assert_receive {:telemetry, @event, %{}, %{hit: false}}
|
|
end
|
|
|
|
test "fetch_point/3 emits hit: false when the point is missing from a cached grid" do
|
|
GridCache.put(~U[2026-04-12 12:00:00Z], [%{lat: 32.0, lon: -97.0}])
|
|
:miss = GridCache.fetch_point(~U[2026-04-12 12:00:00Z], 40.0, -74.0)
|
|
assert_receive {:telemetry, @event, %{}, %{hit: false}}
|
|
end
|
|
end
|
|
|
|
describe "latest_valid_time/0" do
|
|
test "returns the most recent cached valid_time" do
|
|
GridCache.put(~U[2026-04-12 10:00:00Z], [])
|
|
GridCache.put(~U[2026-04-12 14:00:00Z], [])
|
|
GridCache.put(~U[2026-04-12 12:00:00Z], [])
|
|
assert GridCache.latest_valid_time() == ~U[2026-04-12 14:00:00Z]
|
|
end
|
|
|
|
test "returns nil when nothing is cached" do
|
|
assert GridCache.latest_valid_time() == nil
|
|
end
|
|
end
|
|
|
|
describe "prune_keep_latest/1" do
|
|
test "drops oldest valid_times beyond the cap" do
|
|
GridCache.put(~U[2026-04-12 10:00:00Z], [%{lat: 32.0, lon: -97.0}])
|
|
GridCache.put(~U[2026-04-12 11:00:00Z], [%{lat: 32.0, lon: -97.0}])
|
|
GridCache.put(~U[2026-04-12 12:00:00Z], [%{lat: 32.0, lon: -97.0}])
|
|
GridCache.put(~U[2026-04-12 13:00:00Z], [%{lat: 32.0, lon: -97.0}])
|
|
|
|
removed = GridCache.prune_keep_latest(2)
|
|
assert removed == 2
|
|
|
|
assert GridCache.fetch(~U[2026-04-12 10:00:00Z]) == :miss
|
|
assert GridCache.fetch(~U[2026-04-12 11:00:00Z]) == :miss
|
|
assert {:ok, _} = GridCache.fetch(~U[2026-04-12 12:00:00Z])
|
|
assert {:ok, _} = GridCache.fetch(~U[2026-04-12 13:00:00Z])
|
|
end
|
|
|
|
test "is a no-op when the cap is not exceeded" do
|
|
GridCache.put(~U[2026-04-12 10:00:00Z], [%{lat: 32.0, lon: -97.0}])
|
|
GridCache.put(~U[2026-04-12 11:00:00Z], [%{lat: 32.0, lon: -97.0}])
|
|
|
|
assert GridCache.prune_keep_latest(5) == 0
|
|
assert {:ok, _} = GridCache.fetch(~U[2026-04-12 10:00:00Z])
|
|
assert {:ok, _} = GridCache.fetch(~U[2026-04-12 11:00:00Z])
|
|
end
|
|
end
|
|
end
|