Graham McIntire
a24ce027f4
The /weather LiveView was timing out on pod restart because latest_weather_grid/1 ran load_weather_grid_from_db/1 synchronously on cache miss — that query reads 176k hrrr_profiles rows with two huge JSONB columns (profile array + duct_characteristics), and JSONB decoding blocked the LiveView process past the 15-second pool ownership timeout. On cache miss latest_weather_grid/1 now returns empty immediately and kicks off a deduped background fill through GridCache.claim_fill/1 (a per-valid_time ETS lock so N concurrent mounts after restart don't each fire the slow query). When the fill completes it broadcasts weather:updated, and the handle_info in WeatherMapLive hits the warm cache and pushes rows over the socket. Added load_weather_grid/1 as a synchronous sibling for tests and any caller that genuinely needs inline data. Tests updated to use it and to clear GridCache in setup so ETS state doesn't leak between cases. Also untrack k8s/secret.yaml and add both it and k8s/*-secret.yaml to .gitignore — those manifests hold plaintext SMTP and database credentials that should not live in the repo. Secrets already in git history should be rotated separately.
277 lines
8.3 KiB
Elixir
277 lines
8.3 KiB
Elixir
defmodule Microwaveprop.WeatherGridTest do
|
|
use Microwaveprop.DataCase, async: false
|
|
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Weather
|
|
alias Microwaveprop.Weather.GridCache
|
|
|
|
setup do
|
|
# GridCache ETS state leaks between tests (it's a named table outside the
|
|
# test sandbox). Clear it so each test gets a fresh cache and the DB load
|
|
# path actually runs.
|
|
GridCache.clear()
|
|
:ok
|
|
end
|
|
|
|
describe "load_weather_grid/1" do
|
|
test "returns grid data for latest valid_time within bounds" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
min_refractivity_gradient: -280.0,
|
|
ducting_detected: true
|
|
})
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.250,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 26.0,
|
|
surface_dewpoint_c: 19.0,
|
|
surface_pressure_mb: 1012.0,
|
|
hpbl_m: 600.0,
|
|
pwat_mm: 32.0,
|
|
min_refractivity_gradient: -150.0,
|
|
ducting_detected: false
|
|
})
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
result = Weather.load_weather_grid(bounds)
|
|
|
|
assert length(result) == 2
|
|
point = Enum.find(result, &(&1.lat == 32.125))
|
|
assert point.temperature == 25.0
|
|
assert point.dewpoint_depression == 7.0
|
|
assert point.bl_height == 500.0
|
|
assert point.pwat == 30.0
|
|
assert point.refractivity_gradient == -280.0
|
|
assert point.ducting == true
|
|
end
|
|
|
|
test "returns empty list when no data exists" do
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
assert Weather.load_weather_grid(bounds) == []
|
|
end
|
|
|
|
test "only returns points on 0.125 grid" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
})
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.1337,
|
|
lon: -97.3821,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
})
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
result = Weather.load_weather_grid(bounds)
|
|
|
|
assert length(result) == 1
|
|
assert hd(result).lat == 32.125
|
|
end
|
|
|
|
test "uses most recent valid_time only" do
|
|
old = DateTime.utc_now() |> DateTime.add(-7200, :second) |> DateTime.truncate(:second)
|
|
new = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: old,
|
|
surface_temp_c: 20.0,
|
|
surface_dewpoint_c: 15.0,
|
|
surface_pressure_mb: 1010.0,
|
|
hpbl_m: 400.0,
|
|
pwat_mm: 25.0
|
|
})
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: new,
|
|
surface_temp_c: 28.0,
|
|
surface_dewpoint_c: 20.0,
|
|
surface_pressure_mb: 1015.0,
|
|
hpbl_m: 700.0,
|
|
pwat_mm: 35.0
|
|
})
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
result = Weather.load_weather_grid(bounds)
|
|
|
|
assert length(result) == 1
|
|
assert hd(result).temperature == 28.0
|
|
end
|
|
|
|
test "includes derived upper-air fields and strips raw profile data" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0},
|
|
%{"pres" => 700.0, "tmpc" => 5.0, "dwpc" => -2.0, "hght" => 3100.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
|
|
duct_chars = [
|
|
%{"base" => 200.0, "strength" => 50.0},
|
|
%{"base" => 800.0, "strength" => 30.0}
|
|
]
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
surface_refractivity: 320.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
min_refractivity_gradient: -280.0,
|
|
ducting_detected: true,
|
|
profile: profile,
|
|
duct_characteristics: duct_chars
|
|
})
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
[point] = Weather.load_weather_grid(bounds)
|
|
|
|
# Derived fields should be present
|
|
assert is_float(point.surface_rh)
|
|
assert point.surface_refractivity == 320.0
|
|
assert point.temp_850mb == 15.0
|
|
assert point.dewpoint_850mb == 10.0
|
|
assert is_float(point.lapse_rate)
|
|
assert point.duct_base_m == 200.0
|
|
assert point.duct_strength == 50.0
|
|
|
|
# Raw bulky fields should be stripped
|
|
refute Map.has_key?(point, :profile)
|
|
refute Map.has_key?(point, :duct_characteristics)
|
|
refute Map.has_key?(point, :surface_temp_c)
|
|
refute Map.has_key?(point, :surface_dewpoint_c)
|
|
end
|
|
end
|
|
|
|
describe "weather_point_detail/3" do
|
|
test "returns all fields for a single grid point" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
min_refractivity_gradient: -280.0,
|
|
ducting_detected: true
|
|
})
|
|
|
|
result = Weather.weather_point_detail(32.15, -97.40, now)
|
|
|
|
assert result.lat == 32.125
|
|
assert result.temperature == 25.0
|
|
assert result.surface_pressure_mb == 1013.0
|
|
end
|
|
|
|
test "returns nil when no data at point" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
assert Weather.weather_point_detail(32.15, -97.40, now) == nil
|
|
end
|
|
|
|
test "includes derived upper-air fields and strips raw profile data" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
|
|
duct_chars = [%{"base" => 300.0, "strength" => 40.0}]
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
surface_refractivity: 320.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
min_refractivity_gradient: -280.0,
|
|
ducting_detected: true,
|
|
profile: profile,
|
|
duct_characteristics: duct_chars
|
|
})
|
|
|
|
result = Weather.weather_point_detail(32.15, -97.40, now)
|
|
|
|
# Derived fields
|
|
assert is_float(result.surface_rh)
|
|
assert result.surface_refractivity == 320.0
|
|
assert result.temp_850mb == 15.0
|
|
assert result.duct_base_m == 300.0
|
|
assert result.duct_strength == 40.0
|
|
|
|
# Raw fields stripped
|
|
refute Map.has_key?(result, :profile)
|
|
refute Map.has_key?(result, :duct_characteristics)
|
|
refute Map.has_key?(result, :surface_temp_c)
|
|
refute Map.has_key?(result, :surface_dewpoint_c)
|
|
end
|
|
end
|
|
|
|
defp insert_hrrr_profile(attrs) do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
is_gp =
|
|
rem(round(attrs.lat * 1000), 125) == 0 and
|
|
rem(round(attrs.lon * 1000), 125) == 0
|
|
|
|
defaults = %{
|
|
id: Ecto.UUID.bingenerate(),
|
|
run_time: DateTime.add(attrs.valid_time, -3600, :second),
|
|
profile: [],
|
|
surface_refractivity: nil,
|
|
min_refractivity_gradient: nil,
|
|
ducting_detected: nil,
|
|
duct_characteristics: nil,
|
|
is_grid_point: is_gp,
|
|
inserted_at: now,
|
|
updated_at: now
|
|
}
|
|
|
|
entry = Map.merge(defaults, attrs)
|
|
Repo.insert_all("hrrr_profiles", [entry])
|
|
end
|
|
end
|