Graham McIntire
2405c5f169
warm_grid_cache_and_broadcast was re-SELECTing 92k hrrr_profiles rows with JSONB profile/duct_characteristics columns on every forecast hour. Row decoding exceeded the 15s default DB connection timeout, killing the worker before f01-f18 could run. Oban retried from f00 and got stuck in a loop: for the last several hours, no forecast hours were being written and even f00 was stale. Build the weather cache rows directly from the in-memory grid_data the worker already has (Weather.build_grid_cache_rows/2) and GridCache.broadcast_put directly. No DB round trip, no JSONB decode. Also widen the cron from hourly to every 3 hours so a full f00-f18 sweep (~95 min) can actually complete before the next run starts, and drop the redundant prune_old_scores() at worker start (PropagationPruneWorker already runs every 15 min). Add a 60s query timeout to load_weather_grid_from_db as defense-in-depth for the cold-cache fill path that still uses it.
429 lines
13 KiB
Elixir
429 lines
13 KiB
Elixir
defmodule Microwaveprop.WeatherGridTest do
|
|
use Microwaveprop.DataCase, async: false
|
|
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Weather
|
|
alias Microwaveprop.Weather.GridCache
|
|
|
|
setup do
|
|
# GridCache ETS state leaks between tests (it's a named table outside the
|
|
# test sandbox). Clear it so each test gets a fresh cache and the DB load
|
|
# path actually runs.
|
|
GridCache.clear()
|
|
:ok
|
|
end
|
|
|
|
describe "load_weather_grid/1" do
|
|
test "returns grid data for latest valid_time within bounds" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
min_refractivity_gradient: -280.0,
|
|
ducting_detected: true
|
|
})
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.250,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 26.0,
|
|
surface_dewpoint_c: 19.0,
|
|
surface_pressure_mb: 1012.0,
|
|
hpbl_m: 600.0,
|
|
pwat_mm: 32.0,
|
|
min_refractivity_gradient: -150.0,
|
|
ducting_detected: false
|
|
})
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
result = Weather.load_weather_grid(bounds)
|
|
|
|
assert length(result) == 2
|
|
point = Enum.find(result, &(&1.lat == 32.125))
|
|
assert point.temperature == 25.0
|
|
assert point.dewpoint_depression == 7.0
|
|
assert point.bl_height == 500.0
|
|
assert point.pwat == 30.0
|
|
assert point.refractivity_gradient == -280.0
|
|
assert point.ducting == true
|
|
end
|
|
|
|
test "returns empty list when no data exists" do
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
assert Weather.load_weather_grid(bounds) == []
|
|
end
|
|
|
|
test "only returns points on 0.125 grid" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
})
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.1337,
|
|
lon: -97.3821,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
})
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
result = Weather.load_weather_grid(bounds)
|
|
|
|
assert length(result) == 1
|
|
assert hd(result).lat == 32.125
|
|
end
|
|
|
|
test "uses most recent valid_time only" do
|
|
old = DateTime.utc_now() |> DateTime.add(-7200, :second) |> DateTime.truncate(:second)
|
|
new = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: old,
|
|
surface_temp_c: 20.0,
|
|
surface_dewpoint_c: 15.0,
|
|
surface_pressure_mb: 1010.0,
|
|
hpbl_m: 400.0,
|
|
pwat_mm: 25.0
|
|
})
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: new,
|
|
surface_temp_c: 28.0,
|
|
surface_dewpoint_c: 20.0,
|
|
surface_pressure_mb: 1015.0,
|
|
hpbl_m: 700.0,
|
|
pwat_mm: 35.0
|
|
})
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
result = Weather.load_weather_grid(bounds)
|
|
|
|
assert length(result) == 1
|
|
assert hd(result).temperature == 28.0
|
|
end
|
|
|
|
test "includes derived upper-air fields and strips raw profile data" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0},
|
|
%{"pres" => 700.0, "tmpc" => 5.0, "dwpc" => -2.0, "hght" => 3100.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
|
|
duct_chars = [
|
|
%{"base" => 200.0, "strength" => 50.0},
|
|
%{"base" => 800.0, "strength" => 30.0}
|
|
]
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
surface_refractivity: 320.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
min_refractivity_gradient: -280.0,
|
|
ducting_detected: true,
|
|
profile: profile,
|
|
duct_characteristics: duct_chars
|
|
})
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
[point] = Weather.load_weather_grid(bounds)
|
|
|
|
# Derived fields should be present
|
|
assert is_float(point.surface_rh)
|
|
assert point.surface_refractivity == 320.0
|
|
assert point.temp_850mb == 15.0
|
|
assert point.dewpoint_850mb == 10.0
|
|
assert is_float(point.lapse_rate)
|
|
assert point.duct_base_m == 200.0
|
|
assert point.duct_strength == 50.0
|
|
|
|
# Raw bulky fields should be stripped
|
|
refute Map.has_key?(point, :profile)
|
|
refute Map.has_key?(point, :duct_characteristics)
|
|
refute Map.has_key?(point, :surface_temp_c)
|
|
refute Map.has_key?(point, :surface_dewpoint_c)
|
|
end
|
|
end
|
|
|
|
describe "build_grid_cache_rows/2" do
|
|
test "returns an empty list for empty grid_data" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
assert Weather.build_grid_cache_rows(%{}, now) == []
|
|
end
|
|
|
|
test "builds a derived row for each in-memory grid point without hitting the database" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0},
|
|
%{"pres" => 700.0, "tmpc" => 5.0, "dwpc" => -2.0, "hght" => 3100.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
|
|
grid_data = %{
|
|
{32.125, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
},
|
|
{32.250, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 26.0,
|
|
surface_dewpoint_c: 19.0,
|
|
surface_pressure_mb: 1012.0,
|
|
hpbl_m: 600.0,
|
|
pwat_mm: 32.0
|
|
}
|
|
}
|
|
|
|
result = Weather.build_grid_cache_rows(grid_data, now)
|
|
|
|
assert length(result) == 2
|
|
|
|
point = Enum.find(result, &(&1.lat == 32.125))
|
|
assert point.lon == -97.375
|
|
assert point.valid_time == now
|
|
assert point.temperature == 25.0
|
|
assert point.dewpoint_depression == 7.0
|
|
assert point.bl_height == 500.0
|
|
assert point.pwat == 30.0
|
|
assert point.surface_pressure_mb == 1013.0
|
|
end
|
|
|
|
test "computes surface_refractivity and min_refractivity_gradient from the profile" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0},
|
|
%{"pres" => 700.0, "tmpc" => 5.0, "dwpc" => -2.0, "hght" => 3100.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
|
|
grid_data = %{
|
|
{32.125, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
}
|
|
}
|
|
|
|
[point] = Weather.build_grid_cache_rows(grid_data, now)
|
|
|
|
assert is_float(point.surface_refractivity)
|
|
assert is_float(point.refractivity_gradient)
|
|
assert is_float(point.surface_rh)
|
|
assert point.temp_850mb == 15.0
|
|
assert point.dewpoint_850mb == 10.0
|
|
end
|
|
|
|
test "skips points with missing or physically impossible surface temperature" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}
|
|
]
|
|
|
|
grid_data = %{
|
|
{32.125, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: nil,
|
|
surface_dewpoint_c: nil,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
},
|
|
{32.250, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 250.0,
|
|
surface_dewpoint_c: 200.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
},
|
|
{32.375, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
}
|
|
}
|
|
|
|
result = Weather.build_grid_cache_rows(grid_data, now)
|
|
|
|
assert length(result) == 1
|
|
assert hd(result).lat == 32.375
|
|
end
|
|
|
|
test "strips raw profile/duct_characteristics/surface_temp_c/surface_dewpoint_c fields" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}
|
|
]
|
|
|
|
grid_data = %{
|
|
{32.125, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
}
|
|
}
|
|
|
|
[point] = Weather.build_grid_cache_rows(grid_data, now)
|
|
|
|
refute Map.has_key?(point, :profile)
|
|
refute Map.has_key?(point, :duct_characteristics)
|
|
refute Map.has_key?(point, :surface_temp_c)
|
|
refute Map.has_key?(point, :surface_dewpoint_c)
|
|
end
|
|
end
|
|
|
|
describe "weather_point_detail/3" do
|
|
test "returns all fields for a single grid point" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
min_refractivity_gradient: -280.0,
|
|
ducting_detected: true
|
|
})
|
|
|
|
result = Weather.weather_point_detail(32.15, -97.40, now)
|
|
|
|
assert result.lat == 32.125
|
|
assert result.temperature == 25.0
|
|
assert result.surface_pressure_mb == 1013.0
|
|
end
|
|
|
|
test "returns nil when no data at point" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
assert Weather.weather_point_detail(32.15, -97.40, now) == nil
|
|
end
|
|
|
|
test "includes derived upper-air fields and strips raw profile data" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
|
|
duct_chars = [%{"base" => 300.0, "strength" => 40.0}]
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
surface_refractivity: 320.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
min_refractivity_gradient: -280.0,
|
|
ducting_detected: true,
|
|
profile: profile,
|
|
duct_characteristics: duct_chars
|
|
})
|
|
|
|
result = Weather.weather_point_detail(32.15, -97.40, now)
|
|
|
|
# Derived fields
|
|
assert is_float(result.surface_rh)
|
|
assert result.surface_refractivity == 320.0
|
|
assert result.temp_850mb == 15.0
|
|
assert result.duct_base_m == 300.0
|
|
assert result.duct_strength == 40.0
|
|
|
|
# Raw fields stripped
|
|
refute Map.has_key?(result, :profile)
|
|
refute Map.has_key?(result, :duct_characteristics)
|
|
refute Map.has_key?(result, :surface_temp_c)
|
|
refute Map.has_key?(result, :surface_dewpoint_c)
|
|
end
|
|
end
|
|
|
|
defp insert_hrrr_profile(attrs) do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
is_gp =
|
|
rem(round(attrs.lat * 1000), 125) == 0 and
|
|
rem(round(attrs.lon * 1000), 125) == 0
|
|
|
|
defaults = %{
|
|
id: Ecto.UUID.bingenerate(),
|
|
run_time: DateTime.add(attrs.valid_time, -3600, :second),
|
|
profile: [],
|
|
surface_refractivity: nil,
|
|
min_refractivity_gradient: nil,
|
|
ducting_detected: nil,
|
|
duct_characteristics: nil,
|
|
is_grid_point: is_gp,
|
|
inserted_at: now,
|
|
updated_at: now
|
|
}
|
|
|
|
entry = Map.merge(defaults, attrs)
|
|
Repo.insert_all("hrrr_profiles", [entry])
|
|
end
|
|
end
|