Graham McIntire
fb0c4f7c77
When the on-disk ProfilesFile is empty SkewtLive falls back to the
hrrr_profiles Postgres table, but per-QSO HrrrFetchWorker rows
sometimes land with a partial profile (4–12 levels) when individual
byte-range fetches fail. The fallback was happily picking those rows,
which rendered a stubby Skew-T missing the upper troposphere.
HrrrProfileLookup.{list_valid_times_near, read_point_near}/3 now
filter on `coalesce(cardinality(profile), 0) >= min_profile_levels`,
defaulting to 13 (HRRR's standard pressure-level set: 1000/925/850/
700/500/400/300/250/200/150/100 mb plus the two boundary surfaces).
Pass `min_profile_levels: 0` to keep the partial rows for diagnostic
queries.
Confirmed against the dev mirror: the recent table has ~3.1 M rows
at 13 levels and ~250 k at 4–12 levels; the new default skips the
partial slice cleanly.
Tests: 8/8 HrrrProfileLookupTest cases (added two new ones for the
threshold). mix credo --strict clean.
180 lines
7.2 KiB
Elixir
180 lines
7.2 KiB
Elixir
defmodule Microwaveprop.Weather.HrrrProfileLookupTest do
|
|
use Microwaveprop.DataCase, async: true
|
|
|
|
alias Microwaveprop.Weather.HrrrProfile
|
|
alias Microwaveprop.Weather.HrrrProfileLookup
|
|
|
|
# 13-level standard HRRR profile so test fixtures pass the default
|
|
# min_profile_levels filter. Tests that want to exercise the
|
|
# partial-profile rejection path pass an explicit shorter profile.
|
|
@full_profile [
|
|
%{"pres" => 1000.0, "hght" => 100.0, "tmpc" => 25.0, "dwpc" => 18.0},
|
|
%{"pres" => 925.0, "hght" => 800.0, "tmpc" => 19.0, "dwpc" => 14.0},
|
|
%{"pres" => 850.0, "hght" => 1500.0, "tmpc" => 13.0, "dwpc" => 8.0},
|
|
%{"pres" => 700.0, "hght" => 3100.0, "tmpc" => 2.0, "dwpc" => -8.0},
|
|
%{"pres" => 500.0, "hght" => 5800.0, "tmpc" => -15.0, "dwpc" => -28.0},
|
|
%{"pres" => 400.0, "hght" => 7200.0, "tmpc" => -28.0, "dwpc" => -42.0},
|
|
%{"pres" => 300.0, "hght" => 9300.0, "tmpc" => -42.0, "dwpc" => -56.0},
|
|
%{"pres" => 250.0, "hght" => 10_400.0, "tmpc" => -50.0, "dwpc" => -64.0},
|
|
%{"pres" => 200.0, "hght" => 11_700.0, "tmpc" => -55.0, "dwpc" => -70.0},
|
|
%{"pres" => 150.0, "hght" => 13_500.0, "tmpc" => -60.0, "dwpc" => -75.0},
|
|
%{"pres" => 100.0, "hght" => 16_000.0, "tmpc" => -65.0, "dwpc" => -80.0},
|
|
%{"pres" => 70.0, "hght" => 18_500.0, "tmpc" => -65.0, "dwpc" => -82.0},
|
|
%{"pres" => 50.0, "hght" => 20_500.0, "tmpc" => -62.0, "dwpc" => -84.0}
|
|
]
|
|
|
|
describe "list_valid_times_near/3" do
|
|
test "returns valid_times of every profile within the spatial tolerance" do
|
|
insert_profile!(%{lat: 32.625, lon: -97.125, valid_time: ~U[2026-04-25 14:00:00Z]})
|
|
insert_profile!(%{lat: 32.625, lon: -97.125, valid_time: ~U[2026-04-25 15:00:00Z]})
|
|
# Far cell — must be excluded.
|
|
insert_profile!(%{lat: 40.000, lon: -80.000, valid_time: ~U[2026-04-25 14:00:00Z]})
|
|
|
|
assert [t1, t2] = HrrrProfileLookup.list_valid_times_near(32.65, -97.13, [])
|
|
assert DateTime.to_iso8601(t1) == "2026-04-25T14:00:00Z"
|
|
assert DateTime.to_iso8601(t2) == "2026-04-25T15:00:00Z"
|
|
end
|
|
|
|
test "limit option caps the number of times returned (most recent first kept)" do
|
|
for h <- 0..4 do
|
|
valid_time = DateTime.add(~U[2026-04-25 12:00:00Z], h * 3600, :second)
|
|
insert_profile!(%{lat: 32.625, lon: -97.125, valid_time: valid_time})
|
|
end
|
|
|
|
times = HrrrProfileLookup.list_valid_times_near(32.65, -97.13, limit: 3)
|
|
assert length(times) == 3
|
|
# Capped to the 3 most recent — but still sorted ascending in the
|
|
# returned list so SkewtLive's selector renders left-to-right.
|
|
assert Enum.map(times, &DateTime.to_iso8601/1) == [
|
|
"2026-04-25T14:00:00Z",
|
|
"2026-04-25T15:00:00Z",
|
|
"2026-04-25T16:00:00Z"
|
|
]
|
|
end
|
|
|
|
test "tolerance option widens the spatial search box" do
|
|
insert_profile!(%{lat: 32.625, lon: -97.125, valid_time: ~U[2026-04-25 14:00:00Z]})
|
|
# 0.20° away, outside the default 0.07° box but inside 0.25°.
|
|
insert_profile!(%{lat: 32.825, lon: -97.125, valid_time: ~U[2026-04-25 15:00:00Z]})
|
|
|
|
tight = HrrrProfileLookup.list_valid_times_near(32.625, -97.125, tolerance_deg: 0.07)
|
|
wide = HrrrProfileLookup.list_valid_times_near(32.625, -97.125, tolerance_deg: 0.25)
|
|
assert length(tight) == 1
|
|
assert length(wide) == 2
|
|
end
|
|
end
|
|
|
|
describe "read_point_near/3" do
|
|
test "returns the profile of the nearest cell at the requested valid_time" do
|
|
insert_profile!(%{
|
|
lat: 32.625,
|
|
lon: -97.125,
|
|
valid_time: ~U[2026-04-25 14:00:00Z],
|
|
surface_temp_c: 20.5,
|
|
surface_dewpoint_c: 15.0,
|
|
surface_pressure_mb: 1013.0
|
|
})
|
|
|
|
assert %{} = cell = HrrrProfileLookup.read_point_near(~U[2026-04-25 14:00:00Z], 32.65, -97.13)
|
|
# Default fixture is the 13-level @full_profile (HRRR's standard
|
|
# canonical pressure-level set).
|
|
assert length(cell[:profile]) == 13
|
|
[%{"pres" => pres} | _] = cell[:profile]
|
|
assert pres == 1000.0 or pres == 1000
|
|
assert cell[:surface_temp_c] == 20.5
|
|
assert cell[:surface_pressure_mb] == 1013.0
|
|
end
|
|
|
|
test "returns nil if no profile is within tolerance at that valid_time" do
|
|
assert HrrrProfileLookup.read_point_near(~U[2026-04-25 14:00:00Z], 32.65, -97.13) == nil
|
|
end
|
|
|
|
test "picks the spatially closest cell when multiple match" do
|
|
# Both cells use the 13-level default profile so they pass the
|
|
# `min_profile_levels` filter; surface T/Td differ so we can tell
|
|
# which row read_point_near returned.
|
|
[first | rest] = @full_profile
|
|
close_profile = [Map.put(first, "tmpc", 25.0) | rest]
|
|
far_profile = [Map.put(first, "tmpc", 30.0) | rest]
|
|
|
|
insert_profile!(%{
|
|
lat: 32.65,
|
|
lon: -97.13,
|
|
valid_time: ~U[2026-04-25 14:00:00Z],
|
|
profile: close_profile
|
|
})
|
|
|
|
insert_profile!(%{
|
|
lat: 32.69,
|
|
lon: -97.18,
|
|
valid_time: ~U[2026-04-25 14:00:00Z],
|
|
profile: far_profile
|
|
})
|
|
|
|
cell = HrrrProfileLookup.read_point_near(~U[2026-04-25 14:00:00Z], 32.65, -97.13)
|
|
[%{"tmpc" => surface_t} | _] = cell[:profile]
|
|
assert surface_t == 25.0
|
|
end
|
|
end
|
|
|
|
describe "min_profile_levels filtering" do
|
|
test "list_valid_times_near rejects rows whose profile is shorter than the threshold" do
|
|
# Two rows: one full (13 levels), one partial (3 levels).
|
|
insert_profile!(%{lat: 32.625, lon: -97.125, valid_time: ~U[2026-04-25 14:00:00Z]})
|
|
|
|
insert_profile!(%{
|
|
lat: 32.625,
|
|
lon: -97.125,
|
|
valid_time: ~U[2026-04-25 15:00:00Z],
|
|
profile: [
|
|
%{"pres" => 1000.0, "hght" => 100.0, "tmpc" => 25.0, "dwpc" => 18.0},
|
|
%{"pres" => 850.0, "hght" => 1500.0, "tmpc" => 13.0, "dwpc" => 8.0},
|
|
%{"pres" => 500.0, "hght" => 5800.0, "tmpc" => -15.0, "dwpc" => -28.0}
|
|
]
|
|
})
|
|
|
|
# Default threshold (13) → only the 14 :00 row.
|
|
times = HrrrProfileLookup.list_valid_times_near(32.65, -97.13)
|
|
assert Enum.map(times, &DateTime.to_iso8601/1) == ["2026-04-25T14:00:00Z"]
|
|
|
|
# Threshold of 0 → both rows.
|
|
both = HrrrProfileLookup.list_valid_times_near(32.65, -97.13, min_profile_levels: 0)
|
|
assert length(both) == 2
|
|
end
|
|
|
|
test "read_point_near returns nil for partial-profile valid_times under the default threshold" do
|
|
insert_profile!(%{
|
|
lat: 32.625,
|
|
lon: -97.125,
|
|
valid_time: ~U[2026-04-25 15:00:00Z],
|
|
profile: [%{"pres" => 1000.0, "hght" => 100.0, "tmpc" => 25.0, "dwpc" => 18.0}]
|
|
})
|
|
|
|
assert HrrrProfileLookup.read_point_near(~U[2026-04-25 15:00:00Z], 32.65, -97.13) == nil
|
|
|
|
assert %{} =
|
|
HrrrProfileLookup.read_point_near(~U[2026-04-25 15:00:00Z], 32.65, -97.13, min_profile_levels: 0)
|
|
end
|
|
end
|
|
|
|
defp insert_profile!(attrs) do
|
|
defaults = %{
|
|
profile: @full_profile,
|
|
run_time: nil,
|
|
hpbl_m: 800.0,
|
|
pwat_mm: 25.0,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
surface_refractivity: 320.0,
|
|
min_refractivity_gradient: -90.0,
|
|
ducting_detected: false,
|
|
duct_characteristics: [],
|
|
is_grid_point: false
|
|
}
|
|
|
|
%HrrrProfile{}
|
|
|> HrrrProfile.changeset(Map.merge(defaults, attrs))
|
|
|> Microwaveprop.Repo.insert!()
|
|
end
|
|
end
|