Graham McIntire
e11ebc9af8
Adds 67 test cases + 12 property tests across four surfaces — parallel agents targeted the lowest-coverage files in the tree. - ContactLive.Show (48.56% → 72.01%): 16 unit tests + 2 properties covering IEMRE/radar/ducting/propagation-analysis render branches, every terrain-verdict class, expanded HRRR/terrain sections, pending- edit banner, internal-network conn, and total-over-fields properties for obs + sounding sort handlers. - HrrrNativeClient (33% → 58%): 8 unit tests + 4 properties covering partial-surface fallback, empty input, optional-var nils, bogus binary dispatch, and level-count + monotonic-height + URL-encoding + duct-subset invariants. - NexradClient / HrrrClient / GefsClient: 13 unit tests + 4 properties covering 500/404/timeout paths, malformed idx bodies, parse_idx totality, byte-range invariants, and URL-builder round-trips. - HrrrBackfill / HrrrNativeDeriveFields / ImportContestLogs: 7 unit tests + 1 property. Seeds contacts + native profiles, stubs HRRR with Req.Test, exercises the `--limit 0` and `filter_points_needing_ backfill` paths, CSV import happy + malformed paths, and a band-string round-trip property. 2599 → 2666 tests, 170 → 182 properties, 0 failures.
513 lines
18 KiB
Elixir
513 lines
18 KiB
Elixir
defmodule Microwaveprop.Weather.HrrrNativeClientTest do
|
||
use ExUnit.Case, async: true
|
||
use ExUnitProperties
|
||
|
||
alias Microwaveprop.Weather.HrrrClient
|
||
alias Microwaveprop.Weather.HrrrNativeClient
|
||
|
||
describe "hrrr_native_url/3" do
|
||
test "builds the wrfnatf00 URL for a given date and hour" do
|
||
url = HrrrNativeClient.hrrr_native_url(~D[2026-04-09], 12)
|
||
|
||
assert url ==
|
||
"https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260409/conus/hrrr.t12z.wrfnatf00.grib2"
|
||
end
|
||
|
||
test "pads single-digit hours" do
|
||
url = HrrrNativeClient.hrrr_native_url(~D[2026-04-09], 6)
|
||
assert url =~ "hrrr.t06z."
|
||
end
|
||
|
||
test "supports non-zero forecast hours" do
|
||
url = HrrrNativeClient.hrrr_native_url(~D[2026-04-09], 12, 3)
|
||
assert url =~ "wrfnatf03.grib2"
|
||
end
|
||
end
|
||
|
||
describe "native_messages/0" do
|
||
test "lists 350 messages (7 vars × 50 levels)" do
|
||
messages = HrrrNativeClient.native_messages()
|
||
assert length(messages) == 350
|
||
end
|
||
|
||
test "every level appears with every essential variable" do
|
||
messages = HrrrNativeClient.native_messages()
|
||
|
||
# Pick level 5 — should have all 7 vars
|
||
level5 = Enum.filter(messages, fn %{level: l} -> l == "5 hybrid level" end)
|
||
vars = level5 |> Enum.map(& &1.var) |> Enum.sort()
|
||
|
||
assert vars == ~w(HGT PRES SPFH TKE TMP UGRD VGRD)
|
||
end
|
||
|
||
test "covers levels 1 through 50" do
|
||
levels =
|
||
HrrrNativeClient.native_messages()
|
||
|> Enum.map(& &1.level)
|
||
|> Enum.uniq()
|
||
|> Enum.map(fn level_str ->
|
||
[digits, _] = String.split(level_str, " ", parts: 2)
|
||
String.to_integer(digits)
|
||
end)
|
||
|> Enum.sort()
|
||
|
||
assert levels == Enum.to_list(1..50)
|
||
end
|
||
end
|
||
|
||
describe "build_native_profile/1" do
|
||
test "assembles arrays in level order and caches surface scalars" do
|
||
parsed = %{
|
||
"HGT:1 hybrid level" => 8.0,
|
||
"TMP:1 hybrid level" => 295.0,
|
||
"SPFH:1 hybrid level" => 0.010,
|
||
"PRES:1 hybrid level" => 101_000.0,
|
||
"UGRD:1 hybrid level" => 2.0,
|
||
"VGRD:1 hybrid level" => 1.0,
|
||
"TKE:1 hybrid level" => 0.5,
|
||
"HGT:2 hybrid level" => 25.0,
|
||
"TMP:2 hybrid level" => 293.0,
|
||
"SPFH:2 hybrid level" => 0.008,
|
||
"PRES:2 hybrid level" => 99_800.0,
|
||
"UGRD:2 hybrid level" => 3.0,
|
||
"VGRD:2 hybrid level" => 1.5,
|
||
"TKE:2 hybrid level" => 0.3,
|
||
"TMP:surface" => 295.5,
|
||
"SPFH:2 m above ground" => 0.011,
|
||
"PRES:surface" => 101_325.0
|
||
}
|
||
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
|
||
assert profile.level_count == 2
|
||
assert profile.heights_m == [8.0, 25.0]
|
||
assert profile.temp_k == [295.0, 293.0]
|
||
assert profile.spfh == [0.010, 0.008]
|
||
assert profile.u_wind_ms == [2.0, 3.0]
|
||
assert profile.surface_temp_k == 295.5
|
||
assert profile.surface_spfh == 0.011
|
||
assert profile.surface_pressure_pa == 101_325.0
|
||
end
|
||
|
||
test "skips levels missing either HGT or TMP" do
|
||
parsed = %{
|
||
"HGT:1 hybrid level" => 8.0,
|
||
"TMP:1 hybrid level" => 295.0,
|
||
# level 2 has HGT but no TMP → skipped
|
||
"HGT:2 hybrid level" => 25.0,
|
||
"SPFH:2 hybrid level" => 0.008
|
||
}
|
||
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
assert profile.level_count == 1
|
||
assert profile.heights_m == [8.0]
|
||
end
|
||
|
||
test "orders levels by ascending height, not by hybrid level number" do
|
||
# Pathological: level 2 happens to be lower than level 1 (terrain).
|
||
parsed = %{
|
||
"HGT:1 hybrid level" => 50.0,
|
||
"TMP:1 hybrid level" => 290.0,
|
||
"HGT:2 hybrid level" => 30.0,
|
||
"TMP:2 hybrid level" => 292.0
|
||
}
|
||
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
assert profile.heights_m == [30.0, 50.0]
|
||
assert profile.temp_k == [292.0, 290.0]
|
||
end
|
||
|
||
test "falls back to lowest level for surface scalars when parsed has no surface entries" do
|
||
parsed = %{
|
||
"HGT:1 hybrid level" => 8.0,
|
||
"TMP:1 hybrid level" => 295.0,
|
||
"SPFH:1 hybrid level" => 0.010,
|
||
"PRES:1 hybrid level" => 101_000.0
|
||
}
|
||
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
assert profile.surface_temp_k == 295.0
|
||
assert profile.surface_spfh == 0.010
|
||
assert profile.surface_pressure_pa == 101_000.0
|
||
end
|
||
end
|
||
|
||
describe "essential_byte_ranges/1" do
|
||
test "produces one range per essential message present in the idx" do
|
||
# Minimal fake idx: level 1 TMP/SPFH/HGT with nothing else.
|
||
idx_entries = [
|
||
%{msg: 1, offset: 0, var: "TMP", level: "1 hybrid level"},
|
||
%{msg: 2, offset: 1000, var: "SPFH", level: "1 hybrid level"},
|
||
%{msg: 3, offset: 2000, var: "HGT", level: "1 hybrid level"},
|
||
%{msg: 4, offset: 3000, var: "REFC", level: "entire atmosphere"}
|
||
]
|
||
|
||
ranges = HrrrNativeClient.essential_byte_ranges(idx_entries)
|
||
|
||
# Expect 3 ranges (the three essentials we actually have)
|
||
assert length(ranges) == 3
|
||
# Each range is {start, end} tuples, all integers
|
||
Enum.each(ranges, fn {s, e} ->
|
||
assert is_integer(s) and is_integer(e)
|
||
assert s < e
|
||
end)
|
||
end
|
||
|
||
test "delegates to HrrrClient.byte_ranges_for_messages/2" do
|
||
# Just check the two stay in sync by computing the same ranges
|
||
# two different ways.
|
||
idx_entries = [
|
||
%{msg: 1, offset: 0, var: "TMP", level: "1 hybrid level"},
|
||
%{msg: 2, offset: 5000, var: "UGRD", level: "1 hybrid level"}
|
||
]
|
||
|
||
assert HrrrNativeClient.essential_byte_ranges(idx_entries) ==
|
||
HrrrClient.byte_ranges_for_messages(idx_entries, HrrrNativeClient.native_messages())
|
||
end
|
||
end
|
||
|
||
describe "native_level_count/0 and native_variables/0" do
|
||
test "native_level_count is 50 (full hybrid stack)" do
|
||
assert HrrrNativeClient.native_level_count() == 50
|
||
end
|
||
|
||
test "native_variables lists exactly the 7 native-grid fields" do
|
||
vars = HrrrNativeClient.native_variables()
|
||
assert length(vars) == 7
|
||
assert "TMP" in vars
|
||
assert "SPFH" in vars
|
||
assert "HGT" in vars
|
||
assert "PRES" in vars
|
||
assert "UGRD" in vars
|
||
assert "VGRD" in vars
|
||
assert "TKE" in vars
|
||
end
|
||
end
|
||
|
||
describe "native_messages/0 shape" do
|
||
test "emits one message per (level, variable) across the 50-level hybrid stack" do
|
||
messages = HrrrNativeClient.native_messages()
|
||
expected = HrrrNativeClient.native_level_count() * length(HrrrNativeClient.native_variables())
|
||
assert length(messages) == expected
|
||
|
||
# Every message carries a non-empty var and a hybrid-level string.
|
||
Enum.each(messages, fn %{var: var, level: level} ->
|
||
assert var in HrrrNativeClient.native_variables()
|
||
assert level =~ ~r/\A\d+ hybrid level\z/
|
||
end)
|
||
end
|
||
|
||
test "each (var, level) pair is unique" do
|
||
messages = HrrrNativeClient.native_messages()
|
||
unique = messages |> Enum.uniq_by(fn %{var: v, level: l} -> {v, l} end) |> length()
|
||
assert unique == length(messages)
|
||
end
|
||
end
|
||
|
||
describe "extract_native_profiles/2 Elixir fallback" do
|
||
# When wgrib2 isn't on the PATH, extract_native_profiles routes to
|
||
# the pure-Elixir Extractor and returns a map keyed by the requested
|
||
# points. We can't easily validate real GRIB output here, but we can
|
||
# drive the dispatch once with a deliberately malformed binary and
|
||
# show the client tolerates the Extractor's error without crashing.
|
||
test "surfaces the decoder's error when the binary is bogus" do
|
||
points = [{32.9, -97.0}]
|
||
|
||
case HrrrNativeClient.extract_native_profiles(<<0, 1, 2>>, points) do
|
||
{:ok, %{} = by_point} ->
|
||
# Some decoder paths return an empty ok map — that's still a
|
||
# valid contract ({:ok, map()}).
|
||
assert Map.has_key?(by_point, {32.9, -97.0}) or map_size(by_point) == 0
|
||
|
||
{:error, _reason} ->
|
||
:ok
|
||
end
|
||
end
|
||
end
|
||
|
||
describe "duct_messages/0 and duct_byte_ranges/1" do
|
||
test "emits one message per (level, duct-var) across the 50-level stack (4 vars)" do
|
||
messages = HrrrNativeClient.duct_messages()
|
||
assert length(messages) == 50 * 4
|
||
|
||
# Exactly the duct-detection variables — no UGRD/VGRD/TKE.
|
||
vars = messages |> Enum.map(& &1.var) |> Enum.uniq() |> Enum.sort()
|
||
assert vars == ["HGT", "PRES", "SPFH", "TMP"]
|
||
end
|
||
|
||
test "duct_byte_ranges/1 returns integer ranges for matching idx entries" do
|
||
idx_entries = [
|
||
%{msg: 1, offset: 0, var: "TMP", level: "1 hybrid level"},
|
||
%{msg: 2, offset: 1000, var: "SPFH", level: "1 hybrid level"},
|
||
%{msg: 3, offset: 2000, var: "UGRD", level: "1 hybrid level"},
|
||
%{msg: 4, offset: 3000, var: "HGT", level: "1 hybrid level"}
|
||
]
|
||
|
||
ranges = HrrrNativeClient.duct_byte_ranges(idx_entries)
|
||
# UGRD is not in the duct set, so only 3 ranges should come back.
|
||
assert length(ranges) == 3
|
||
|
||
Enum.each(ranges, fn {s, e} ->
|
||
assert is_integer(s) and is_integer(e)
|
||
assert s < e
|
||
end)
|
||
end
|
||
end
|
||
|
||
describe "build_native_profile/1 with partial surface coverage" do
|
||
test "uses parsed TMP:surface but falls back to lowest level for SPFH/PRES" do
|
||
parsed = %{
|
||
"HGT:1 hybrid level" => 10.0,
|
||
"TMP:1 hybrid level" => 294.0,
|
||
"SPFH:1 hybrid level" => 0.009,
|
||
"PRES:1 hybrid level" => 101_200.0,
|
||
"TMP:surface" => 296.2
|
||
# No "SPFH:2 m above ground", no "PRES:surface" — both fall back.
|
||
}
|
||
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
|
||
assert profile.surface_temp_k == 296.2
|
||
assert profile.surface_spfh == 0.009
|
||
assert profile.surface_pressure_pa == 101_200.0
|
||
end
|
||
|
||
test "returns nil surface scalars when there are no levels and no surface entries" do
|
||
profile = HrrrNativeClient.build_native_profile(%{})
|
||
|
||
assert profile.level_count == 0
|
||
assert profile.heights_m == []
|
||
assert profile.surface_temp_k == nil
|
||
assert profile.surface_spfh == nil
|
||
assert profile.surface_pressure_pa == nil
|
||
end
|
||
end
|
||
|
||
describe "extract_native_profiles/2 dispatch" do
|
||
test "returns {:ok, map} or {:error, _} for a bogus binary with multiple points" do
|
||
points = [{32.9, -97.0}, {33.5, -96.5}]
|
||
|
||
case HrrrNativeClient.extract_native_profiles(<<1, 2, 3, 4>>, points) do
|
||
{:ok, %{} = by_point} ->
|
||
# Either empty or keyed by points. Profiles either have level_count
|
||
# present or are nil-level; either is a valid contract.
|
||
Enum.each(by_point, fn {pt, profile} ->
|
||
assert pt in points
|
||
assert is_map(profile)
|
||
end)
|
||
|
||
{:error, _reason} ->
|
||
:ok
|
||
end
|
||
end
|
||
end
|
||
|
||
describe "extract_native_profiles_from_file/2" do
|
||
test "returns {:error, _} when the GRIB2 path does not exist" do
|
||
missing = Path.join(System.tmp_dir!(), "hrrr_native_client_missing_#{System.unique_integer([:positive])}.grib2")
|
||
refute File.exists?(missing)
|
||
|
||
assert {:error, _reason} = HrrrNativeClient.extract_native_profiles_from_file(missing, [{32.9, -97.0}])
|
||
end
|
||
|
||
test "returns an ok-map-with-empty-profiles or error tuple for a non-GRIB file" do
|
||
# wgrib2 may silently succeed with no matching messages, producing
|
||
# {:ok, %{point => %{level_count: 0}}}; the pure-Elixir decoder
|
||
# surfaces the parse failure as {:error, _}. Accept both shapes.
|
||
bogus_path = Path.join(System.tmp_dir!(), "hrrr_native_client_bogus_#{System.unique_integer([:positive])}.grib2")
|
||
File.write!(bogus_path, <<0, 1, 2, 3>>)
|
||
point = {32.9, -97.0}
|
||
|
||
try do
|
||
case HrrrNativeClient.extract_native_profiles_from_file(bogus_path, [point]) do
|
||
{:ok, %{} = by_point} ->
|
||
assert Map.has_key?(by_point, point)
|
||
profile = Map.fetch!(by_point, point)
|
||
assert is_map(profile)
|
||
assert Map.get(profile, :level_count, 0) == 0
|
||
|
||
{:error, _reason} ->
|
||
:ok
|
||
end
|
||
after
|
||
File.rm(bogus_path)
|
||
end
|
||
end
|
||
end
|
||
|
||
describe "build_native_profile/1 additional cases" do
|
||
test "preserves nil per-level values for optional variables (UGRD/VGRD/TKE)" do
|
||
# Only HGT + TMP present on level 1 — all other per-level arrays
|
||
# should contain nil in that slot, keeping array length = level_count.
|
||
parsed = %{
|
||
"HGT:1 hybrid level" => 12.0,
|
||
"TMP:1 hybrid level" => 290.0
|
||
}
|
||
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
|
||
assert profile.level_count == 1
|
||
assert profile.heights_m == [12.0]
|
||
assert profile.temp_k == [290.0]
|
||
assert profile.spfh == [nil]
|
||
assert profile.pressure_pa == [nil]
|
||
assert profile.u_wind_ms == [nil]
|
||
assert profile.v_wind_ms == [nil]
|
||
assert profile.tke_m2s2 == [nil]
|
||
end
|
||
|
||
test "tolerates extra noise keys in the parsed map" do
|
||
parsed = %{
|
||
"HGT:1 hybrid level" => 10.0,
|
||
"TMP:1 hybrid level" => 291.0,
|
||
# Noise that build_native_profile should simply ignore.
|
||
"REFC:entire atmosphere" => 30.0,
|
||
"TMP:500 mb" => 260.0,
|
||
"garbage" => :whatever
|
||
}
|
||
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
assert profile.level_count == 1
|
||
assert profile.heights_m == [10.0]
|
||
assert profile.temp_k == [291.0]
|
||
end
|
||
end
|
||
|
||
describe "properties" do
|
||
# Generators -----------------------------------------------------------
|
||
|
||
# Random subset of hybrid levels (1..50) to exercise build_native_profile
|
||
# against sparse parsed maps.
|
||
defp level_subset_generator do
|
||
1..50
|
||
|> StreamData.integer()
|
||
|> StreamData.list_of(min_length: 0, max_length: 10)
|
||
|> StreamData.map(&Enum.uniq/1)
|
||
end
|
||
|
||
# Build a parsed map for a set of levels with a random subset of vars.
|
||
defp parsed_map_generator do
|
||
gen all(
|
||
levels <- level_subset_generator(),
|
||
include_hgt <- StreamData.boolean(),
|
||
include_tmp <- StreamData.boolean(),
|
||
include_spfh <- StreamData.boolean(),
|
||
include_pres <- StreamData.boolean(),
|
||
include_ugrd <- StreamData.boolean(),
|
||
include_vgrd <- StreamData.boolean(),
|
||
include_tke <- StreamData.boolean()
|
||
) do
|
||
Enum.reduce(levels, %{}, fn level, acc ->
|
||
lvl_str = "#{level} hybrid level"
|
||
|
||
# Use the level number as a monotonic-ish HGT seed so order-by-hgt
|
||
# aligns with level order, making the monotonic property meaningful.
|
||
acc
|
||
|> maybe_put(include_hgt, "HGT:#{lvl_str}", level * 10.0)
|
||
|> maybe_put(include_tmp, "TMP:#{lvl_str}", 290.0 - level * 0.5)
|
||
|> maybe_put(include_spfh, "SPFH:#{lvl_str}", 0.01 / level)
|
||
|> maybe_put(include_pres, "PRES:#{lvl_str}", 101_325.0 - level * 100.0)
|
||
|> maybe_put(include_ugrd, "UGRD:#{lvl_str}", 1.0 * level)
|
||
|> maybe_put(include_vgrd, "VGRD:#{lvl_str}", -1.0 * level)
|
||
|> maybe_put(include_tke, "TKE:#{lvl_str}", 0.1 * level)
|
||
end)
|
||
end
|
||
end
|
||
|
||
defp maybe_put(map, true, key, value), do: Map.put(map, key, value)
|
||
defp maybe_put(map, false, _key, _value), do: map
|
||
|
||
# Generator for arbitrary idx entries. Mixes duct-vars with non-duct
|
||
# vars at a variety of hybrid levels so duct_byte_ranges ⊂ essential
|
||
# is actually exercised.
|
||
defp idx_entries_generator do
|
||
all_vars = ~w(TMP SPFH HGT PRES UGRD VGRD TKE REFC)
|
||
|
||
gen all(
|
||
entries <-
|
||
StreamData.list_of(
|
||
StreamData.tuple({
|
||
StreamData.member_of(all_vars),
|
||
StreamData.integer(1..50)
|
||
}),
|
||
min_length: 1,
|
||
max_length: 30
|
||
)
|
||
) do
|
||
entries
|
||
|> Enum.uniq()
|
||
|> Enum.with_index()
|
||
|> Enum.map(fn {{var, level}, i} ->
|
||
%{msg: i + 1, offset: i * 1000, var: var, level: "#{level} hybrid level"}
|
||
end)
|
||
end
|
||
end
|
||
|
||
# Properties -----------------------------------------------------------
|
||
|
||
property "build_native_profile/1 array lengths all equal level_count" do
|
||
check all(parsed <- parsed_map_generator()) do
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
|
||
lc = profile.level_count
|
||
assert length(profile.heights_m) == lc
|
||
assert length(profile.temp_k) == lc
|
||
assert length(profile.spfh) == lc
|
||
assert length(profile.pressure_pa) == lc
|
||
assert length(profile.u_wind_ms) == lc
|
||
assert length(profile.v_wind_ms) == lc
|
||
assert length(profile.tke_m2s2) == lc
|
||
end
|
||
end
|
||
|
||
property "build_native_profile/1 heights_m is monotonic non-decreasing" do
|
||
check all(parsed <- parsed_map_generator()) do
|
||
profile = HrrrNativeClient.build_native_profile(parsed)
|
||
|
||
profile.heights_m
|
||
|> Enum.chunk_every(2, 1, :discard)
|
||
|> Enum.each(fn [a, b] -> assert a <= b end)
|
||
end
|
||
end
|
||
|
||
property "hrrr_native_url/3 round-trips date, hour, and forecast hour into the URL" do
|
||
check all(
|
||
year <- StreamData.integer(2015..2100),
|
||
month <- StreamData.integer(1..12),
|
||
day <- StreamData.integer(1..28),
|
||
hour <- StreamData.integer(0..23),
|
||
fh <- StreamData.integer(0..48)
|
||
) do
|
||
{:ok, date} = Date.new(year, month, day)
|
||
url = HrrrNativeClient.hrrr_native_url(date, hour, fh)
|
||
|
||
date_str = Calendar.strftime(date, "%Y%m%d")
|
||
hour_str = hour |> Integer.to_string() |> String.pad_leading(2, "0")
|
||
fh_str = fh |> Integer.to_string() |> String.pad_leading(2, "0")
|
||
|
||
assert String.starts_with?(url, "https://")
|
||
assert url =~ "hrrr.#{date_str}"
|
||
assert url =~ "t#{hour_str}z"
|
||
assert url =~ "wrfnatf#{fh_str}"
|
||
assert String.ends_with?(url, ".grib2")
|
||
end
|
||
end
|
||
|
||
property "duct_byte_ranges/1 returns a subset-count of essential_byte_ranges/1" do
|
||
check all(idx_entries <- idx_entries_generator()) do
|
||
duct = HrrrNativeClient.duct_byte_ranges(idx_entries)
|
||
essential = HrrrNativeClient.essential_byte_ranges(idx_entries)
|
||
|
||
# Duct vars are a proper subset of native vars, same levels.
|
||
# Therefore: every matching (var, level) pair counted for duct
|
||
# also counts for essential.
|
||
assert length(duct) <= length(essential)
|
||
|
||
# And every duct range appears somewhere in essential.
|
||
Enum.each(duct, fn range ->
|
||
assert range in essential
|
||
end)
|
||
end
|
||
end
|
||
end
|
||
end
|