fix(skewt): only fall back to profiles with all 13 pressure levels
When the on-disk ProfilesFile is empty SkewtLive falls back to the
hrrr_profiles Postgres table, but per-QSO HrrrFetchWorker rows
sometimes land with a partial profile (4–12 levels) when individual
byte-range fetches fail. The fallback was happily picking those rows,
which rendered a stubby Skew-T missing the upper troposphere.
HrrrProfileLookup.{list_valid_times_near, read_point_near}/3 now
filter on `coalesce(cardinality(profile), 0) >= min_profile_levels`,
defaulting to 13 (HRRR's standard pressure-level set: 1000/925/850/
700/500/400/300/250/200/150/100 mb plus the two boundary surfaces).
Pass `min_profile_levels: 0` to keep the partial rows for diagnostic
queries.
Confirmed against the dev mirror: the recent table has ~3.1 M rows
at 13 levels and ~250 k at 4–12 levels; the new default skips the
partial slice cleanly.
Tests: 8/8 HrrrProfileLookupTest cases (added two new ones for the
threshold). mix credo --strict clean.
This commit is contained in:
parent
5a0bd3bbf5
commit
fb0c4f7c77
2 changed files with 91 additions and 15 deletions
|
|
@ -19,6 +19,13 @@ defmodule Microwaveprop.Weather.HrrrProfileLookup do
|
|||
|
||||
@default_tolerance_deg 0.07
|
||||
@default_limit 32
|
||||
# HRRR persists 13 standard pressure levels per profile (1000/925/
|
||||
# 850/700/500/400/300/250/200/150/100 mb plus 2 boundary surfaces).
|
||||
# Per-QSO HrrrFetchWorker rows sometimes land with a partial profile
|
||||
# (4–11 levels) when individual byte-range fetches fail. SkewtLive
|
||||
# only wants complete profiles — anything below this threshold draws
|
||||
# a stubby diagram and gets filtered out by default.
|
||||
@default_min_levels 13
|
||||
|
||||
@doc """
|
||||
Distinct `valid_time`s with at least one HRRR profile within
|
||||
|
|
@ -29,17 +36,21 @@ defmodule Microwaveprop.Weather.HrrrProfileLookup do
|
|||
Options:
|
||||
* `:tolerance_deg` — default `#{@default_tolerance_deg}`
|
||||
* `:limit` — default `#{@default_limit}`
|
||||
* `:min_profile_levels` — minimum array length on `profile` (default
|
||||
`#{@default_min_levels}`). Use `0` to keep partial-profile rows.
|
||||
"""
|
||||
@spec list_valid_times_near(float(), float(), keyword()) :: [DateTime.t()]
|
||||
def list_valid_times_near(lat, lon, opts \\ []) when is_number(lat) and is_number(lon) do
|
||||
tolerance = Keyword.get(opts, :tolerance_deg, @default_tolerance_deg)
|
||||
limit = Keyword.get(opts, :limit, @default_limit)
|
||||
min_levels = Keyword.get(opts, :min_profile_levels, @default_min_levels)
|
||||
|
||||
query =
|
||||
from p in HrrrProfile,
|
||||
where:
|
||||
p.lat >= ^(lat - tolerance) and p.lat <= ^(lat + tolerance) and
|
||||
p.lon >= ^(lon - tolerance) and p.lon <= ^(lon + tolerance),
|
||||
p.lon >= ^(lon - tolerance) and p.lon <= ^(lon + tolerance) and
|
||||
fragment("coalesce(cardinality(?), 0) >= ?", p.profile, ^min_levels),
|
||||
group_by: p.valid_time,
|
||||
order_by: [desc: p.valid_time],
|
||||
limit: ^limit,
|
||||
|
|
@ -59,10 +70,13 @@ defmodule Microwaveprop.Weather.HrrrProfileLookup do
|
|||
|
||||
Options:
|
||||
* `:tolerance_deg` — default `#{@default_tolerance_deg}`
|
||||
* `:min_profile_levels` — minimum array length on `profile` (default
|
||||
`#{@default_min_levels}`). Use `0` to keep partial-profile rows.
|
||||
"""
|
||||
@spec read_point_near(DateTime.t(), float(), float(), keyword()) :: map() | nil
|
||||
def read_point_near(%DateTime{} = valid_time, lat, lon, opts \\ []) when is_number(lat) and is_number(lon) do
|
||||
tolerance = Keyword.get(opts, :tolerance_deg, @default_tolerance_deg)
|
||||
min_levels = Keyword.get(opts, :min_profile_levels, @default_min_levels)
|
||||
naive = DateTime.to_naive(valid_time)
|
||||
|
||||
query =
|
||||
|
|
@ -70,7 +84,8 @@ defmodule Microwaveprop.Weather.HrrrProfileLookup do
|
|||
where:
|
||||
p.valid_time == ^naive and
|
||||
p.lat >= ^(lat - tolerance) and p.lat <= ^(lat + tolerance) and
|
||||
p.lon >= ^(lon - tolerance) and p.lon <= ^(lon + tolerance),
|
||||
p.lon >= ^(lon - tolerance) and p.lon <= ^(lon + tolerance) and
|
||||
fragment("coalesce(cardinality(?), 0) >= ?", p.profile, ^min_levels),
|
||||
# squared-Euclidean distance is monotonic with great-circle at
|
||||
# this scale and avoids a trig call per row
|
||||
order_by: [
|
||||
|
|
|
|||
|
|
@ -4,6 +4,25 @@ defmodule Microwaveprop.Weather.HrrrProfileLookupTest do
|
|||
alias Microwaveprop.Weather.HrrrProfile
|
||||
alias Microwaveprop.Weather.HrrrProfileLookup
|
||||
|
||||
# 13-level standard HRRR profile so test fixtures pass the default
|
||||
# min_profile_levels filter. Tests that want to exercise the
|
||||
# partial-profile rejection path pass an explicit shorter profile.
|
||||
@full_profile [
|
||||
%{"pres" => 1000.0, "hght" => 100.0, "tmpc" => 25.0, "dwpc" => 18.0},
|
||||
%{"pres" => 925.0, "hght" => 800.0, "tmpc" => 19.0, "dwpc" => 14.0},
|
||||
%{"pres" => 850.0, "hght" => 1500.0, "tmpc" => 13.0, "dwpc" => 8.0},
|
||||
%{"pres" => 700.0, "hght" => 3100.0, "tmpc" => 2.0, "dwpc" => -8.0},
|
||||
%{"pres" => 500.0, "hght" => 5800.0, "tmpc" => -15.0, "dwpc" => -28.0},
|
||||
%{"pres" => 400.0, "hght" => 7200.0, "tmpc" => -28.0, "dwpc" => -42.0},
|
||||
%{"pres" => 300.0, "hght" => 9300.0, "tmpc" => -42.0, "dwpc" => -56.0},
|
||||
%{"pres" => 250.0, "hght" => 10_400.0, "tmpc" => -50.0, "dwpc" => -64.0},
|
||||
%{"pres" => 200.0, "hght" => 11_700.0, "tmpc" => -55.0, "dwpc" => -70.0},
|
||||
%{"pres" => 150.0, "hght" => 13_500.0, "tmpc" => -60.0, "dwpc" => -75.0},
|
||||
%{"pres" => 100.0, "hght" => 16_000.0, "tmpc" => -65.0, "dwpc" => -80.0},
|
||||
%{"pres" => 70.0, "hght" => 18_500.0, "tmpc" => -65.0, "dwpc" => -82.0},
|
||||
%{"pres" => 50.0, "hght" => 20_500.0, "tmpc" => -62.0, "dwpc" => -84.0}
|
||||
]
|
||||
|
||||
describe "list_valid_times_near/3" do
|
||||
test "returns valid_times of every profile within the spatial tolerance" do
|
||||
insert_profile!(%{lat: 32.625, lon: -97.125, valid_time: ~U[2026-04-25 14:00:00Z]})
|
||||
|
|
@ -51,22 +70,17 @@ defmodule Microwaveprop.Weather.HrrrProfileLookupTest do
|
|||
lat: 32.625,
|
||||
lon: -97.125,
|
||||
valid_time: ~U[2026-04-25 14:00:00Z],
|
||||
profile: [
|
||||
%{"pres" => 1000.0, "hght" => 100.0, "tmpc" => 20.0, "dwpc" => 15.0},
|
||||
%{"pres" => 850.0, "hght" => 1500.0, "tmpc" => 12.0, "dwpc" => 9.0}
|
||||
],
|
||||
surface_temp_c: 20.5,
|
||||
surface_dewpoint_c: 15.0,
|
||||
surface_pressure_mb: 1013.0
|
||||
})
|
||||
|
||||
assert %{} = cell = HrrrProfileLookup.read_point_near(~U[2026-04-25 14:00:00Z], 32.65, -97.13)
|
||||
# Returned cell preserves the profile array shape SkewtSvg expects.
|
||||
# Postgres jsonb collapses 1000.0 → 1000 on round-trip; accept
|
||||
# either since the renderer/normalizer handles both.
|
||||
# Default fixture is the 13-level @full_profile (HRRR's standard
|
||||
# canonical pressure-level set).
|
||||
assert length(cell[:profile]) == 13
|
||||
[%{"pres" => pres} | _] = cell[:profile]
|
||||
assert pres == 1000.0 or pres == 1000
|
||||
# Surface scalars are surfaced for SoundingParams.derive/1 reuse.
|
||||
assert cell[:surface_temp_c] == 20.5
|
||||
assert cell[:surface_pressure_mb] == 1013.0
|
||||
end
|
||||
|
|
@ -76,29 +90,76 @@ defmodule Microwaveprop.Weather.HrrrProfileLookupTest do
|
|||
end
|
||||
|
||||
test "picks the spatially closest cell when multiple match" do
|
||||
# Both cells use the 13-level default profile so they pass the
|
||||
# `min_profile_levels` filter; surface T/Td differ so we can tell
|
||||
# which row read_point_near returned.
|
||||
[first | rest] = @full_profile
|
||||
close_profile = [Map.put(first, "tmpc", 25.0) | rest]
|
||||
far_profile = [Map.put(first, "tmpc", 30.0) | rest]
|
||||
|
||||
insert_profile!(%{
|
||||
lat: 32.65,
|
||||
lon: -97.13,
|
||||
valid_time: ~U[2026-04-25 14:00:00Z],
|
||||
profile: [%{"pres" => 1000.0, "hght" => 100.0, "tmpc" => 25.0, "dwpc" => 18.0}]
|
||||
profile: close_profile
|
||||
})
|
||||
|
||||
insert_profile!(%{
|
||||
lat: 32.69,
|
||||
lon: -97.18,
|
||||
valid_time: ~U[2026-04-25 14:00:00Z],
|
||||
profile: [%{"pres" => 1000.0, "hght" => 100.0, "tmpc" => 30.0, "dwpc" => 25.0}]
|
||||
profile: far_profile
|
||||
})
|
||||
|
||||
cell = HrrrProfileLookup.read_point_near(~U[2026-04-25 14:00:00Z], 32.65, -97.13)
|
||||
# Closest cell is the first one (exact match).
|
||||
assert [%{"tmpc" => 25.0}] = cell[:profile]
|
||||
[%{"tmpc" => surface_t} | _] = cell[:profile]
|
||||
assert surface_t == 25.0
|
||||
end
|
||||
end
|
||||
|
||||
describe "min_profile_levels filtering" do
|
||||
test "list_valid_times_near rejects rows whose profile is shorter than the threshold" do
|
||||
# Two rows: one full (13 levels), one partial (3 levels).
|
||||
insert_profile!(%{lat: 32.625, lon: -97.125, valid_time: ~U[2026-04-25 14:00:00Z]})
|
||||
|
||||
insert_profile!(%{
|
||||
lat: 32.625,
|
||||
lon: -97.125,
|
||||
valid_time: ~U[2026-04-25 15:00:00Z],
|
||||
profile: [
|
||||
%{"pres" => 1000.0, "hght" => 100.0, "tmpc" => 25.0, "dwpc" => 18.0},
|
||||
%{"pres" => 850.0, "hght" => 1500.0, "tmpc" => 13.0, "dwpc" => 8.0},
|
||||
%{"pres" => 500.0, "hght" => 5800.0, "tmpc" => -15.0, "dwpc" => -28.0}
|
||||
]
|
||||
})
|
||||
|
||||
# Default threshold (13) → only the 14 :00 row.
|
||||
times = HrrrProfileLookup.list_valid_times_near(32.65, -97.13)
|
||||
assert Enum.map(times, &DateTime.to_iso8601/1) == ["2026-04-25T14:00:00Z"]
|
||||
|
||||
# Threshold of 0 → both rows.
|
||||
both = HrrrProfileLookup.list_valid_times_near(32.65, -97.13, min_profile_levels: 0)
|
||||
assert length(both) == 2
|
||||
end
|
||||
|
||||
test "read_point_near returns nil for partial-profile valid_times under the default threshold" do
|
||||
insert_profile!(%{
|
||||
lat: 32.625,
|
||||
lon: -97.125,
|
||||
valid_time: ~U[2026-04-25 15:00:00Z],
|
||||
profile: [%{"pres" => 1000.0, "hght" => 100.0, "tmpc" => 25.0, "dwpc" => 18.0}]
|
||||
})
|
||||
|
||||
assert HrrrProfileLookup.read_point_near(~U[2026-04-25 15:00:00Z], 32.65, -97.13) == nil
|
||||
|
||||
assert %{} =
|
||||
HrrrProfileLookup.read_point_near(~U[2026-04-25 15:00:00Z], 32.65, -97.13, min_profile_levels: 0)
|
||||
end
|
||||
end
|
||||
|
||||
defp insert_profile!(attrs) do
|
||||
defaults = %{
|
||||
profile: [%{"pres" => 1000.0, "hght" => 100.0, "tmpc" => 25.0, "dwpc" => 18.0}],
|
||||
profile: @full_profile,
|
||||
run_time: nil,
|
||||
hpbl_m: 800.0,
|
||||
pwat_mm: 25.0,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue