prop/lib/microwaveprop/weather/hrrr_profile_lookup.ex
Graham McIntire fb0c4f7c77
fix(skewt): only fall back to profiles with all 13 pressure levels
When the on-disk ProfilesFile is empty SkewtLive falls back to the
hrrr_profiles Postgres table, but per-QSO HrrrFetchWorker rows
sometimes land with a partial profile (4–12 levels) when individual
byte-range fetches fail. The fallback was happily picking those rows,
which rendered a stubby Skew-T missing the upper troposphere.

HrrrProfileLookup.{list_valid_times_near, read_point_near}/3 now
filter on `coalesce(cardinality(profile), 0) >= min_profile_levels`,
defaulting to 13 (HRRR's standard pressure-level set: 1000/925/850/
700/500/400/300/250/200/150/100 mb plus the two boundary surfaces).
Pass `min_profile_levels: 0` to keep the partial rows for diagnostic
queries.

Confirmed against the dev mirror: the recent table has ~3.1 M rows
at 13 levels and ~250 k at 4–12 levels; the new default skips the
partial slice cleanly.

Tests: 8/8 HrrrProfileLookupTest cases (added two new ones for the
threshold). mix credo --strict clean.
2026-04-25 14:19:56 -05:00

127 lines
5 KiB
Elixir
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

defmodule Microwaveprop.Weather.HrrrProfileLookup do
@moduledoc """
Read-side lookups against the `hrrr_profiles` table for clients that
cannot use the on-disk `Propagation.ProfilesFile` store — typically
the dev environment (no NFS mount, no live grid pipeline) or any
point query for a location the grid file doesn't cover.
Uses the same ±0.07° spatial bounding-box tolerance that
`Weather.find_nearest_hrrr/3` uses (one HRRR grid cell at 0.125° is
~0.07° at mid-latitudes). The 4-column `(lat, lon, valid_time)`
unique index makes the bbox + ORDER BY (squared distance) plan
index-only.
"""
import Ecto.Query
alias Microwaveprop.Repo
alias Microwaveprop.Weather.HrrrProfile
@default_tolerance_deg 0.07
@default_limit 32
# HRRR persists 13 standard pressure levels per profile (1000/925/
# 850/700/500/400/300/250/200/150/100 mb plus 2 boundary surfaces).
# Per-QSO HrrrFetchWorker rows sometimes land with a partial profile
# (411 levels) when individual byte-range fetches fail. SkewtLive
# only wants complete profiles — anything below this threshold draws
# a stubby diagram and gets filtered out by default.
@default_min_levels 13
@doc """
Distinct `valid_time`s with at least one HRRR profile within
`tolerance_deg` of the point, sorted ascending. The `:limit` keeps
the most recent N (then re-sorts ascending so the SkewtLive time
selector renders left-to-right in chronological order).
Options:
* `:tolerance_deg` — default `#{@default_tolerance_deg}`
* `:limit` — default `#{@default_limit}`
* `:min_profile_levels` — minimum array length on `profile` (default
`#{@default_min_levels}`). Use `0` to keep partial-profile rows.
"""
@spec list_valid_times_near(float(), float(), keyword()) :: [DateTime.t()]
def list_valid_times_near(lat, lon, opts \\ []) when is_number(lat) and is_number(lon) do
tolerance = Keyword.get(opts, :tolerance_deg, @default_tolerance_deg)
limit = Keyword.get(opts, :limit, @default_limit)
min_levels = Keyword.get(opts, :min_profile_levels, @default_min_levels)
query =
from p in HrrrProfile,
where:
p.lat >= ^(lat - tolerance) and p.lat <= ^(lat + tolerance) and
p.lon >= ^(lon - tolerance) and p.lon <= ^(lon + tolerance) and
fragment("coalesce(cardinality(?), 0) >= ?", p.profile, ^min_levels),
group_by: p.valid_time,
order_by: [desc: p.valid_time],
limit: ^limit,
select: p.valid_time
query
|> Repo.all()
|> Enum.map(&ensure_utc/1)
|> Enum.sort(DateTime)
end
@doc """
Returns the spatially closest HRRR profile within `tolerance_deg`
for the requested `valid_time`, or `nil` if none. Returned shape
matches what `Propagation.ProfilesFile.read_point/3` yields so
SkewtLive can substitute one for the other transparently.
Options:
* `:tolerance_deg` — default `#{@default_tolerance_deg}`
* `:min_profile_levels` — minimum array length on `profile` (default
`#{@default_min_levels}`). Use `0` to keep partial-profile rows.
"""
@spec read_point_near(DateTime.t(), float(), float(), keyword()) :: map() | nil
def read_point_near(%DateTime{} = valid_time, lat, lon, opts \\ []) when is_number(lat) and is_number(lon) do
tolerance = Keyword.get(opts, :tolerance_deg, @default_tolerance_deg)
min_levels = Keyword.get(opts, :min_profile_levels, @default_min_levels)
naive = DateTime.to_naive(valid_time)
query =
from p in HrrrProfile,
where:
p.valid_time == ^naive and
p.lat >= ^(lat - tolerance) and p.lat <= ^(lat + tolerance) and
p.lon >= ^(lon - tolerance) and p.lon <= ^(lon + tolerance) and
fragment("coalesce(cardinality(?), 0) >= ?", p.profile, ^min_levels),
# squared-Euclidean distance is monotonic with great-circle at
# this scale and avoids a trig call per row
order_by: [
asc: fragment("(? - ?)*(? - ?) + (? - ?)*(? - ?)", p.lat, ^lat, p.lat, ^lat, p.lon, ^lon, p.lon, ^lon)
],
limit: 1
case Repo.one(query) do
nil -> nil
%HrrrProfile{} = row -> to_cell(row)
end
end
defp to_cell(%HrrrProfile{} = row) do
%{
lat: row.lat,
lon: row.lon,
valid_time: ensure_utc(row.valid_time),
profile: row.profile || [],
hpbl_m: row.hpbl_m,
pwat_mm: row.pwat_mm,
surface_temp_c: row.surface_temp_c,
surface_dewpoint_c: row.surface_dewpoint_c,
surface_pressure_mb: row.surface_pressure_mb,
surface_refractivity: row.surface_refractivity,
min_refractivity_gradient: row.min_refractivity_gradient,
ducting_detected: row.ducting_detected,
duct_characteristics: row.duct_characteristics
}
end
defp ensure_utc(%DateTime{time_zone: "Etc/UTC"} = dt), do: dt
defp ensure_utc(%DateTime{} = dt), do: DateTime.shift_zone!(dt, "Etc/UTC")
defp ensure_utc(%NaiveDateTime{} = ndt) do
{:ok, dt} = DateTime.from_naive(ndt, "Etc/UTC")
dt
end
end