When the on-disk ProfilesFile is empty SkewtLive falls back to the
hrrr_profiles Postgres table, but per-QSO HrrrFetchWorker rows
sometimes land with a partial profile (4–12 levels) when individual
byte-range fetches fail. The fallback was happily picking those rows,
which rendered a stubby Skew-T missing the upper troposphere.
HrrrProfileLookup.{list_valid_times_near, read_point_near}/3 now
filter on `coalesce(cardinality(profile), 0) >= min_profile_levels`,
defaulting to 13 (HRRR's standard pressure-level set: 1000/925/850/
700/500/400/300/250/200/150/100 mb plus the two boundary surfaces).
Pass `min_profile_levels: 0` to keep the partial rows for diagnostic
queries.
Confirmed against the dev mirror: the recent table has ~3.1 M rows
at 13 levels and ~250 k at 4–12 levels; the new default skips the
partial slice cleanly.
Tests: 8/8 HrrrProfileLookupTest cases (added two new ones for the
threshold). mix credo --strict clean.
127 lines
5 KiB
Elixir
127 lines
5 KiB
Elixir
defmodule Microwaveprop.Weather.HrrrProfileLookup do
|
||
@moduledoc """
|
||
Read-side lookups against the `hrrr_profiles` table for clients that
|
||
cannot use the on-disk `Propagation.ProfilesFile` store — typically
|
||
the dev environment (no NFS mount, no live grid pipeline) or any
|
||
point query for a location the grid file doesn't cover.
|
||
|
||
Uses the same ±0.07° spatial bounding-box tolerance that
|
||
`Weather.find_nearest_hrrr/3` uses (one HRRR grid cell at 0.125° is
|
||
~0.07° at mid-latitudes). The 4-column `(lat, lon, valid_time)`
|
||
unique index makes the bbox + ORDER BY (squared distance) plan
|
||
index-only.
|
||
"""
|
||
|
||
import Ecto.Query
|
||
|
||
alias Microwaveprop.Repo
|
||
alias Microwaveprop.Weather.HrrrProfile
|
||
|
||
@default_tolerance_deg 0.07
|
||
@default_limit 32
|
||
# HRRR persists 13 standard pressure levels per profile (1000/925/
|
||
# 850/700/500/400/300/250/200/150/100 mb plus 2 boundary surfaces).
|
||
# Per-QSO HrrrFetchWorker rows sometimes land with a partial profile
|
||
# (4–11 levels) when individual byte-range fetches fail. SkewtLive
|
||
# only wants complete profiles — anything below this threshold draws
|
||
# a stubby diagram and gets filtered out by default.
|
||
@default_min_levels 13
|
||
|
||
@doc """
|
||
Distinct `valid_time`s with at least one HRRR profile within
|
||
`tolerance_deg` of the point, sorted ascending. The `:limit` keeps
|
||
the most recent N (then re-sorts ascending so the SkewtLive time
|
||
selector renders left-to-right in chronological order).
|
||
|
||
Options:
|
||
* `:tolerance_deg` — default `#{@default_tolerance_deg}`
|
||
* `:limit` — default `#{@default_limit}`
|
||
* `:min_profile_levels` — minimum array length on `profile` (default
|
||
`#{@default_min_levels}`). Use `0` to keep partial-profile rows.
|
||
"""
|
||
@spec list_valid_times_near(float(), float(), keyword()) :: [DateTime.t()]
|
||
def list_valid_times_near(lat, lon, opts \\ []) when is_number(lat) and is_number(lon) do
|
||
tolerance = Keyword.get(opts, :tolerance_deg, @default_tolerance_deg)
|
||
limit = Keyword.get(opts, :limit, @default_limit)
|
||
min_levels = Keyword.get(opts, :min_profile_levels, @default_min_levels)
|
||
|
||
query =
|
||
from p in HrrrProfile,
|
||
where:
|
||
p.lat >= ^(lat - tolerance) and p.lat <= ^(lat + tolerance) and
|
||
p.lon >= ^(lon - tolerance) and p.lon <= ^(lon + tolerance) and
|
||
fragment("coalesce(cardinality(?), 0) >= ?", p.profile, ^min_levels),
|
||
group_by: p.valid_time,
|
||
order_by: [desc: p.valid_time],
|
||
limit: ^limit,
|
||
select: p.valid_time
|
||
|
||
query
|
||
|> Repo.all()
|
||
|> Enum.map(&ensure_utc/1)
|
||
|> Enum.sort(DateTime)
|
||
end
|
||
|
||
@doc """
|
||
Returns the spatially closest HRRR profile within `tolerance_deg`
|
||
for the requested `valid_time`, or `nil` if none. Returned shape
|
||
matches what `Propagation.ProfilesFile.read_point/3` yields so
|
||
SkewtLive can substitute one for the other transparently.
|
||
|
||
Options:
|
||
* `:tolerance_deg` — default `#{@default_tolerance_deg}`
|
||
* `:min_profile_levels` — minimum array length on `profile` (default
|
||
`#{@default_min_levels}`). Use `0` to keep partial-profile rows.
|
||
"""
|
||
@spec read_point_near(DateTime.t(), float(), float(), keyword()) :: map() | nil
|
||
def read_point_near(%DateTime{} = valid_time, lat, lon, opts \\ []) when is_number(lat) and is_number(lon) do
|
||
tolerance = Keyword.get(opts, :tolerance_deg, @default_tolerance_deg)
|
||
min_levels = Keyword.get(opts, :min_profile_levels, @default_min_levels)
|
||
naive = DateTime.to_naive(valid_time)
|
||
|
||
query =
|
||
from p in HrrrProfile,
|
||
where:
|
||
p.valid_time == ^naive and
|
||
p.lat >= ^(lat - tolerance) and p.lat <= ^(lat + tolerance) and
|
||
p.lon >= ^(lon - tolerance) and p.lon <= ^(lon + tolerance) and
|
||
fragment("coalesce(cardinality(?), 0) >= ?", p.profile, ^min_levels),
|
||
# squared-Euclidean distance is monotonic with great-circle at
|
||
# this scale and avoids a trig call per row
|
||
order_by: [
|
||
asc: fragment("(? - ?)*(? - ?) + (? - ?)*(? - ?)", p.lat, ^lat, p.lat, ^lat, p.lon, ^lon, p.lon, ^lon)
|
||
],
|
||
limit: 1
|
||
|
||
case Repo.one(query) do
|
||
nil -> nil
|
||
%HrrrProfile{} = row -> to_cell(row)
|
||
end
|
||
end
|
||
|
||
defp to_cell(%HrrrProfile{} = row) do
|
||
%{
|
||
lat: row.lat,
|
||
lon: row.lon,
|
||
valid_time: ensure_utc(row.valid_time),
|
||
profile: row.profile || [],
|
||
hpbl_m: row.hpbl_m,
|
||
pwat_mm: row.pwat_mm,
|
||
surface_temp_c: row.surface_temp_c,
|
||
surface_dewpoint_c: row.surface_dewpoint_c,
|
||
surface_pressure_mb: row.surface_pressure_mb,
|
||
surface_refractivity: row.surface_refractivity,
|
||
min_refractivity_gradient: row.min_refractivity_gradient,
|
||
ducting_detected: row.ducting_detected,
|
||
duct_characteristics: row.duct_characteristics
|
||
}
|
||
end
|
||
|
||
defp ensure_utc(%DateTime{time_zone: "Etc/UTC"} = dt), do: dt
|
||
defp ensure_utc(%DateTime{} = dt), do: DateTime.shift_zone!(dt, "Etc/UTC")
|
||
|
||
defp ensure_utc(%NaiveDateTime{} = ndt) do
|
||
{:ok, dt} = DateTime.from_naive(ndt, "Etc/UTC")
|
||
dt
|
||
end
|
||
end
|