fix(enrichment): mark OCONUS contacts hrrr_status :unavailable
Root cause of 14 stuck hrrr-queued contacts: hrrr_point_rs's CONUS-only grid silently writes zero profiles for out-of-bounds points (UK, Winnipeg, Alberta, >50° lat), but the fetch task still completes successfully. ContactWeatherEnqueueWorker's hrrr_placeholder_jobs always emitted a queued placeholder for any contact with a pos1, which meant mark_hrrr_status! re-flagged :queued every backfill tick forever. Introduce Propagation.Grid.contains?/1 as the canonical in-CONUS check (inclusive on all four edges, nil-safe). Use it in two places of ContactWeatherEnqueueWorker: - hrrr_placeholder_jobs/1 now returns [] for OCONUS contacts, same as pre-2014 / already-complete ones. - mark_hrrr_status!/3 empty-list clause now branches on Grid.contains? first (→ :unavailable) before falling through to the existing NARR-coverage / :complete logic. Next backfill tick will flip the 14 stuck OCONUS contacts in prod to :unavailable automatically. The remaining 13 "CONUS" stuck ones are right at the 50° edge (lat=50.1875 rounds outside the box) — same reconciliation path. Coverage additions: - Grid.contains?/1 — corner-inclusive, OCONUS positive/negative fixtures based on the actual stuck-contact pos1 values, nil/missing- key/nil-value handling. - enqueue_for_contact regression tests: pos1 in the UK and pos1 just north of the lat cap both land at :unavailable, not :queued. Suite: 2,416 tests + 159 properties (was 2,409 + 159); credo strict clean.
This commit is contained in:
parent
8435d95900
commit
54cabdb609
4 changed files with 102 additions and 8 deletions
|
|
@ -24,6 +24,25 @@ defmodule Microwaveprop.Propagation.Grid do
|
|||
@spec bounds() :: %{lat_min: float(), lat_max: float(), lon_min: float(), lon_max: float()}
|
||||
def bounds, do: %{lat_min: @lat_min, lat_max: @lat_max, lon_min: @lon_min, lon_max: @lon_max}
|
||||
|
||||
@doc """
|
||||
True when the given position lies inside the CONUS bounding box
|
||||
(inclusive on all four edges). Callers use this to gate HRRR
|
||||
enrichment — contacts whose path origin is outside the grid would
|
||||
be enqueued forever because `hrrr_point_rs` silently writes no
|
||||
profiles for out-of-grid points.
|
||||
|
||||
Accepts the canonical `%{"lat" => _, "lon" => _}` pos map. Returns
|
||||
`false` for nil, missing keys, or nil numeric values.
|
||||
"""
|
||||
@spec contains?(map() | nil) :: boolean()
|
||||
def contains?(nil), do: false
|
||||
|
||||
def contains?(%{"lat" => lat, "lon" => lon}) when is_number(lat) and is_number(lon) do
|
||||
lat >= @lat_min and lat <= @lat_max and lon >= @lon_min and lon <= @lon_max
|
||||
end
|
||||
|
||||
def contains?(_), do: false
|
||||
|
||||
@doc "Returns the grid specification for wgrib2 -lola extraction."
|
||||
@spec wgrib2_grid_spec() :: %{
|
||||
lon_start: float(),
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
@moduledoc false
|
||||
use Oban.Worker, queue: :enqueue, max_attempts: 3
|
||||
|
||||
alias Microwaveprop.Propagation.Grid
|
||||
alias Microwaveprop.Radio
|
||||
alias Microwaveprop.Radio.Contact
|
||||
alias Microwaveprop.Weather
|
||||
|
|
@ -97,6 +98,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
Enum.flat_map(contacts, fn contact ->
|
||||
cond do
|
||||
is_nil(contact.pos1) -> []
|
||||
not Grid.contains?(contact.pos1) -> []
|
||||
NarrClient.in_coverage?(contact.qso_timestamp) -> []
|
||||
Weather.hrrr_data_fully_present?(contact) -> []
|
||||
true -> [contact.id]
|
||||
|
|
@ -159,15 +161,24 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
|
||||
defp mark_radar_status!(_contact, ids, [_ | _]), do: Radio.set_enrichment_status!(ids, :radar_status, :queued)
|
||||
|
||||
# Pre-2014 contacts have no HRRR data — NarrClient.in_coverage?/1 is the
|
||||
# authoritative "HRRR can never serve this" check. Mark :unavailable so
|
||||
# BackfillEnqueueWorker's :narr filter picks them up and the reconcile
|
||||
# loop stops flipping the row back to :queued each cron cycle.
|
||||
# Empty-job-list HRRR path has three terminal states:
|
||||
#
|
||||
# 1. pos1 outside the CONUS grid → :unavailable. hrrr_point_rs silently
|
||||
# returns zero profiles for OCONUS points and the backfill cron
|
||||
# would otherwise re-flag :queued every tick forever.
|
||||
# 2. pre-2014 contacts (NarrClient.in_coverage?) → :unavailable so the
|
||||
# :narr filter in BackfillEnqueueWorker picks them up.
|
||||
# 3. modern CONUS contact whose profiles already exist → :complete.
|
||||
defp mark_hrrr_status!(contact, ids, []) do
|
||||
if NarrClient.in_coverage?(contact.qso_timestamp) do
|
||||
Radio.set_enrichment_status!(ids, :hrrr_status, :unavailable)
|
||||
else
|
||||
Radio.set_enrichment_status!(ids, :hrrr_status, :complete)
|
||||
cond do
|
||||
not Grid.contains?(contact.pos1) ->
|
||||
Radio.set_enrichment_status!(ids, :hrrr_status, :unavailable)
|
||||
|
||||
NarrClient.in_coverage?(contact.qso_timestamp) ->
|
||||
Radio.set_enrichment_status!(ids, :hrrr_status, :unavailable)
|
||||
|
||||
true ->
|
||||
Radio.set_enrichment_status!(ids, :hrrr_status, :complete)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -54,6 +54,39 @@ defmodule Microwaveprop.Propagation.GridTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "contains?/1" do
|
||||
test "true for DFW (firmly inside the box)" do
|
||||
assert Grid.contains?(%{"lat" => 32.9, "lon" => -97.0})
|
||||
end
|
||||
|
||||
test "true for every corner of the CONUS bounding box (inclusive)" do
|
||||
assert Grid.contains?(%{"lat" => 25.0, "lon" => -125.0})
|
||||
assert Grid.contains?(%{"lat" => 50.0, "lon" => -66.0})
|
||||
assert Grid.contains?(%{"lat" => 25.0, "lon" => -66.0})
|
||||
assert Grid.contains?(%{"lat" => 50.0, "lon" => -125.0})
|
||||
end
|
||||
|
||||
test "false for OCONUS positions (UK, Winnipeg, Alberta, etc.)" do
|
||||
# 27 stuck hrrr-queued contacts in prod landed on points like
|
||||
# these. Add them as regression fixtures.
|
||||
refute Grid.contains?(%{"lat" => 51.4, "lon" => 0.47})
|
||||
refute Grid.contains?(%{"lat" => 51.39, "lon" => -114.04})
|
||||
refute Grid.contains?(%{"lat" => 50.1875, "lon" => -97.625})
|
||||
end
|
||||
|
||||
test "false for nil pos or missing keys" do
|
||||
refute Grid.contains?(nil)
|
||||
refute Grid.contains?(%{})
|
||||
refute Grid.contains?(%{"lat" => 32.9})
|
||||
refute Grid.contains?(%{"lon" => -97.0})
|
||||
end
|
||||
|
||||
test "false when lat or lon is nil" do
|
||||
refute Grid.contains?(%{"lat" => nil, "lon" => -97.0})
|
||||
refute Grid.contains?(%{"lat" => 32.9, "lon" => nil})
|
||||
end
|
||||
end
|
||||
|
||||
describe "wgrib2_grid_spec/0" do
|
||||
test "lon_start / lat_start match the bounding box SW corner" do
|
||||
spec = Grid.wgrib2_grid_spec()
|
||||
|
|
|
|||
|
|
@ -286,6 +286,37 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
|||
assert [] = Repo.all(from t in "hrrr_fetch_tasks", select: t.id)
|
||||
end
|
||||
|
||||
test "enqueue_for_contact marks hrrr_status :unavailable when pos1 is outside the CONUS grid" do
|
||||
# Regression: 14 stuck hrrr-queued contacts in prod had pos1 in
|
||||
# the UK, Winnipeg, or Alberta. The Rust hrrr_point_rs worker
|
||||
# completes the fetch task silently (CONUS-only grid yields zero
|
||||
# profiles) and the backfill cron re-flags the contact :queued
|
||||
# forever. OCONUS contacts should land directly at :unavailable.
|
||||
contact =
|
||||
create_contact(%{
|
||||
pos1: %{"lat" => 51.4, "lon" => 0.47},
|
||||
pos2: %{"lat" => 51.45, "lon" => 0.44},
|
||||
qso_timestamp: ~U[2023-01-19 00:00:00Z]
|
||||
})
|
||||
|
||||
:ok = ContactWeatherEnqueueWorker.enqueue_for_contact(contact, [:hrrr])
|
||||
|
||||
assert %Contact{hrrr_status: :unavailable} = Repo.get!(Contact, contact.id)
|
||||
end
|
||||
|
||||
test "enqueue_for_contact marks hrrr_status :unavailable when pos1 is just north of the 50° lat cap" do
|
||||
contact =
|
||||
create_contact(%{
|
||||
pos1: %{"lat" => 50.1875, "lon" => -97.625},
|
||||
pos2: %{"lat" => 50.2, "lon" => -97.6},
|
||||
qso_timestamp: ~U[2023-09-17 17:00:00Z]
|
||||
})
|
||||
|
||||
:ok = ContactWeatherEnqueueWorker.enqueue_for_contact(contact, [:hrrr])
|
||||
|
||||
assert %Contact{hrrr_status: :unavailable} = Repo.get!(Contact, contact.id)
|
||||
end
|
||||
|
||||
test "enqueue_for_contact marks hrrr_status :complete when all path points already have profiles" do
|
||||
# Regression: ~22k contacts stuck at :queued forever because
|
||||
# HrrrPointEnqueuer saw 0 missing points (data already landed)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue