NarrClient.in_coverage?/1 + callsite guards (1979 → 2014-10-02)

Post-2014 contacts with hrrr_status=:unavailable were being dispatched to
NARR, which 404s because NCEI's archive ends 2014-10-02. 14K of the 14.3K
candidates in prod are post-2014 — these are HRRR's responsibility.

- NarrClient.in_coverage?/1 returns true only inside 1979-01-01 → 2014-10-02
- narr_jobs_for_contact and maybe_enqueue_narr (contact_live) bail out of
  coverage returning []
- BackfillEnqueueWorker.type_filter scopes :narr to qso_timestamp <
  coverage_end so the cron doesn't keep picking post-2014 candidates
This commit is contained in:
Graham McIntire 2026-04-16 09:31:18 -05:00
parent f5132a991d
commit 55d4f289c2
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
6 changed files with 85 additions and 5 deletions

View file

@ -20,6 +20,12 @@ defmodule Microwaveprop.Weather.NarrClient do
@base_url "https://www.ncei.noaa.gov/data/north-american-regional-reanalysis/access/3-hourly"
@valid_hours [0, 3, 6, 9, 12, 15, 18, 21]
# NCEI's NARR archive publishes 1979-01-01 through 2014-10-01 21 UTC
# (the last 3-hourly analysis). Dispatching fetches outside that window
# just 404s, so callers filter on `in_coverage?/1` first.
@coverage_start ~U[1979-01-01 00:00:00Z]
@coverage_end ~U[2014-10-02 00:00:00Z]
# Surface records we need from the inventory. Tuples of {var, level} keyed
# exactly as `parse_inventory/2` returns them. Pressure-level records are
# added dynamically below from whatever (TMP|HGT|SPFH):"NN mb" tuples the
@ -89,6 +95,21 @@ defmodule Microwaveprop.Weather.NarrClient do
%{datetime | hour: snapped_hour, minute: 0, second: 0, microsecond: {0, 0}}
end
@doc """
True when `datetime` falls inside NCEI's NARR 3-hourly archive window
(1979-01-01 through 2014-10-01 21 UTC). Callers use this to avoid
dispatching fetches that would just 404.
"""
@spec in_coverage?(DateTime.t()) :: boolean()
def in_coverage?(%DateTime{} = datetime) do
DateTime.compare(datetime, @coverage_start) != :lt and
DateTime.before?(datetime, @coverage_end)
end
@doc "Exclusive end of the NARR coverage window (first timestamp NOT covered)."
@spec coverage_end() :: DateTime.t()
def coverage_end, do: @coverage_end
@doc """
Parses the raw text of a NARR `.inv` file into a map keyed by
`{var, level}` with values of `{byte_offset, length}`.

View file

@ -9,6 +9,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Repo
alias Microwaveprop.Weather.NarrClient
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
require Logger
@ -147,8 +148,12 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
defp type_filter(types) do
Enum.reduce(types, dynamic(false), fn
:narr, acc ->
# NARR targets contacts where HRRR is unavailable (pre-2014, missing from archive)
dynamic([c], ^acc or c.hrrr_status == :unavailable)
# NARR targets pre-2014 contacts where HRRR is unavailable. The NCEI
# archive ends 2014-10-02 — post-cutoff fetches would 404, so scope
# the candidate set here to keep the cron from scanning contacts we
# can't serve.
coverage_end = NarrClient.coverage_end()
dynamic([c], ^acc or (c.hrrr_status == :unavailable and c.qso_timestamp < ^coverage_end))
type, acc ->
field = :"#{type}_status"

View file

@ -186,6 +186,17 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
# rejects anything else, so snap here in the caller.
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
# NCEI's NARR archive covers 1979-01-01 → 2014-10-02. Post-coverage contacts
# (hrrr_status = :unavailable but qso_timestamp >= 2014-10-02) are HRRR's
# responsibility — NARR would just 404.
if NarrClient.in_coverage?(valid_time) do
narr_jobs_for_path(contact, valid_time)
else
[]
end
end
defp narr_jobs_for_path(contact, valid_time) do
contact
|> Radio.contact_path_points()
|> Enum.flat_map(fn {lat, lon} ->

View file

@ -669,10 +669,9 @@ defmodule MicrowavepropWeb.ContactLive.Show do
defp maybe_enqueue_narr(nil, %{hrrr_status: :unavailable} = contact) do
lat = contact.pos1 && contact.pos1["lat"]
lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"])
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
if lat && lon do
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
if lat && lon && NarrClient.in_coverage?(valid_time) do
%{
"lat" => lat,
"lon" => lon,

View file

@ -74,6 +74,27 @@ defmodule Microwaveprop.Weather.NarrClientTest do
end
end
# NCEI's NARR archive stops at 2014-10-02. Post-cutoff fetches hit a 404.
# Callers filter on this helper to skip post-coverage dispatches.
describe "in_coverage?/1" do
test "true for dates inside NARR's 1979-01-01 → 2014-10-02 window" do
assert NarrClient.in_coverage?(~U[1979-01-01 00:00:00Z])
assert NarrClient.in_coverage?(~U[1995-07-04 12:00:00Z])
assert NarrClient.in_coverage?(~U[2014-10-01 21:00:00Z])
end
test "false on or after the 2014-10-02 cutoff" do
refute NarrClient.in_coverage?(~U[2014-10-02 00:00:00Z])
refute NarrClient.in_coverage?(~U[2020-09-20 21:00:00Z])
refute NarrClient.in_coverage?(~U[2026-04-16 12:00:00Z])
end
test "false before the 1979-01-01 start" do
refute NarrClient.in_coverage?(~U[1978-12-31 23:00:00Z])
refute NarrClient.in_coverage?(~U[1970-01-01 00:00:00Z])
end
end
describe "parse_inventory/2" do
@fixture_path "test/fixtures/narr/narr-a_221_20100615_1200_000.inv"
@fixture_file_size 56_221_430

View file

@ -638,6 +638,29 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == []
end
test "skips post-2014 contacts (outside NARR coverage — HRRR's job)" do
# NARR's NCEI archive stops at 2014-10-02; post-coverage fetches 404.
contact =
create_contact(%{
qso_timestamp: ~U[2020-09-20 21:00:00Z],
pos1: %{"lat" => 43.25, "lon" => -78.25},
pos2: nil
})
assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == []
end
test "skips pre-1979 contacts (below NARR coverage start)" do
contact =
create_contact(%{
qso_timestamp: ~U[1975-06-15 12:00:00Z],
pos1: %{"lat" => 32.9, "lon" => -97.0},
pos2: nil
})
assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == []
end
test "deduplicates jobs with identical args across contacts" do
q1 =
create_contact(%{