NarrClient.in_coverage?/1 + callsite guards (1979 → 2014-10-02)
Post-2014 contacts with hrrr_status=:unavailable were being dispatched to NARR, which 404s because NCEI's archive ends 2014-10-02. 14K of the 14.3K candidates in prod are post-2014 — these are HRRR's responsibility. - NarrClient.in_coverage?/1 returns true only inside 1979-01-01 → 2014-10-02 - narr_jobs_for_contact and maybe_enqueue_narr (contact_live) bail out of coverage returning [] - BackfillEnqueueWorker.type_filter scopes :narr to qso_timestamp < coverage_end so the cron doesn't keep picking post-2014 candidates
This commit is contained in:
parent
f5132a991d
commit
55d4f289c2
6 changed files with 85 additions and 5 deletions
|
|
@ -20,6 +20,12 @@ defmodule Microwaveprop.Weather.NarrClient do
|
|||
@base_url "https://www.ncei.noaa.gov/data/north-american-regional-reanalysis/access/3-hourly"
|
||||
@valid_hours [0, 3, 6, 9, 12, 15, 18, 21]
|
||||
|
||||
# NCEI's NARR archive publishes 1979-01-01 through 2014-10-01 21 UTC
|
||||
# (the last 3-hourly analysis). Dispatching fetches outside that window
|
||||
# just 404s, so callers filter on `in_coverage?/1` first.
|
||||
@coverage_start ~U[1979-01-01 00:00:00Z]
|
||||
@coverage_end ~U[2014-10-02 00:00:00Z]
|
||||
|
||||
# Surface records we need from the inventory. Tuples of {var, level} keyed
|
||||
# exactly as `parse_inventory/2` returns them. Pressure-level records are
|
||||
# added dynamically below from whatever (TMP|HGT|SPFH):"NN mb" tuples the
|
||||
|
|
@ -89,6 +95,21 @@ defmodule Microwaveprop.Weather.NarrClient do
|
|||
%{datetime | hour: snapped_hour, minute: 0, second: 0, microsecond: {0, 0}}
|
||||
end
|
||||
|
||||
@doc """
|
||||
True when `datetime` falls inside NCEI's NARR 3-hourly archive window
|
||||
(1979-01-01 through 2014-10-01 21 UTC). Callers use this to avoid
|
||||
dispatching fetches that would just 404.
|
||||
"""
|
||||
@spec in_coverage?(DateTime.t()) :: boolean()
|
||||
def in_coverage?(%DateTime{} = datetime) do
|
||||
DateTime.compare(datetime, @coverage_start) != :lt and
|
||||
DateTime.before?(datetime, @coverage_end)
|
||||
end
|
||||
|
||||
@doc "Exclusive end of the NARR coverage window (first timestamp NOT covered)."
|
||||
@spec coverage_end() :: DateTime.t()
|
||||
def coverage_end, do: @coverage_end
|
||||
|
||||
@doc """
|
||||
Parses the raw text of a NARR `.inv` file into a map keyed by
|
||||
`{var, level}` with values of `{byte_offset, length}`.
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
|
||||
alias Microwaveprop.Radio.Contact
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Weather.NarrClient
|
||||
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
|
||||
|
||||
require Logger
|
||||
|
|
@ -147,8 +148,12 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
defp type_filter(types) do
|
||||
Enum.reduce(types, dynamic(false), fn
|
||||
:narr, acc ->
|
||||
# NARR targets contacts where HRRR is unavailable (pre-2014, missing from archive)
|
||||
dynamic([c], ^acc or c.hrrr_status == :unavailable)
|
||||
# NARR targets pre-2014 contacts where HRRR is unavailable. The NCEI
|
||||
# archive ends 2014-10-02 — post-cutoff fetches would 404, so scope
|
||||
# the candidate set here to keep the cron from scanning contacts we
|
||||
# can't serve.
|
||||
coverage_end = NarrClient.coverage_end()
|
||||
dynamic([c], ^acc or (c.hrrr_status == :unavailable and c.qso_timestamp < ^coverage_end))
|
||||
|
||||
type, acc ->
|
||||
field = :"#{type}_status"
|
||||
|
|
|
|||
|
|
@ -186,6 +186,17 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
# rejects anything else, so snap here in the caller.
|
||||
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
|
||||
|
||||
# NCEI's NARR archive covers 1979-01-01 → 2014-10-02. Post-coverage contacts
|
||||
# (hrrr_status = :unavailable but qso_timestamp >= 2014-10-02) are HRRR's
|
||||
# responsibility — NARR would just 404.
|
||||
if NarrClient.in_coverage?(valid_time) do
|
||||
narr_jobs_for_path(contact, valid_time)
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
defp narr_jobs_for_path(contact, valid_time) do
|
||||
contact
|
||||
|> Radio.contact_path_points()
|
||||
|> Enum.flat_map(fn {lat, lon} ->
|
||||
|
|
|
|||
|
|
@ -669,10 +669,9 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
|||
defp maybe_enqueue_narr(nil, %{hrrr_status: :unavailable} = contact) do
|
||||
lat = contact.pos1 && contact.pos1["lat"]
|
||||
lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"])
|
||||
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
|
||||
|
||||
if lat && lon do
|
||||
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
|
||||
|
||||
if lat && lon && NarrClient.in_coverage?(valid_time) do
|
||||
%{
|
||||
"lat" => lat,
|
||||
"lon" => lon,
|
||||
|
|
|
|||
|
|
@ -74,6 +74,27 @@ defmodule Microwaveprop.Weather.NarrClientTest do
|
|||
end
|
||||
end
|
||||
|
||||
# NCEI's NARR archive stops at 2014-10-02. Post-cutoff fetches hit a 404.
|
||||
# Callers filter on this helper to skip post-coverage dispatches.
|
||||
describe "in_coverage?/1" do
|
||||
test "true for dates inside NARR's 1979-01-01 → 2014-10-02 window" do
|
||||
assert NarrClient.in_coverage?(~U[1979-01-01 00:00:00Z])
|
||||
assert NarrClient.in_coverage?(~U[1995-07-04 12:00:00Z])
|
||||
assert NarrClient.in_coverage?(~U[2014-10-01 21:00:00Z])
|
||||
end
|
||||
|
||||
test "false on or after the 2014-10-02 cutoff" do
|
||||
refute NarrClient.in_coverage?(~U[2014-10-02 00:00:00Z])
|
||||
refute NarrClient.in_coverage?(~U[2020-09-20 21:00:00Z])
|
||||
refute NarrClient.in_coverage?(~U[2026-04-16 12:00:00Z])
|
||||
end
|
||||
|
||||
test "false before the 1979-01-01 start" do
|
||||
refute NarrClient.in_coverage?(~U[1978-12-31 23:00:00Z])
|
||||
refute NarrClient.in_coverage?(~U[1970-01-01 00:00:00Z])
|
||||
end
|
||||
end
|
||||
|
||||
describe "parse_inventory/2" do
|
||||
@fixture_path "test/fixtures/narr/narr-a_221_20100615_1200_000.inv"
|
||||
@fixture_file_size 56_221_430
|
||||
|
|
|
|||
|
|
@ -638,6 +638,29 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
|
|||
assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == []
|
||||
end
|
||||
|
||||
test "skips post-2014 contacts (outside NARR coverage — HRRR's job)" do
|
||||
# NARR's NCEI archive stops at 2014-10-02; post-coverage fetches 404.
|
||||
contact =
|
||||
create_contact(%{
|
||||
qso_timestamp: ~U[2020-09-20 21:00:00Z],
|
||||
pos1: %{"lat" => 43.25, "lon" => -78.25},
|
||||
pos2: nil
|
||||
})
|
||||
|
||||
assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == []
|
||||
end
|
||||
|
||||
test "skips pre-1979 contacts (below NARR coverage start)" do
|
||||
contact =
|
||||
create_contact(%{
|
||||
qso_timestamp: ~U[1975-06-15 12:00:00Z],
|
||||
pos1: %{"lat" => 32.9, "lon" => -97.0},
|
||||
pos2: nil
|
||||
})
|
||||
|
||||
assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == []
|
||||
end
|
||||
|
||||
test "deduplicates jobs with identical args across contacts" do
|
||||
q1 =
|
||||
create_contact(%{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue