From 55d4f289c21aacf0ca2fb49630011570c4a6a1df Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 16 Apr 2026 09:31:18 -0500 Subject: [PATCH] =?UTF-8?q?NarrClient.in=5Fcoverage=3F/1=20+=20callsite=20?= =?UTF-8?q?guards=20(1979=20=E2=86=92=202014-10-02)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-2014 contacts with hrrr_status=:unavailable were being dispatched to NARR, which 404s because NCEI's archive ends 2014-10-02. 14K of the 14.3K candidates in prod are post-2014 — these are HRRR's responsibility. - NarrClient.in_coverage?/1 returns true only inside 1979-01-01 → 2014-10-02 - narr_jobs_for_contact and maybe_enqueue_narr (contact_live) bail out of coverage returning [] - BackfillEnqueueWorker.type_filter scopes :narr to qso_timestamp < coverage_end so the cron doesn't keep picking post-2014 candidates --- lib/microwaveprop/weather/narr_client.ex | 21 +++++++++++++++++ .../workers/backfill_enqueue_worker.ex | 9 ++++++-- .../workers/contact_weather_enqueue_worker.ex | 11 +++++++++ .../live/contact_live/show.ex | 5 ++-- .../weather/narr_client_test.exs | 21 +++++++++++++++++ .../contact_weather_enqueue_worker_test.exs | 23 +++++++++++++++++++ 6 files changed, 85 insertions(+), 5 deletions(-) diff --git a/lib/microwaveprop/weather/narr_client.ex b/lib/microwaveprop/weather/narr_client.ex index de29c547..b787794b 100644 --- a/lib/microwaveprop/weather/narr_client.ex +++ b/lib/microwaveprop/weather/narr_client.ex @@ -20,6 +20,12 @@ defmodule Microwaveprop.Weather.NarrClient do @base_url "https://www.ncei.noaa.gov/data/north-american-regional-reanalysis/access/3-hourly" @valid_hours [0, 3, 6, 9, 12, 15, 18, 21] + # NCEI's NARR archive publishes 1979-01-01 through 2014-10-01 21 UTC + # (the last 3-hourly analysis). Dispatching fetches outside that window + # just 404s, so callers filter on `in_coverage?/1` first. + @coverage_start ~U[1979-01-01 00:00:00Z] + @coverage_end ~U[2014-10-02 00:00:00Z] + # Surface records we need from the inventory. Tuples of {var, level} keyed # exactly as `parse_inventory/2` returns them. Pressure-level records are # added dynamically below from whatever (TMP|HGT|SPFH):"NN mb" tuples the @@ -89,6 +95,21 @@ defmodule Microwaveprop.Weather.NarrClient do %{datetime | hour: snapped_hour, minute: 0, second: 0, microsecond: {0, 0}} end + @doc """ + True when `datetime` falls inside NCEI's NARR 3-hourly archive window + (1979-01-01 through 2014-10-01 21 UTC). Callers use this to avoid + dispatching fetches that would just 404. + """ + @spec in_coverage?(DateTime.t()) :: boolean() + def in_coverage?(%DateTime{} = datetime) do + DateTime.compare(datetime, @coverage_start) != :lt and + DateTime.before?(datetime, @coverage_end) + end + + @doc "Exclusive end of the NARR coverage window (first timestamp NOT covered)." + @spec coverage_end() :: DateTime.t() + def coverage_end, do: @coverage_end + @doc """ Parses the raw text of a NARR `.inv` file into a map keyed by `{var, level}` with values of `{byte_offset, length}`. diff --git a/lib/microwaveprop/workers/backfill_enqueue_worker.ex b/lib/microwaveprop/workers/backfill_enqueue_worker.ex index d0e86592..f7593087 100644 --- a/lib/microwaveprop/workers/backfill_enqueue_worker.ex +++ b/lib/microwaveprop/workers/backfill_enqueue_worker.ex @@ -9,6 +9,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do alias Microwaveprop.Radio.Contact alias Microwaveprop.Repo + alias Microwaveprop.Weather.NarrClient alias Microwaveprop.Workers.ContactWeatherEnqueueWorker require Logger @@ -147,8 +148,12 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do defp type_filter(types) do Enum.reduce(types, dynamic(false), fn :narr, acc -> - # NARR targets contacts where HRRR is unavailable (pre-2014, missing from archive) - dynamic([c], ^acc or c.hrrr_status == :unavailable) + # NARR targets pre-2014 contacts where HRRR is unavailable. The NCEI + # archive ends 2014-10-02 — post-cutoff fetches would 404, so scope + # the candidate set here to keep the cron from scanning contacts we + # can't serve. + coverage_end = NarrClient.coverage_end() + dynamic([c], ^acc or (c.hrrr_status == :unavailable and c.qso_timestamp < ^coverage_end)) type, acc -> field = :"#{type}_status" diff --git a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex index 8245f0fb..7590d434 100644 --- a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex +++ b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex @@ -186,6 +186,17 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do # rejects anything else, so snap here in the caller. valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp) + # NCEI's NARR archive covers 1979-01-01 → 2014-10-02. Post-coverage contacts + # (hrrr_status = :unavailable but qso_timestamp >= 2014-10-02) are HRRR's + # responsibility — NARR would just 404. + if NarrClient.in_coverage?(valid_time) do + narr_jobs_for_path(contact, valid_time) + else + [] + end + end + + defp narr_jobs_for_path(contact, valid_time) do contact |> Radio.contact_path_points() |> Enum.flat_map(fn {lat, lon} -> diff --git a/lib/microwaveprop_web/live/contact_live/show.ex b/lib/microwaveprop_web/live/contact_live/show.ex index fe518c11..404cd3b1 100644 --- a/lib/microwaveprop_web/live/contact_live/show.ex +++ b/lib/microwaveprop_web/live/contact_live/show.ex @@ -669,10 +669,9 @@ defmodule MicrowavepropWeb.ContactLive.Show do defp maybe_enqueue_narr(nil, %{hrrr_status: :unavailable} = contact) do lat = contact.pos1 && contact.pos1["lat"] lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"]) + valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp) - if lat && lon do - valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp) - + if lat && lon && NarrClient.in_coverage?(valid_time) do %{ "lat" => lat, "lon" => lon, diff --git a/test/microwaveprop/weather/narr_client_test.exs b/test/microwaveprop/weather/narr_client_test.exs index 3eaf654e..b822d475 100644 --- a/test/microwaveprop/weather/narr_client_test.exs +++ b/test/microwaveprop/weather/narr_client_test.exs @@ -74,6 +74,27 @@ defmodule Microwaveprop.Weather.NarrClientTest do end end + # NCEI's NARR archive stops at 2014-10-02. Post-cutoff fetches hit a 404. + # Callers filter on this helper to skip post-coverage dispatches. + describe "in_coverage?/1" do + test "true for dates inside NARR's 1979-01-01 → 2014-10-02 window" do + assert NarrClient.in_coverage?(~U[1979-01-01 00:00:00Z]) + assert NarrClient.in_coverage?(~U[1995-07-04 12:00:00Z]) + assert NarrClient.in_coverage?(~U[2014-10-01 21:00:00Z]) + end + + test "false on or after the 2014-10-02 cutoff" do + refute NarrClient.in_coverage?(~U[2014-10-02 00:00:00Z]) + refute NarrClient.in_coverage?(~U[2020-09-20 21:00:00Z]) + refute NarrClient.in_coverage?(~U[2026-04-16 12:00:00Z]) + end + + test "false before the 1979-01-01 start" do + refute NarrClient.in_coverage?(~U[1978-12-31 23:00:00Z]) + refute NarrClient.in_coverage?(~U[1970-01-01 00:00:00Z]) + end + end + describe "parse_inventory/2" do @fixture_path "test/fixtures/narr/narr-a_221_20100615_1200_000.inv" @fixture_file_size 56_221_430 diff --git a/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs b/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs index 05ebbe13..2e4570fe 100644 --- a/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs +++ b/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs @@ -638,6 +638,29 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == [] end + test "skips post-2014 contacts (outside NARR coverage — HRRR's job)" do + # NARR's NCEI archive stops at 2014-10-02; post-coverage fetches 404. + contact = + create_contact(%{ + qso_timestamp: ~U[2020-09-20 21:00:00Z], + pos1: %{"lat" => 43.25, "lon" => -78.25}, + pos2: nil + }) + + assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == [] + end + + test "skips pre-1979 contacts (below NARR coverage start)" do + contact = + create_contact(%{ + qso_timestamp: ~U[1975-06-15 12:00:00Z], + pos1: %{"lat" => 32.9, "lon" => -97.0}, + pos2: nil + }) + + assert ContactWeatherEnqueueWorker.build_narr_jobs([contact]) == [] + end + test "deduplicates jobs with identical args across contacts" do q1 = create_contact(%{