diff --git a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex index 7590d434..c34b125c 100644 --- a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex +++ b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex @@ -57,7 +57,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do if contact.pos1 do if :weather in types, do: mark_status!(ids, :weather_status, jobs_by_type[:weather]) - if :hrrr in types, do: mark_status!(ids, :hrrr_status, jobs_by_type[:hrrr]) + if :hrrr in types, do: mark_hrrr_status!(contact, ids, jobs_by_type[:hrrr]) if :iemre in types, do: mark_status!(ids, :iemre_status, jobs_by_type[:iemre]) end @@ -66,6 +66,20 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do end end + # Pre-2014 contacts have no HRRR data — NarrClient.in_coverage?/1 is the + # authoritative "HRRR can never serve this" check. Mark :unavailable so + # BackfillEnqueueWorker's :narr filter picks them up and the reconcile + # loop stops flipping the row back to :queued each cron cycle. + defp mark_hrrr_status!(contact, ids, []) do + if NarrClient.in_coverage?(contact.qso_timestamp) do + Radio.set_enrichment_status!(ids, :hrrr_status, :unavailable) + else + Radio.set_enrichment_status!(ids, :hrrr_status, :complete) + end + end + + defp mark_hrrr_status!(_contact, ids, [_ | _]), do: Radio.set_enrichment_status!(ids, :hrrr_status, :queued) + @impl Oban.Worker def perform(%Oban.Job{}) do enqueue_weather_jobs() @@ -248,19 +262,32 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do defp hrrr_points_for_contact(%{pos1: nil}), do: [] defp hrrr_points_for_contact(contact) do + # HRRR archive starts mid-2014; NARR covers the 1979 → 2014-10-02 window. + # Short-circuit here so build_hrrr_jobs returns nothing for pre-coverage + # contacts and mark_hrrr_status! can pin them to :unavailable. + if NarrClient.in_coverage?(contact.qso_timestamp) do + [] + else + hrrr_points_for_path(contact) + end + end + + defp hrrr_points_for_path(contact) do rounded_time = HrrrClient.nearest_hrrr_hour(contact.qso_timestamp) contact |> Radio.contact_path_points() - |> Enum.flat_map(fn {lat, lon} -> - {rlat, rlon} = Weather.round_to_hrrr_grid(lat, lon) + |> Enum.flat_map(&hrrr_point_or_skip(&1, rounded_time)) + end - if Weather.has_hrrr_profile?(rlat, rlon, rounded_time) do - [] - else - [{{rlat, rlon}, rounded_time}] - end - end) + defp hrrr_point_or_skip({lat, lon}, rounded_time) do + {rlat, rlon} = Weather.round_to_hrrr_grid(lat, lon) + + if Weather.has_hrrr_profile?(rlat, rlon, rounded_time) do + [] + else + [{{rlat, rlon}, rounded_time}] + end end defp build_asos_jobs(lat, lon, timestamp) do diff --git a/lib/microwaveprop_web/live/status_live.ex b/lib/microwaveprop_web/live/status_live.ex index 4125f888..9cbd2b08 100644 --- a/lib/microwaveprop_web/live/status_live.ex +++ b/lib/microwaveprop_web/live/status_live.ex @@ -7,6 +7,7 @@ defmodule MicrowavepropWeb.StatusLive do alias Microwaveprop.Cache alias Microwaveprop.Radio.Contact alias Microwaveprop.Repo + alias Microwaveprop.Weather.NarrClient @impl true def mount(_params, _session, socket) do @@ -76,9 +77,7 @@ defmodule MicrowavepropWeb.StatusLive do weather = count_incomplete(:weather_status, incomplete) iemre = count_incomplete(:iemre_status, incomplete) - narr_candidates = - Repo.one(from(c in Contact, where: c.hrrr_status == :unavailable and not is_nil(c.pos1), select: count())) - + narr_candidates = count_narr_candidates() narr_done = count_narr_done() all_done = count_all_done(done) @@ -97,23 +96,42 @@ defmodule MicrowavepropWeb.StatusLive do end # NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014). + # Candidate eligibility is defined by qso_timestamp, not hrrr_status — + # pre-2014 rows bounce between :queued and :unavailable every cron cycle + # and the UI must count them either way. + defp count_narr_candidates do + coverage_end = NarrClient.coverage_end() + + Repo.one( + from(c in Contact, + where: c.qso_timestamp < ^coverage_end and not is_nil(c.pos1), + select: count() + ) + ) + end + # "Done" means a narr_profiles row within 0.15° + 30 min of the contact, # matching `Weather.find_nearest_narr/3` lookup tolerance. defp count_narr_done do + coverage_end = NarrClient.coverage_end() + %{rows: [[done]]} = - Repo.query!(""" - SELECT count(*) - FROM contacts c - WHERE c.hrrr_status = 'unavailable' - AND c.pos1 IS NOT NULL - AND EXISTS ( - SELECT 1 - FROM narr_profiles p - WHERE p.lat BETWEEN ((c.pos1->>'lat')::float - 0.15) AND ((c.pos1->>'lat')::float + 0.15) - AND p.lon BETWEEN ((c.pos1->>'lon')::float - 0.15) AND ((c.pos1->>'lon')::float + 0.15) - AND p.valid_time BETWEEN (c.qso_timestamp - INTERVAL '30 minutes') AND (c.qso_timestamp + INTERVAL '30 minutes') - ) - """) + Repo.query!( + """ + SELECT count(*) + FROM contacts c + WHERE c.qso_timestamp < $1 + AND c.pos1 IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM narr_profiles p + WHERE p.lat BETWEEN ((c.pos1->>'lat')::float - 0.15) AND ((c.pos1->>'lat')::float + 0.15) + AND p.lon BETWEEN ((c.pos1->>'lon')::float - 0.15) AND ((c.pos1->>'lon')::float + 0.15) + AND p.valid_time BETWEEN (c.qso_timestamp - INTERVAL '30 minutes') AND (c.qso_timestamp + INTERVAL '30 minutes') + ) + """, + [coverage_end] + ) done end diff --git a/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs b/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs index 2e4570fe..a3887627 100644 --- a/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs +++ b/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs @@ -335,6 +335,17 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do assert ContactWeatherEnqueueWorker.build_hrrr_jobs([contact]) == [] end + # HRRR archive starts mid-2014; pre-coverage fetches always 404. Skipping + # here stops the :queued ↔ :unavailable ping-pong in BackfillEnqueueWorker + # where the reconcile flips pre-2014 rows to :unavailable and the same + # run immediately flips them back to :queued from a build_hrrr_jobs call + # that never had a chance of succeeding. + test "skips pre-2014 contacts (outside HRRR coverage — NARR's job)" do + contact = create_contact(%{qso_timestamp: ~U[2010-06-15 18:00:00Z]}) + + assert ContactWeatherEnqueueWorker.build_hrrr_jobs([contact]) == [] + end + test "excludes existing points from batch" do contact = create_contact(%{pos1: %{"lat" => 32.907, "lon" => -97.038}}) @@ -745,5 +756,18 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do assert updated.terrain_status == :pending assert updated.iemre_status == :queued end + + # Pre-2014 contacts have no HRRR data ever; mark hrrr_status :unavailable + # directly so BackfillEnqueueWorker's NARR filter picks them up and the + # reconcile/mark_status ping-pong stops. + test "pre-2014 contact: hrrr_status set to :unavailable and no HRRR jobs" do + _station = create_asos_station("KDFW", 32.90, -97.04) + contact = create_contact(%{qso_timestamp: ~U[2010-06-15 18:00:00Z]}) + + assert :ok = ContactWeatherEnqueueWorker.enqueue_for_contact(contact) + + updated = Repo.get!(Contact, contact.id) + assert updated.hrrr_status == :unavailable + end end end diff --git a/test/microwaveprop_web/live/status_live_test.exs b/test/microwaveprop_web/live/status_live_test.exs index d3956850..a512b389 100644 --- a/test/microwaveprop_web/live/status_live_test.exs +++ b/test/microwaveprop_web/live/status_live_test.exs @@ -67,4 +67,61 @@ defmodule MicrowavepropWeb.StatusLiveTest do narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render() assert narr_row =~ "1 / 1" end + + # NARR serves pre-2014 contacts only (NCEI archive ends 2014-10-02). A + # post-2014 contact with hrrr_status=:unavailable is a HRRR retry case, + # not a NARR candidate, and must not inflate the NARR denominator. + test "NARR progress excludes post-2014 hrrr-unavailable contacts", %{conn: conn} do + # pre-2014 candidate (counts toward NARR) + create_narr_candidate() + + # post-2014 hrrr-unavailable contact (does NOT count toward NARR) + {:ok, _} = + %Contact{} + |> Contact.changeset(%{ + station1: "W5POST", + station2: "K5POST", + qso_timestamp: ~U[2020-06-15 18:00:00Z], + mode: "CW", + band: Decimal.new("1296"), + grid1: "EM12", + grid2: "EM00", + pos1: %{"lat" => 33.0, "lon" => -97.0}, + pos2: %{"lat" => 30.3, "lon" => -97.7}, + distance_km: Decimal.new("300") + }) + |> Ecto.Changeset.change(%{hrrr_status: :unavailable}) + |> Repo.insert() + + {:ok, lv, _html} = live(conn, ~p"/status") + narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render() + assert narr_row =~ "0 / 1" + end + + # NARR eligibility is defined by qso_timestamp, not hrrr_status. A + # pre-2014 contact whose hrrr_status is still :queued (the ping-pong + # window before reconcile flips it) must still be counted — otherwise + # the denominator misses rows for several cron cycles after submission. + test "NARR candidates include pre-2014 contacts regardless of hrrr_status", %{conn: conn} do + {:ok, _} = + %Contact{} + |> Contact.changeset(%{ + station1: "W5QUE", + station2: "K5QUE", + qso_timestamp: ~U[2010-06-15 18:00:00Z], + mode: "CW", + band: Decimal.new("1296"), + grid1: "EM12", + grid2: "EM00", + pos1: %{"lat" => 33.0, "lon" => -97.0}, + pos2: %{"lat" => 30.3, "lon" => -97.7}, + distance_km: Decimal.new("300") + }) + |> Ecto.Changeset.change(%{hrrr_status: :queued}) + |> Repo.insert() + + {:ok, lv, _html} = live(conn, ~p"/status") + narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render() + assert narr_row =~ "0 / 1" + end end