Fix NARR accounting on status page and stop pre-2014 hrrr_status churn

Status page NARR progress counted `hrrr_status = :unavailable` as the
candidate set, which swept in ~13.6K post-2014 HRRR retry rows and
showed "0 / 13,652". Candidate eligibility is qso_timestamp-based, not
hrrr_status-based — switch both the numerator and denominator to
`qso_timestamp < NarrClient.coverage_end()` so pre-2014 rows are
counted whether their hrrr_status is :queued, :unavailable, or
:complete.

ContactWeatherEnqueueWorker: short-circuit hrrr_points_for_contact/1
for pre-coverage timestamps and mark the contact :unavailable when no
HRRR jobs are built. Was producing a ping-pong where reconcile flipped
pre-2014 :queued → :unavailable and the same cron run flipped it back
to :queued from a rebuilt HRRR job that could never land data.

lib/microwaveprop/workers/contact_weather_enqueue_worker.ex

@@ -57,7 +57,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
 
     if contact.pos1 do
       if :weather in types, do: mark_status!(ids, :weather_status, jobs_by_type[:weather])
-      if :hrrr in types, do: mark_status!(ids, :hrrr_status, jobs_by_type[:hrrr])
+      if :hrrr in types, do: mark_hrrr_status!(contact, ids, jobs_by_type[:hrrr])
       if :iemre in types, do: mark_status!(ids, :iemre_status, jobs_by_type[:iemre])
     end
 
@@ -66,6 +66,20 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
     end
   end
 
+  # Pre-2014 contacts have no HRRR data — NarrClient.in_coverage?/1 is the
+  # authoritative "HRRR can never serve this" check. Mark :unavailable so
+  # BackfillEnqueueWorker's :narr filter picks them up and the reconcile
+  # loop stops flipping the row back to :queued each cron cycle.
+  defp mark_hrrr_status!(contact, ids, []) do
+    if NarrClient.in_coverage?(contact.qso_timestamp) do
+      Radio.set_enrichment_status!(ids, :hrrr_status, :unavailable)
+    else
+      Radio.set_enrichment_status!(ids, :hrrr_status, :complete)
+    end
+  end
+
+  defp mark_hrrr_status!(_contact, ids, [_ | _]), do: Radio.set_enrichment_status!(ids, :hrrr_status, :queued)
+
   @impl Oban.Worker
   def perform(%Oban.Job{}) do
     enqueue_weather_jobs()
@@ -248,19 +262,32 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
   defp hrrr_points_for_contact(%{pos1: nil}), do: []
 
   defp hrrr_points_for_contact(contact) do
+    # HRRR archive starts mid-2014; NARR covers the 1979 → 2014-10-02 window.
+    # Short-circuit here so build_hrrr_jobs returns nothing for pre-coverage
+    # contacts and mark_hrrr_status! can pin them to :unavailable.
+    if NarrClient.in_coverage?(contact.qso_timestamp) do
+      []
+    else
+      hrrr_points_for_path(contact)
+    end
+  end
+
+  defp hrrr_points_for_path(contact) do
     rounded_time = HrrrClient.nearest_hrrr_hour(contact.qso_timestamp)
 
     contact
     |> Radio.contact_path_points()
-    |> Enum.flat_map(fn {lat, lon} ->
-      {rlat, rlon} = Weather.round_to_hrrr_grid(lat, lon)
+    |> Enum.flat_map(&hrrr_point_or_skip(&1, rounded_time))
+  end
 
-      if Weather.has_hrrr_profile?(rlat, rlon, rounded_time) do
-        []
-      else
-        [{{rlat, rlon}, rounded_time}]
-      end
-    end)
+  defp hrrr_point_or_skip({lat, lon}, rounded_time) do
+    {rlat, rlon} = Weather.round_to_hrrr_grid(lat, lon)
+
+    if Weather.has_hrrr_profile?(rlat, rlon, rounded_time) do
+      []
+    else
+      [{{rlat, rlon}, rounded_time}]
+    end
   end
 
   defp build_asos_jobs(lat, lon, timestamp) do

lib/microwaveprop_web/live/status_live.ex

@@ -7,6 +7,7 @@ defmodule MicrowavepropWeb.StatusLive do
   alias Microwaveprop.Cache
   alias Microwaveprop.Radio.Contact
   alias Microwaveprop.Repo
+  alias Microwaveprop.Weather.NarrClient
 
   @impl true
   def mount(_params, _session, socket) do
@@ -76,9 +77,7 @@ defmodule MicrowavepropWeb.StatusLive do
     weather = count_incomplete(:weather_status, incomplete)
     iemre = count_incomplete(:iemre_status, incomplete)
 
-    narr_candidates =
-      Repo.one(from(c in Contact, where: c.hrrr_status == :unavailable and not is_nil(c.pos1), select: count()))
-
+    narr_candidates = count_narr_candidates()
     narr_done = count_narr_done()
 
     all_done = count_all_done(done)
@@ -97,23 +96,42 @@ defmodule MicrowavepropWeb.StatusLive do
   end
 
   # NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014).
+  # Candidate eligibility is defined by qso_timestamp, not hrrr_status —
+  # pre-2014 rows bounce between :queued and :unavailable every cron cycle
+  # and the UI must count them either way.
+  defp count_narr_candidates do
+    coverage_end = NarrClient.coverage_end()
+
+    Repo.one(
+      from(c in Contact,
+        where: c.qso_timestamp < ^coverage_end and not is_nil(c.pos1),
+        select: count()
+      )
+    )
+  end
+
   # "Done" means a narr_profiles row within 0.15° + 30 min of the contact,
   # matching `Weather.find_nearest_narr/3` lookup tolerance.
   defp count_narr_done do
+    coverage_end = NarrClient.coverage_end()
+
     %{rows: [[done]]} =
-      Repo.query!("""
-      SELECT count(*)
-        FROM contacts c
-       WHERE c.hrrr_status = 'unavailable'
-         AND c.pos1 IS NOT NULL
-         AND EXISTS (
-           SELECT 1
-             FROM narr_profiles p
-            WHERE p.lat BETWEEN ((c.pos1->>'lat')::float - 0.15) AND ((c.pos1->>'lat')::float + 0.15)
-              AND p.lon BETWEEN ((c.pos1->>'lon')::float - 0.15) AND ((c.pos1->>'lon')::float + 0.15)
-              AND p.valid_time BETWEEN (c.qso_timestamp - INTERVAL '30 minutes') AND (c.qso_timestamp + INTERVAL '30 minutes')
-         )
-      """)
+      Repo.query!(
+        """
+        SELECT count(*)
+          FROM contacts c
+         WHERE c.qso_timestamp < $1
+           AND c.pos1 IS NOT NULL
+           AND EXISTS (
+             SELECT 1
+               FROM narr_profiles p
+              WHERE p.lat BETWEEN ((c.pos1->>'lat')::float - 0.15) AND ((c.pos1->>'lat')::float + 0.15)
+                AND p.lon BETWEEN ((c.pos1->>'lon')::float - 0.15) AND ((c.pos1->>'lon')::float + 0.15)
+                AND p.valid_time BETWEEN (c.qso_timestamp - INTERVAL '30 minutes') AND (c.qso_timestamp + INTERVAL '30 minutes')
+           )
+        """,
+        [coverage_end]
+      )
 
     done
   end

test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs

@@ -335,6 +335,17 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
       assert ContactWeatherEnqueueWorker.build_hrrr_jobs([contact]) == []
     end
 
+    # HRRR archive starts mid-2014; pre-coverage fetches always 404. Skipping
+    # here stops the :queued ↔ :unavailable ping-pong in BackfillEnqueueWorker
+    # where the reconcile flips pre-2014 rows to :unavailable and the same
+    # run immediately flips them back to :queued from a build_hrrr_jobs call
+    # that never had a chance of succeeding.
+    test "skips pre-2014 contacts (outside HRRR coverage — NARR's job)" do
+      contact = create_contact(%{qso_timestamp: ~U[2010-06-15 18:00:00Z]})
+
+      assert ContactWeatherEnqueueWorker.build_hrrr_jobs([contact]) == []
+    end
+
     test "excludes existing points from batch" do
       contact = create_contact(%{pos1: %{"lat" => 32.907, "lon" => -97.038}})
 
@@ -745,5 +756,18 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
       assert updated.terrain_status == :pending
       assert updated.iemre_status == :queued
     end
+
+    # Pre-2014 contacts have no HRRR data ever; mark hrrr_status :unavailable
+    # directly so BackfillEnqueueWorker's NARR filter picks them up and the
+    # reconcile/mark_status ping-pong stops.
+    test "pre-2014 contact: hrrr_status set to :unavailable and no HRRR jobs" do
+      _station = create_asos_station("KDFW", 32.90, -97.04)
+      contact = create_contact(%{qso_timestamp: ~U[2010-06-15 18:00:00Z]})
+
+      assert :ok = ContactWeatherEnqueueWorker.enqueue_for_contact(contact)
+
+      updated = Repo.get!(Contact, contact.id)
+      assert updated.hrrr_status == :unavailable
+    end
   end
 end

test/microwaveprop_web/live/status_live_test.exs

@@ -67,4 +67,61 @@ defmodule MicrowavepropWeb.StatusLiveTest do
     narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render()
     assert narr_row =~ "1 / 1"
   end
+
+  # NARR serves pre-2014 contacts only (NCEI archive ends 2014-10-02). A
+  # post-2014 contact with hrrr_status=:unavailable is a HRRR retry case,
+  # not a NARR candidate, and must not inflate the NARR denominator.
+  test "NARR progress excludes post-2014 hrrr-unavailable contacts", %{conn: conn} do
+    # pre-2014 candidate (counts toward NARR)
+    create_narr_candidate()
+
+    # post-2014 hrrr-unavailable contact (does NOT count toward NARR)
+    {:ok, _} =
+      %Contact{}
+      |> Contact.changeset(%{
+        station1: "W5POST",
+        station2: "K5POST",
+        qso_timestamp: ~U[2020-06-15 18:00:00Z],
+        mode: "CW",
+        band: Decimal.new("1296"),
+        grid1: "EM12",
+        grid2: "EM00",
+        pos1: %{"lat" => 33.0, "lon" => -97.0},
+        pos2: %{"lat" => 30.3, "lon" => -97.7},
+        distance_km: Decimal.new("300")
+      })
+      |> Ecto.Changeset.change(%{hrrr_status: :unavailable})
+      |> Repo.insert()
+
+    {:ok, lv, _html} = live(conn, ~p"/status")
+    narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render()
+    assert narr_row =~ "0 / 1"
+  end
+
+  # NARR eligibility is defined by qso_timestamp, not hrrr_status. A
+  # pre-2014 contact whose hrrr_status is still :queued (the ping-pong
+  # window before reconcile flips it) must still be counted — otherwise
+  # the denominator misses rows for several cron cycles after submission.
+  test "NARR candidates include pre-2014 contacts regardless of hrrr_status", %{conn: conn} do
+    {:ok, _} =
+      %Contact{}
+      |> Contact.changeset(%{
+        station1: "W5QUE",
+        station2: "K5QUE",
+        qso_timestamp: ~U[2010-06-15 18:00:00Z],
+        mode: "CW",
+        band: Decimal.new("1296"),
+        grid1: "EM12",
+        grid2: "EM00",
+        pos1: %{"lat" => 33.0, "lon" => -97.0},
+        pos2: %{"lat" => 30.3, "lon" => -97.7},
+        distance_km: Decimal.new("300")
+      })
+      |> Ecto.Changeset.change(%{hrrr_status: :queued})
+      |> Repo.insert()
+
+    {:ok, lv, _html} = live(conn, ~p"/status")
+    narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render()
+    assert narr_row =~ "0 / 1"
+  end
 end