Refactor HRRR fetch to batch points per hour

QSO enrichment now groups all path points by HRRR hour and creates
one batch job per hour instead of one job per point. The batch job
downloads the GRIB2 data once and extracts all needed points from
the same binary. Legacy single-point jobs are still supported for
backward compatibility.

lib/microwaveprop/workers/hrrr_fetch_worker.ex

@@ -15,9 +15,38 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do
   end
 
   @impl Oban.Worker
-  def perform(%Oban.Job{args: args}) do
+  def perform(%Oban.Job{args: %{"points" => points, "valid_time" => valid_time_str}}) do
-    %{"lat" => raw_lat, "lon" => raw_lon, "valid_time" => valid_time_str} = args
+    # Batch mode: download GRIB2 once, extract multiple points
+    {:ok, valid_time, _} = DateTime.from_iso8601(valid_time_str)
 
+    point_tuples =
+      points
+      |> Enum.map(fn %{"lat" => lat, "lon" => lon} -> {lat, lon} end)
+      |> Enum.reject(fn {lat, lon} -> Weather.has_hrrr_profile?(lat, lon, valid_time) end)
+
+    if point_tuples == [] do
+      Logger.info("HRRR batch: all #{length(points)} points already exist for #{valid_time_str}")
+      :ok
+    else
+      Logger.info("HRRR batch: fetching #{length(point_tuples)} points for #{valid_time_str}")
+
+      case HrrrClient.fetch_grid(point_tuples, valid_time) do
+        {:ok, grid_data} ->
+          Enum.each(grid_data, fn {{lat, lon}, data} ->
+            store_profile(lat, lon, valid_time, data)
+          end)
+
+          Logger.info("HRRR batch: saved #{map_size(grid_data)} profiles for #{valid_time_str}")
+          :ok
+
+        {:error, reason} ->
+          handle_error(reason, "batch @ #{valid_time_str}")
+      end
+    end
+  end
+
+  def perform(%Oban.Job{args: %{"lat" => raw_lat, "lon" => raw_lon, "valid_time" => valid_time_str}}) do
+    # Legacy single-point mode
     {:ok, valid_time, _} = DateTime.from_iso8601(valid_time_str)
     {lat, lon} = Weather.round_to_hrrr_grid(raw_lat, raw_lon)
 
@@ -29,8 +58,17 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do
 
       case HrrrClient.fetch_profile(lat, lon, valid_time) do
         {:ok, data} ->
+          store_profile(lat, lon, valid_time, data)
+          :ok
+
+        {:error, reason} ->
+          handle_error(reason, "#{lat},#{lon} @ #{valid_time_str}")
+      end
+    end
+  end
+
+  defp store_profile(lat, lon, valid_time, data) do
     params = SoundingParams.derive(data.profile)
-          levels = length(data.profile)
 
     attrs =
       maybe_add_derived_params(
@@ -50,22 +88,17 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do
       )
 
     Weather.upsert_hrrr_profile(attrs)
+  end
 
-          Logger.info("HRRR profile saved for #{lat},#{lon} @ #{valid_time_str} (#{levels} levels, run #{data.run_time})")
+  defp handle_error(reason, label) do
-
-          :ok
-
-        {:error, reason} ->
     if transient_failure?(reason) do
-            Logger.error("HRRR transient error for #{lat},#{lon} @ #{valid_time_str}: #{inspect(reason)}")
+      Logger.error("HRRR transient error for #{label}: #{inspect(reason)}")
       {:error, reason}
     else
-            Logger.warning("HRRR permanent failure for #{lat},#{lon} @ #{valid_time_str}: #{inspect(reason)}")
+      Logger.warning("HRRR permanent failure for #{label}: #{inspect(reason)}")
       {:cancel, reason}
     end
   end
-    end
-  end
 
   # Only retry on transient network/server errors — everything else is permanent
   defp transient_failure?(%{__exception__: true}), do: true

lib/microwaveprop/workers/qso_weather_enqueue_worker.ex

@@ -100,9 +100,22 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorker do
   end
 
   def build_hrrr_jobs(qsos) do
+    # Group all QSO points by HRRR hour, then create one batch job per hour
+    # instead of one job per point. This deduplicates GRIB2 downloads.
     qsos
-    |> Enum.flat_map(&hrrr_job_for_qso/1)
+    |> Enum.flat_map(&hrrr_points_for_qso/1)
-    |> Enum.uniq_by(fn changeset -> changeset.changes.args end)
+    |> Enum.group_by(fn {_point, hour} -> hour end, fn {point, _hour} -> point end)
+    |> Enum.map(fn {hour, points} ->
+      unique_points =
+        points
+        |> Enum.uniq()
+        |> Enum.map(fn {lat, lon} -> %{"lat" => lat, "lon" => lon} end)
+
+      HrrrFetchWorker.new(%{
+        "points" => unique_points,
+        "valid_time" => DateTime.to_iso8601(hour)
+      })
+    end)
   end
 
   def build_iemre_jobs(qsos) do
@@ -139,9 +152,9 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorker do
     end
   end
 
-  defp hrrr_job_for_qso(%{pos1: nil}), do: []
+  defp hrrr_points_for_qso(%{pos1: nil}), do: []
 
-  defp hrrr_job_for_qso(qso) do
+  defp hrrr_points_for_qso(qso) do
     rounded_time = HrrrClient.nearest_hrrr_hour(qso.qso_timestamp)
 
     qso
@@ -152,13 +165,7 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorker do
       if Weather.has_hrrr_profile?(rlat, rlon, rounded_time) do
         []
       else
-        [
+        [{{rlat, rlon}, rounded_time}]
-          HrrrFetchWorker.new(%{
-            "lat" => rlat,
-            "lon" => rlon,
-            "valid_time" => DateTime.to_iso8601(rounded_time)
-          })
-        ]
       end
     end)
   end

test/microwaveprop/workers/qso_weather_enqueue_worker_test.exs

@@ -255,26 +255,31 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
   end
 
   describe "build_hrrr_jobs/1" do
-    test "builds HRRR jobs for all path points (pos1, midpoint, pos2)" do
+    test "batches all path points into one job per HRRR hour" do
       qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}})
 
       jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso])
 
-      # pos1, midpoint, pos2 = 3 distinct grid points
+      # All 3 path points share the same HRRR hour → 1 batch job
-      assert length(jobs) == 3
+      assert length(jobs) == 1
-      lats = jobs |> Enum.map(& &1.changes.args["lat"]) |> Enum.sort()
+      job = hd(jobs)
+      points = job.changes.args["points"]
+      # pos1, midpoint, pos2 = 3 distinct grid points in the batch
+      assert length(points) == 3
+      lats = Enum.map(points, & &1["lat"])
       assert 32.91 in lats
     end
 
-    test "builds only one HRRR job when pos2 is nil" do
+    test "batch contains single point when pos2 is nil" do
       qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}, pos2: nil})
 
       jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso])
 
       assert length(jobs) == 1
       job = hd(jobs)
-      assert job.changes.args["lat"] == 32.91
+      [point] = job.changes.args["points"]
-      assert job.changes.args["lon"] == -97.04
+      assert point["lat"] == 32.91
+      assert point["lon"] == -97.04
     end
 
     test "rounds valid_time to nearest hour" do
@@ -329,7 +334,7 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
       assert QsoWeatherEnqueueWorker.build_hrrr_jobs([qso]) == []
     end
 
-    test "skips only path points where HRRR profile already exists" do
+    test "excludes existing points from batch" do
       qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}})
 
       # Insert a profile at the pos1 rounded grid point only
@@ -348,9 +353,11 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
 
       jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso])
 
-      # pos1 skipped (exists), midpoint + pos2 still need fetching
+      # 1 batch job, but pos1 excluded from points list
-      assert length(jobs) == 2
+      assert length(jobs) == 1
-      lats = Enum.map(jobs, & &1.changes.args["lat"])
+      points = hd(jobs).changes.args["points"]
+      assert length(points) == 2
+      lats = Enum.map(points, & &1["lat"])
       refute 32.91 in lats
     end
   end