Add type filter to backfill enqueue

Backfill dashboard now has checkboxes to select which enrichment types
to process (HRRR, Weather, Terrain, IEMRE). Prevents weather jobs from
flooding the queue when only HRRR needs work.

config/runtime.exs

@@ -46,6 +46,20 @@ if config_env() == :prod do
 
   host = System.get_env("PHX_HOST") || "example.com"
 
+  config :libcluster,
+    topologies: [
+      k8s: [
+        strategy: Cluster.Strategy.Kubernetes,
+        config: [
+          mode: :ip,
+          kubernetes_ip_lookup_mode: :pods,
+          kubernetes_node_basename: "microwaveprop",
+          kubernetes_selector: "app=prop",
+          kubernetes_namespace: "prop"
+        ]
+      ]
+    ]
+
   # ## SSL Support
   #
   # To get SSL working, you will need to add the `https` key
@@ -130,20 +144,6 @@ if config_env() == :prod do
        ]}
     ]
 
-  config :libcluster,
-    topologies: [
-      k8s: [
-        strategy: Cluster.Strategy.Kubernetes,
-        config: [
-          mode: :ip,
-          kubernetes_ip_lookup_mode: :pods,
-          kubernetes_node_basename: "microwaveprop",
-          kubernetes_selector: "app=prop",
-          kubernetes_namespace: "prop"
-        ]
-      ]
-    ]
-
   config :microwaveprop, :email_from, {"NTMS Propagation", "graham@w5isp.com"}
   config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080")
   config :microwaveprop, srtm_tiles_dir: "/srtm"

lib/microwaveprop/workers/backfill_enqueue_worker.ex

@@ -14,25 +14,24 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
   require Logger
 
   @enrichable [:pending, :queued, :failed]
+  @valid_types ~w(hrrr weather terrain iemre)
 
   @impl Oban.Worker
-  def perform(%Oban.Job{args: %{"limit" => limit}}) do
+  def perform(%Oban.Job{args: %{"limit" => limit} = args}) do
+    types = parse_types(args)
+
     contacts =
-      Repo.all(
+      Contact
-        from(c in Contact,
+      |> where([c], not is_nil(c.pos1) or not is_nil(c.grid1))
-          where:
+      |> where(^type_filter(types))
-            (not is_nil(c.pos1) or not is_nil(c.grid1)) and
+      |> order_by([c], desc: c.qso_timestamp)
-              (c.hrrr_status in ^@enrichable or c.weather_status in ^@enrichable or
+      |> limit(^limit)
-                 c.terrain_status in ^@enrichable or c.iemre_status in ^@enrichable),
+      |> Repo.all()
-          order_by: [desc: c.qso_timestamp],
-          limit: ^limit
-        )
-      )
 
     count = length(contacts)
-    Logger.info("BackfillEnqueue: processing #{count} contacts")
+    Logger.info("BackfillEnqueue: processing #{count} contacts (types: #{inspect(types)})")
 
-    Enum.each(contacts, &ContactWeatherEnqueueWorker.enqueue_for_contact/1)
+    Enum.each(contacts, &ContactWeatherEnqueueWorker.enqueue_for_contact(&1, types))
 
     Phoenix.PubSub.broadcast(
       Microwaveprop.PubSub,
@@ -42,4 +41,17 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
 
     :ok
   end
+
+  defp parse_types(%{"types" => types}) when is_list(types) do
+    types |> Enum.filter(&(&1 in @valid_types)) |> Enum.map(&String.to_existing_atom/1)
+  end
+
+  defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre]
+
+  defp type_filter(types) do
+    Enum.reduce(types, dynamic(false), fn type, acc ->
+      field = :"#{type}_status"
+      dynamic([c], ^acc or field(c, ^field) in ^@enrichable)
+    end)
+  end
 end

lib/microwaveprop/workers/contact_weather_enqueue_worker.ex

@@ -18,13 +18,14 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
   @doc """
   Enqueue all enrichment jobs (weather, HRRR, terrain, IEMRE) for a single contact.
   Called directly from submission flow — no Oban indirection.
+  Optionally pass a list of types to limit which enrichments run.
   """
-  def enqueue_for_contact(contact) do
+  def enqueue_for_contact(contact, types \\ [:hrrr, :weather, :terrain, :iemre]) do
     contact = Radio.ensure_positions!(contact)
-    weather_jobs = build_weather_jobs([contact])
+    weather_jobs = if :weather in types, do: build_weather_jobs([contact]), else: []
-    hrrr_jobs = build_hrrr_jobs([contact])
+    hrrr_jobs = if :hrrr in types, do: build_hrrr_jobs([contact]), else: []
-    terrain_jobs = build_terrain_jobs([contact])
+    terrain_jobs = if :terrain in types, do: build_terrain_jobs([contact]), else: []
-    iemre_jobs = build_iemre_jobs([contact])
+    iemre_jobs = if :iemre in types, do: build_iemre_jobs([contact]), else: []
 
     all_jobs = weather_jobs ++ hrrr_jobs ++ terrain_jobs ++ iemre_jobs
 
@@ -35,13 +36,13 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
     ids = [contact.id]
 
     if contact.pos1 do
-      mark_status!(ids, :weather_status, weather_jobs)
+      if :weather in types, do: mark_status!(ids, :weather_status, weather_jobs)
-      mark_status!(ids, :hrrr_status, hrrr_jobs)
+      if :hrrr in types, do: mark_status!(ids, :hrrr_status, hrrr_jobs)
-      mark_status!(ids, :iemre_status, iemre_jobs)
+      if :iemre in types, do: mark_status!(ids, :iemre_status, iemre_jobs)
     end
 
     if contact.pos1 && contact.pos2 do
-      mark_status!(ids, :terrain_status, terrain_jobs)
+      if :terrain in types, do: mark_status!(ids, :terrain_status, terrain_jobs)
     end
 
     :ok

lib/microwaveprop/workers/weather_fetch_worker.ex

@@ -5,14 +5,14 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do
     max_attempts: 20,
     unique: [period: 300, states: [:available, :scheduled, :executing, :retryable]]
 
-  require Logger
-
   alias Microwaveprop.Repo
   alias Microwaveprop.Weather
   alias Microwaveprop.Weather.IemClient
   alias Microwaveprop.Weather.SoundingParams
   alias Microwaveprop.Weather.Station
 
+  require Logger
+
   @impl Oban.Worker
   def backoff(%Oban.Job{attempt: attempt}) do
     min(120 * Integer.pow(2, attempt - 1), _six_hours = 21_600)

lib/microwaveprop_web/live/backfill_live.ex

@@ -41,10 +41,18 @@ defmodule MicrowavepropWeb.BackfillLive do
   end
 
   @impl true
-  def handle_event("enqueue", %{"limit" => limit_str}, socket) do
+  def handle_event("enqueue", %{"limit" => limit_str} = params, socket) do
     limit = String.to_integer(limit_str)
 
-    %{"limit" => limit}
+    types =
+      ~w(hrrr weather terrain iemre)
+      |> Enum.filter(&(params[&1] == "true"))
+      |> then(fn
+        [] -> ~w(hrrr weather terrain iemre)
+        types -> types
+      end)
+
+    %{"limit" => limit, "types" => types}
     |> BackfillEnqueueWorker.new()
     |> Oban.insert()
 
@@ -295,7 +303,8 @@ defmodule MicrowavepropWeb.BackfillLive do
           </div>
           <div class="flex flex-col gap-1.5 mt-3">
             <%= for {label, complete} <- [{"HRRR", @unprocessed.total - @unprocessed.hrrr}, {"Weather", @unprocessed.total - @unprocessed.weather}, {"Terrain", @unprocessed.total - @unprocessed.terrain}, {"IEMRE", @unprocessed.total - @unprocessed.iemre}] do %>
-              <% pct = if @unprocessed.total > 0, do: round(complete * 100 / @unprocessed.total), else: 0 %>
+              <% pct =
+                if @unprocessed.total > 0, do: round(complete * 100 / @unprocessed.total), else: 0 %>
               <div class="flex items-center gap-2 text-xs">
                 <span class="w-14 shrink-0">{label}</span>
                 <progress
@@ -327,7 +336,7 @@ defmodule MicrowavepropWeb.BackfillLive do
       </div>
 
       <div class="bg-base-200 rounded-box p-4 mb-6">
-        <form phx-submit="enqueue" class="flex items-end gap-4">
+        <form phx-submit="enqueue" class="flex items-end gap-4 flex-wrap">
           <div>
             <label class="label text-sm">Contacts to enqueue</label>
             <input
@@ -339,6 +348,14 @@ defmodule MicrowavepropWeb.BackfillLive do
               class="input input-bordered w-32"
             />
           </div>
+          <div class="flex gap-3 items-center">
+            <%= for type <- ~w(hrrr weather terrain iemre) do %>
+              <label class="label cursor-pointer gap-1">
+                <input type="checkbox" name={type} value="true" checked class="checkbox checkbox-sm" />
+                <span class="text-xs uppercase">{type}</span>
+              </label>
+            <% end %>
+          </div>
           <button class="btn btn-primary" disabled={@enqueuing}>
             <%= if @enqueuing do %>
               <span class="loading loading-spinner loading-sm"></span> Enqueuing...

lib/microwaveprop_web/router.ex

@@ -1,6 +1,8 @@
 defmodule MicrowavepropWeb.Router do
   use MicrowavepropWeb, :router
 
+  import Phoenix.LiveDashboard.Router
+
   pipeline :browser do
     plug :accepts, ["html"]
     plug :fetch_session
@@ -47,8 +49,6 @@ defmodule MicrowavepropWeb.Router do
   #   pipe_through :api
   # end
 
-  import Phoenix.LiveDashboard.Router
-
   scope "/" do
     pipe_through :browser
 

tail_logs.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+stern -n prop -l app=prop

test/microwaveprop/workers/solar_index_worker_test.exs

@@ -49,7 +49,7 @@ defmodule Microwaveprop.Workers.SolarIndexWorkerTest do
 
   describe "perform/1 with date arg" do
     test "fetches and upserts only the matching date" do
-      today = Date.utc_today() |> Date.to_iso8601()
+      today = Date.to_iso8601(Date.utc_today())
 
       Req.Test.stub(SolarClient, fn conn ->
         Req.Test.text(conn, gfz_body_with_recent_dates())
@@ -64,7 +64,7 @@ defmodule Microwaveprop.Workers.SolarIndexWorkerTest do
         Plug.Conn.send_resp(conn, 500, "Internal Server Error")
       end)
 
-      today = Date.utc_today() |> Date.to_iso8601()
+      today = Date.to_iso8601(Date.utc_today())
       assert {:error, "HTTP 500"} = SolarIndexWorker.perform(%Oban.Job{args: %{"date" => today}})
     end
   end

test/microwaveprop_web/plugs/remote_ip_test.exs

@@ -13,7 +13,8 @@ defmodule MicrowavepropWeb.Plugs.RemoteIpTest do
   describe "call/2" do
     test "extracts IP from cf-connecting-ip header" do
       conn =
-        conn(:get, "/")
+        :get
+        |> conn("/")
         |> put_req_header("cf-connecting-ip", "1.2.3.4")
         |> call_plug()
 
@@ -22,7 +23,8 @@ defmodule MicrowavepropWeb.Plugs.RemoteIpTest do
 
     test "falls back to x-forwarded-for when no cf-connecting-ip" do
       conn =
-        conn(:get, "/")
+        :get
+        |> conn("/")
         |> put_req_header("x-forwarded-for", "5.6.7.8")
         |> call_plug()
 
@@ -31,7 +33,8 @@ defmodule MicrowavepropWeb.Plugs.RemoteIpTest do
 
     test "prefers cf-connecting-ip over x-forwarded-for" do
       conn =
-        conn(:get, "/")
+        :get
+        |> conn("/")
         |> put_req_header("cf-connecting-ip", "1.2.3.4")
         |> put_req_header("x-forwarded-for", "5.6.7.8")
         |> call_plug()
@@ -41,7 +44,8 @@ defmodule MicrowavepropWeb.Plugs.RemoteIpTest do
 
     test "takes first IP from comma-separated x-forwarded-for" do
       conn =
-        conn(:get, "/")
+        :get
+        |> conn("/")
         |> put_req_header("x-forwarded-for", "10.0.0.1, 10.0.0.2, 10.0.0.3")
         |> call_plug()
 
@@ -52,7 +56,8 @@ defmodule MicrowavepropWeb.Plugs.RemoteIpTest do
       original_ip = {127, 0, 0, 1}
 
       conn =
-        conn(:get, "/")
+        :get
+        |> conn("/")
         |> Map.put(:remote_ip, original_ip)
         |> call_plug()
 
@@ -61,7 +66,8 @@ defmodule MicrowavepropWeb.Plugs.RemoteIpTest do
 
     test "handles IPv6 addresses" do
       conn =
-        conn(:get, "/")
+        :get
+        |> conn("/")
         |> put_req_header("cf-connecting-ip", "::1")
         |> call_plug()
 
@@ -72,7 +78,8 @@ defmodule MicrowavepropWeb.Plugs.RemoteIpTest do
       original_ip = {127, 0, 0, 1}
 
       conn =
-        conn(:get, "/")
+        :get
+        |> conn("/")
         |> Map.put(:remote_ip, original_ip)
         |> put_req_header("cf-connecting-ip", "not-an-ip")
         |> call_plug()