Remove 500-limit on BackfillEnqueueWorker cron

10K+ HRRR-pending contacts were starving the 200-ish NARR candidates
out of the 500 per-run slot each 30 min because NARR candidates sort
last by the :hrrr_status priority. Drop the cap so every eligible
contact gets enqueued on each cron run; queue concurrency still paces
the actual fetches. Worker keeps supporting an explicit "limit" arg
for ad-hoc runs and tests.
This commit is contained in:
Graham McIntire 2026-04-16 12:46:52 -05:00
parent 2871f63cca
commit ac4e43c726
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
3 changed files with 23 additions and 3 deletions

View file

@ -222,7 +222,7 @@ if config_env() == :prod do
# :unavailable (pre-2014, missing from the HRRR archive) and dispatches
# NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "narr"]}},
args: %{"types" => ["hrrr", "weather", "terrain", "iemre", "narr"]}},
# Hourly safety net for pos1/pos2/distance_km. Normally every contact
# gets positions at insert time via Radio.resolve_grids_and_insert/1,
# but direct DB writes (manual fixes, bulk imports) can bypass that.

View file

@ -32,8 +32,9 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
@stale_queued_cutoff_seconds 3 * 86_400
@impl Oban.Worker
def perform(%Oban.Job{args: %{"limit" => limit} = args}) do
def perform(%Oban.Job{args: args}) do
types = parse_types(args)
limit = Map.get(args, "limit")
reconciled = reconcile_stale_queued(types)
@ -52,7 +53,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|> where([c], not is_nil(c.pos1) or not is_nil(c.grid1))
|> where(^type_filter(types))
|> order_by(^status_priority_order(types))
|> limit(^limit)
|> maybe_limit(limit)
|> Repo.all()
count = length(contacts)
@ -69,6 +70,9 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
:ok
end
defp maybe_limit(query, nil), do: query
defp maybe_limit(query, n) when is_integer(n), do: limit(query, ^n)
defp parse_types(%{"types" => types}) when is_list(types) do
types |> Enum.filter(&(&1 in @valid_types)) |> Enum.map(&String.to_existing_atom/1)
end

View file

@ -109,6 +109,22 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
})
end
test "enqueues all pending contacts when limit is omitted" do
Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "backfill:enqueue_complete")
for i <- 1..7 do
ts = DateTime.add(~U[2026-01-01 00:00:00Z], i * 3600, :second)
create_contact(%{qso_timestamp: ts})
end
assert :ok =
BackfillEnqueueWorker.perform(%Oban.Job{
args: %{"types" => @non_era5_types}
})
assert_receive {:enqueue_complete, 7}
end
test "broadcasts enqueue_complete with count" do
Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "backfill:enqueue_complete")