Remove 500-limit on BackfillEnqueueWorker cron
10K+ HRRR-pending contacts were starving the 200-ish NARR candidates out of the 500 per-run slot each 30 min because NARR candidates sort last by the :hrrr_status priority. Drop the cap so every eligible contact gets enqueued on each cron run; queue concurrency still paces the actual fetches. Worker keeps supporting an explicit "limit" arg for ad-hoc runs and tests.
This commit is contained in:
parent
2871f63cca
commit
ac4e43c726
3 changed files with 23 additions and 3 deletions
|
|
@ -222,7 +222,7 @@ if config_env() == :prod do
|
|||
# :unavailable (pre-2014, missing from the HRRR archive) and dispatches
|
||||
# NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
|
||||
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
|
||||
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "narr"]}},
|
||||
args: %{"types" => ["hrrr", "weather", "terrain", "iemre", "narr"]}},
|
||||
# Hourly safety net for pos1/pos2/distance_km. Normally every contact
|
||||
# gets positions at insert time via Radio.resolve_grids_and_insert/1,
|
||||
# but direct DB writes (manual fixes, bulk imports) can bypass that.
|
||||
|
|
|
|||
|
|
@ -32,8 +32,9 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
@stale_queued_cutoff_seconds 3 * 86_400
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{args: %{"limit" => limit} = args}) do
|
||||
def perform(%Oban.Job{args: args}) do
|
||||
types = parse_types(args)
|
||||
limit = Map.get(args, "limit")
|
||||
|
||||
reconciled = reconcile_stale_queued(types)
|
||||
|
||||
|
|
@ -52,7 +53,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
|> where([c], not is_nil(c.pos1) or not is_nil(c.grid1))
|
||||
|> where(^type_filter(types))
|
||||
|> order_by(^status_priority_order(types))
|
||||
|> limit(^limit)
|
||||
|> maybe_limit(limit)
|
||||
|> Repo.all()
|
||||
|
||||
count = length(contacts)
|
||||
|
|
@ -69,6 +70,9 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
:ok
|
||||
end
|
||||
|
||||
defp maybe_limit(query, nil), do: query
|
||||
defp maybe_limit(query, n) when is_integer(n), do: limit(query, ^n)
|
||||
|
||||
defp parse_types(%{"types" => types}) when is_list(types) do
|
||||
types |> Enum.filter(&(&1 in @valid_types)) |> Enum.map(&String.to_existing_atom/1)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -109,6 +109,22 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorkerTest do
|
|||
})
|
||||
end
|
||||
|
||||
test "enqueues all pending contacts when limit is omitted" do
|
||||
Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "backfill:enqueue_complete")
|
||||
|
||||
for i <- 1..7 do
|
||||
ts = DateTime.add(~U[2026-01-01 00:00:00Z], i * 3600, :second)
|
||||
create_contact(%{qso_timestamp: ts})
|
||||
end
|
||||
|
||||
assert :ok =
|
||||
BackfillEnqueueWorker.perform(%Oban.Job{
|
||||
args: %{"types" => @non_era5_types}
|
||||
})
|
||||
|
||||
assert_receive {:enqueue_complete, 7}
|
||||
end
|
||||
|
||||
test "broadcasts enqueue_complete with count" do
|
||||
Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "backfill:enqueue_complete")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue