Graham McIntire
6476bc8045
The Era5FetchWorker now declares an Oban unique constraint on
{lat, lon, valid_time} for any job in available/scheduled/executing/
retryable, so two backfill runs targeting the same contact grid point
can no longer spawn parallel CDS requests for the same hour.
Because Oban OSS insert_all doesn't honor unique, ERA5 jobs are now
routed through Oban.insert/1 from ContactWeatherEnqueueWorker and the
era5_backfill mix task. Other worker types still use insert_all.
76 lines
2.2 KiB
Elixir
76 lines
2.2 KiB
Elixir
defmodule Mix.Tasks.Era5Backfill do
|
|
@shortdoc "Enqueue ERA5 fetch jobs for contacts without HRRR data"
|
|
@moduledoc """
|
|
Finds contacts where HRRR status is 'unavailable' (pre-2014 or missing from archive)
|
|
and enqueues ERA5 fetch jobs for their path points.
|
|
|
|
Usage:
|
|
mix era5_backfill # enqueue all (default limit 1000)
|
|
mix era5_backfill 5000 # enqueue up to 5000
|
|
"""
|
|
use Mix.Task
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Radio
|
|
alias Microwaveprop.Radio.Contact
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Weather
|
|
alias Microwaveprop.Workers.Era5FetchWorker
|
|
|
|
@impl Mix.Task
|
|
def run(args) do
|
|
Mix.Task.run("app.start")
|
|
|
|
limit =
|
|
case args do
|
|
[n | _] -> String.to_integer(n)
|
|
_ -> 1000
|
|
end
|
|
|
|
contacts =
|
|
Contact
|
|
|> where([c], c.hrrr_status == :unavailable and not is_nil(c.pos1))
|
|
|> order_by([c], asc: c.qso_timestamp)
|
|
|> limit(^limit)
|
|
|> Repo.all()
|
|
|
|
Mix.shell().info("Found #{length(contacts)} contacts needing ERA5 data")
|
|
|
|
jobs =
|
|
contacts
|
|
|> Enum.flat_map(fn contact ->
|
|
contact
|
|
|> Radio.contact_path_points()
|
|
|> Enum.map(fn {lat, lon} ->
|
|
rlat = Float.round(lat * 4) / 4
|
|
rlon = Float.round(lon * 4) / 4
|
|
valid_time = %{DateTime.truncate(contact.qso_timestamp, :second) | minute: 0, second: 0}
|
|
|
|
if Weather.find_nearest_era5(rlat, rlon, valid_time) do
|
|
nil
|
|
else
|
|
Era5FetchWorker.new(%{
|
|
"lat" => rlat,
|
|
"lon" => rlon,
|
|
"valid_time" => DateTime.to_iso8601(valid_time)
|
|
})
|
|
end
|
|
end)
|
|
|> Enum.reject(&is_nil/1)
|
|
end)
|
|
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
|
|
|
|
if jobs == [] do
|
|
Mix.shell().info("No ERA5 jobs needed (all data already exists)")
|
|
else
|
|
# Use Oban.insert/1 so the worker's unique constraint collapses
|
|
# duplicate (lat, lon, valid_time) jobs that may already be in flight
|
|
# from a previous backfill run. Oban OSS insert_all does not check
|
|
# uniqueness.
|
|
Enum.each(jobs, &Oban.insert/1)
|
|
|
|
Mix.shell().info("Enqueued #{length(jobs)} ERA5 fetch jobs")
|
|
end
|
|
end
|
|
end
|