prop/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex
Graham McIntire 3604896726
Rename ERA5 → NARR across the codebase
- Schema: Era5Profile → NarrProfile; table renamed via migration
  rename_era5_profiles_to_narr_profiles (ALTER TABLE + rename indexes,
  atomic and instant, no row rewrite)
- Weather helpers: find_nearest_era5/era5_for_contact/era5_profiles_for_path
  → find_nearest_narr/narr_for_contact/narr_profiles_for_path
- BackfillEnqueueWorker: :era5 → :narr type (virtual, no narr_status column);
  reconcile_stale_queued_to_unavailable and status_priority_order now skip
  virtual types via @virtual_types. Fixes the prod crash where the cron tried
  to update a non-existent era5_status column.
- ContactWeatherEnqueueWorker: build_era5_jobs → build_narr_jobs,
  era5_jobs_for_contact → narr_jobs_for_contact
- ContactLive: @era5 → @narr, maybe_enqueue_era5 → maybe_enqueue_narr;
  UI label "ERA5 (0.25°)" → "NARR (32 km)"
- Cron (runtime.exs): args types list now "narr" instead of "era5"
- /status: progress row, status-by-type card, totals.narr_profiles, and
  table-count lookup all target narr_profiles
- Drop obsolete Era5CdsJob schema + era5_cds_jobs table (inspection
  artifact from the retired CDS pipeline; 34 orphan rows in prod)
- Misc docstring/comment cleanups (skew_t, about, wgrib2, propagation_train)

Includes a regression test for the virtual-type crash.
2026-04-16 09:22:23 -05:00

330 lines
9.1 KiB
Elixir

defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
@moduledoc false
use Oban.Worker, queue: :enqueue, max_attempts: 3
alias Microwaveprop.Radio
alias Microwaveprop.Weather
alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Weather.NarrClient
alias Microwaveprop.Workers.HrrrFetchWorker
alias Microwaveprop.Workers.IemreFetchWorker
alias Microwaveprop.Workers.NarrFetchWorker
alias Microwaveprop.Workers.TerrainProfileWorker
alias Microwaveprop.Workers.WeatherFetchWorker
@asos_radius_km 150
@sounding_radius_km 300
# Oban jobs have ~149 params each; PG limit is 65535 params per query
@insert_batch_size 400
@doc """
Enqueue all enrichment jobs (weather, HRRR, terrain, IEMRE) for a single contact.
Called directly from submission flow — no Oban indirection.
Optionally pass a list of types to limit which enrichments run.
"""
def enqueue_for_contact(contact, types \\ [:hrrr, :weather, :terrain, :iemre]) do
contact = Radio.ensure_positions!(contact)
jobs_by_type = build_jobs_by_type(contact, types)
bulk_jobs =
jobs_by_type[:weather] ++ jobs_by_type[:hrrr] ++ jobs_by_type[:terrain] ++ jobs_by_type[:iemre]
if bulk_jobs != [] do
insert_all_chunked(bulk_jobs)
end
# NARR jobs must go through Oban.insert/1 so the worker's unique
# constraint is honored; Oban OSS insert_all does not check uniqueness.
insert_unique(jobs_by_type[:narr])
mark_enrichment_statuses(contact, types, jobs_by_type)
:ok
end
defp build_jobs_by_type(contact, types) do
%{
weather: if(:weather in types, do: build_weather_jobs([contact]), else: []),
hrrr: if(:hrrr in types, do: build_hrrr_jobs([contact]), else: []),
terrain: if(:terrain in types, do: build_terrain_jobs([contact]), else: []),
iemre: if(:iemre in types, do: build_iemre_jobs([contact]), else: []),
narr: if(:narr in types, do: build_narr_jobs([contact]), else: [])
}
end
defp mark_enrichment_statuses(contact, types, jobs_by_type) do
ids = [contact.id]
if contact.pos1 do
if :weather in types, do: mark_status!(ids, :weather_status, jobs_by_type[:weather])
if :hrrr in types, do: mark_status!(ids, :hrrr_status, jobs_by_type[:hrrr])
if :iemre in types, do: mark_status!(ids, :iemre_status, jobs_by_type[:iemre])
end
if contact.pos1 && contact.pos2 do
if :terrain in types, do: mark_status!(ids, :terrain_status, jobs_by_type[:terrain])
end
end
@impl Oban.Worker
def perform(%Oban.Job{}) do
enqueue_weather_jobs()
enqueue_hrrr_jobs()
enqueue_terrain_jobs()
enqueue_iemre_jobs()
:ok
end
defp enqueue_weather_jobs do
case Radio.unprocessed_contacts() do
[] ->
:ok
contacts ->
Radio.backfill_distances(contacts)
jobs = build_weather_jobs(contacts)
if jobs != [] do
insert_all_chunked(jobs)
end
ids = Enum.map(contacts, & &1.id)
Radio.mark_weather_queued!(ids)
enqueue_weather_jobs()
end
end
defp enqueue_hrrr_jobs do
case Radio.unprocessed_hrrr_contacts() do
[] ->
:ok
contacts ->
jobs = build_hrrr_jobs(contacts)
if jobs != [] do
insert_all_chunked(jobs)
end
ids = Enum.map(contacts, & &1.id)
Radio.mark_hrrr_queued!(ids)
enqueue_hrrr_jobs()
end
end
defp enqueue_terrain_jobs do
case Radio.unprocessed_terrain_contacts() do
[] ->
:ok
contacts ->
jobs = build_terrain_jobs(contacts)
if jobs != [] do
insert_all_chunked(jobs)
end
ids = Enum.map(contacts, & &1.id)
Radio.mark_terrain_queued!(ids)
enqueue_terrain_jobs()
end
end
def build_terrain_jobs(contacts) do
contacts
|> Enum.filter(fn contact -> contact.pos1 && contact.pos2 end)
|> Enum.map(fn contact ->
TerrainProfileWorker.new(%{"contact_id" => contact.id})
end)
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end
def build_weather_jobs(contacts) do
contacts
|> Enum.flat_map(&jobs_for_contact/1)
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end
def build_hrrr_jobs(contacts) do
contacts
|> Enum.flat_map(&hrrr_points_for_contact/1)
|> Enum.group_by(fn {_point, hour} -> hour end, fn {point, _hour} -> point end)
|> Enum.map(fn {hour, points} ->
unique_points =
points
|> Enum.uniq()
|> Enum.map(fn {lat, lon} -> %{"lat" => lat, "lon" => lon} end)
HrrrFetchWorker.new(%{
"points" => unique_points,
"valid_time" => DateTime.to_iso8601(hour)
})
end)
end
def build_iemre_jobs(contacts) do
contacts
|> Enum.flat_map(&iemre_job_for_contact/1)
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end
def build_narr_jobs(contacts) do
contacts
|> Enum.flat_map(&narr_jobs_for_contact/1)
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end
defp narr_jobs_for_contact(%{pos1: nil}), do: []
defp narr_jobs_for_contact(contact) do
# NARR analyses are 3-hourly (00/03/06/09/12/15/18/21 UTC). NarrFetchWorker
# rejects anything else, so snap here in the caller.
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
contact
|> Radio.contact_path_points()
|> Enum.flat_map(fn {lat, lon} ->
rlat = Float.round(lat * 4) / 4
rlon = Float.round(lon * 4) / 4
if Weather.find_nearest_narr(rlat, rlon, valid_time) do
[]
else
[
NarrFetchWorker.new(%{
"lat" => rlat,
"lon" => rlon,
"valid_time" => DateTime.to_iso8601(valid_time)
})
]
end
end)
end
defp jobs_for_contact(contact) do
contact
|> Radio.contact_path_points()
|> Enum.flat_map(fn {lat, lon} ->
build_asos_jobs(lat, lon, contact.qso_timestamp) ++
build_raob_jobs(lat, lon, contact.qso_timestamp)
end)
end
defp enqueue_iemre_jobs do
case Radio.unprocessed_iemre_contacts() do
[] ->
:ok
contacts ->
jobs = build_iemre_jobs(contacts)
if jobs != [] do
insert_all_chunked(jobs)
end
ids = Enum.map(contacts, & &1.id)
Radio.mark_iemre_queued!(ids)
enqueue_iemre_jobs()
end
end
defp hrrr_points_for_contact(%{pos1: nil}), do: []
defp hrrr_points_for_contact(contact) do
rounded_time = HrrrClient.nearest_hrrr_hour(contact.qso_timestamp)
contact
|> Radio.contact_path_points()
|> Enum.flat_map(fn {lat, lon} ->
{rlat, rlon} = Weather.round_to_hrrr_grid(lat, lon)
if Weather.has_hrrr_profile?(rlat, rlon, rounded_time) do
[]
else
[{{rlat, rlon}, rounded_time}]
end
end)
end
defp build_asos_jobs(lat, lon, timestamp) do
start_dt = DateTime.add(timestamp, -2 * 3600, :second)
end_dt = DateTime.add(timestamp, 2 * 3600, :second)
stations = Weather.nearby_stations(lat, lon, "asos", @asos_radius_km)
station_ids = Enum.map(stations, & &1.id)
covered = Weather.station_ids_with_surface_observations(station_ids, start_dt, end_dt)
stations
|> Enum.reject(fn station -> MapSet.member?(covered, station.id) end)
|> Enum.map(fn station ->
WeatherFetchWorker.new(%{
"fetch_type" => "asos",
"station_id" => station.id,
"station_code" => station.station_code,
"start_dt" => DateTime.to_iso8601(start_dt),
"end_dt" => DateTime.to_iso8601(end_dt)
})
end)
end
defp iemre_job_for_contact(%{pos1: nil}), do: []
defp iemre_job_for_contact(contact) do
date = DateTime.to_date(contact.qso_timestamp)
contact
|> Radio.contact_path_points()
|> Enum.flat_map(fn {lat, lon} ->
{rlat, rlon} = Weather.round_to_iemre_grid(lat, lon)
if Weather.has_iemre_observation?(rlat, rlon, date) do
[]
else
[
IemreFetchWorker.new(%{
"lat" => rlat,
"lon" => rlon,
"date" => Date.to_iso8601(date)
})
]
end
end)
end
defp build_raob_jobs(lat, lon, timestamp) do
sounding_times = Weather.sounding_times_around(timestamp)
stations = Weather.nearby_stations(lat, lon, "sounding", @sounding_radius_km)
station_ids = Enum.map(stations, & &1.id)
covered = Weather.station_ids_with_soundings(station_ids, sounding_times)
for station <- stations,
sounding_time <- sounding_times,
not MapSet.member?(covered, {station.id, sounding_time}) do
WeatherFetchWorker.new(%{
"fetch_type" => "raob",
"station_id" => station.id,
"station_code" => station.station_code,
"sounding_time" => DateTime.to_iso8601(sounding_time)
})
end
end
defp insert_all_chunked(jobs) do
jobs
|> Enum.chunk_every(@insert_batch_size)
|> Enum.each(&Oban.insert_all/1)
end
defp insert_unique(jobs) do
Enum.each(jobs, &Oban.insert/1)
end
defp mark_status!(ids, field, [_ | _]), do: Radio.set_enrichment_status!(ids, field, :queued)
defp mark_status!(ids, field, []), do: Radio.set_enrichment_status!(ids, field, :complete)
end