prop/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex
Graham McIntire 0c3be97abb
Some checks failed
Build base image / Build and push base image (push) Successful in 3m10s
Build and Push / Build and Push Docker Image (push) Failing after 14s
Build prop-grid-rs / Test, build, push (push) Successful in 12m52s
fix: resolve 27 security, architecture, test, and performance audit findings
P0 (security-critical):
- Gate CSV/ADIF upload tabs behind authentication, add 30s cooldown to all upload handlers
- Cap CSV/ADIF imports at 2,000 rows server-side in both parsers
- Add submitter_verified boolean to contacts (client-cannot-set, anonymous=false)
- Create k8s/secret.example.yaml with placeholders, add LIVE_VIEW_SIGNING_SALT

P1 (high-priority):
- Add Mox.verify_on_exit!() to valkey_test.exs
- Replace DateTime.utc_now() truncation with static ~U literals in map_live_test.exs
- Replace Process.sleep with render_async in pskr_spots_live_test.exs (6 occurrences)
- Add MonitorLive.Show test coverage (4 tests: owner view, non-owner redirect, config success/error)
- Extract duct-detection and mechanism-classification logic from ContactLive.Show into Propagation.PathAnalysis
- Split ContactLive.Show render into 12 function components
- Update CLAUDE.md: remove stale ML model, mark HRDPS active, add backtest/pskr dirs
- Batch CSV import enrichment jobs via new enqueue_for_contacts/1

P2 (medium-priority):
- Set secure:true on session and remember-me cookies in production
- Change SMTP TLS from verify_none to verify_peer with public_key cacerts
- Make /metrics fail-closed in production when PROMETHEUS_AUTH_TOKEN unset
- Add RateLimiter (anon_limit:10, auth_limit:60) to /api/contacts/map
- Add content-security-policy-report-only header
- Add comment noting String.to_atom is compile-time safe in hrdps_client.ex
- Delegate duplicated haversine_km to canonical Microwaveprop.Geo.haversine_km/4
- Consolidate score-tier/color/verdict formatting into Microwaveprop.Format
- Update CLAUDE.md testing section to match actual raw-string-matching practice
- Batch HrrrPointEnqueuer Repo.insert_all calls to single round-trip
- Split weather.ex (1696→216 lines) and radio.ex (1285→54 lines) into purpose-based sub-facades

P3 (low-priority):
- Add LIVE_VIEW_SIGNING_SALT warning comment, extend filter_parameters
- Add host/community validation to snmp_client.ex
- Add raw/1 safety comment in algo_live.ex
- Add hex-audit and cargo-audit Makefile targets
- Add privacy_live smoke test
- Replace notify_listener busy-poll loop with Process.monitor/1 + assert_receive
- Add ContactCommonVolumeRadar changeset validation tests (5 tests)
2026-07-27 18:19:37 -05:00

679 lines
23 KiB
Elixir
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
@moduledoc false
use Oban.Pro.Worker, queue: :enqueue, max_attempts: 3
alias Microwaveprop.Propagation.Grid
alias Microwaveprop.Radio
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Weather
alias Microwaveprop.Weather.HrrrPointEnqueuer
alias Microwaveprop.Weather.NarrClient
alias Microwaveprop.Weather.NexradClient
alias Microwaveprop.Workers.IemreFetchWorker
alias Microwaveprop.Workers.MechanismClassifyWorker
alias Microwaveprop.Workers.NarrFetchWorker
alias Microwaveprop.Workers.RadarFrameWorker
alias Microwaveprop.Workers.TerrainProfileWorker
alias Microwaveprop.Workers.WeatherFetchWorker
require Logger
@asos_radius_km 150
@sounding_radius_km 300
# Oban jobs have ~149 params each; PG limit is 65535 params per query
@insert_batch_size 400
@doc """
Batch-enqueue all enrichment jobs (weather, HRRR, terrain, radar,
mechanism, IEMRE, NARR) for a list of contacts. Used by CSV/ADIF
import to avoid N+1 per-contact enqueuing — all jobs are built and
inserted in bulk, and enrichment statuses are marked for every
contact in one pass.
"""
@spec enqueue_for_contacts([Contact.t()]) :: :ok
def enqueue_for_contacts(contacts) when is_list(contacts) do
if contacts == [] do
:ok
else
do_enqueue_for_contacts(contacts)
end
end
defp do_enqueue_for_contacts(contacts) do
contacts = Enum.map(contacts, &Radio.ensure_positions!/1)
{narr_contacts, modern_contacts} = Enum.split_with(contacts, &NarrClient.in_coverage?(&1.qso_timestamp))
modern_conus = Enum.filter(modern_contacts, &(not is_nil(&1.pos1) and Grid.contains?(&1.pos1)))
# Build all job types in batch using the existing list-accepting builders
weather_jobs = build_weather_jobs(contacts)
terrain_jobs = build_terrain_jobs(contacts)
radar_jobs = build_radar_jobs(contacts)
mechanism_jobs = build_mechanism_jobs(contacts)
iemre_jobs = build_iemre_jobs(contacts)
narr_jobs = build_narr_jobs(contacts)
bulk_jobs = weather_jobs ++ terrain_jobs ++ iemre_jobs ++ radar_jobs ++ mechanism_jobs
if bulk_jobs != [] do
insert_all_chunked(bulk_jobs)
end
if narr_jobs != [] do
Oban.insert_all(narr_jobs)
end
# HRRR — only modern CONUS contacts (pre-2014 / OCONUS contacts
# can't use HRRR; they fall through to NARR or :unavailable)
_ =
if modern_conus != [] do
{:ok, _} = HrrrPointEnqueuer.enqueue_for_contacts(modern_conus)
end
# ── mark enrichment statuses ──────────────────────────────────────
all_ids = Enum.map(contacts, & &1.id)
narr_ids = Enum.map(narr_contacts, & &1.id)
modern_ids = Enum.map(modern_contacts, & &1.id)
conus_ids = Enum.map(modern_conus, & &1.id)
# Per-contact builders enforce pos1 / pos1+pos2 guards, so
# statuses that track "was work created" align correctly.
_ = Radio.mark_weather_queued!(all_ids)
_ = Radio.mark_iemre_queued!(all_ids)
_ = Radio.mark_terrain_queued!(all_ids)
_ = Radio.set_enrichment_status!(all_ids, :mechanism_status, :queued)
# Radar is unavailable pre-2014 (archive gap)
_ = if narr_ids != [], do: Radio.set_enrichment_status!(narr_ids, :radar_status, :unavailable)
_ = if modern_ids != [], do: Radio.set_enrichment_status!(modern_ids, :radar_status, :queued)
# HRRR: queued for modern CONUS; unavailable for everything else
_ = if conus_ids != [], do: Radio.mark_hrrr_queued!(conus_ids)
non_hrrr_ids = all_ids -- conus_ids
_ =
if non_hrrr_ids != [] do
Radio.set_enrichment_status!(non_hrrr_ids, :hrrr_status, :unavailable)
end
:ok
end
@doc """
Enqueue all enrichment jobs (weather, HRRR, terrain, IEMRE) for a single contact.
Called directly from submission flow — no Oban indirection.
Optionally pass a list of types to limit which enrichments run.
"""
@spec enqueue_for_contact(Contact.t(), [atom()]) :: :ok
def enqueue_for_contact(contact, types \\ [:hrrr, :weather, :terrain, :iemre, :radar, :mechanism]) do
contact = Radio.ensure_positions!(contact)
# BackfillEnqueueWorker selects any contact with at least one
# *_status in [:pending, :queued, :failed] and calls this with all
# types. Filter out types that are already :complete for this
# contact so mark_status!/3 doesn't demote them back to :queued on
# every cron tick — that caused tens of thousands of
# mechanism/terrain rows to ping-pong between :complete and :queued.
types = Enum.reject(types, &already_complete?(contact, &1))
jobs_by_type = build_jobs_by_type(contact, types)
bulk_jobs =
jobs_by_type[:weather] ++
jobs_by_type[:terrain] ++
jobs_by_type[:iemre] ++
jobs_by_type[:radar] ++
jobs_by_type[:mechanism]
if bulk_jobs != [] do
insert_all_chunked(bulk_jobs)
end
# HRRR enrichment runs via the Rust hrrr-point-worker via the
# hrrr_fetch_tasks table (Phase 3 Stream C). Grouping by valid_time
# and UPSERT-union means a backfill tick that adds a new contact in
# an already-scheduled hour collapses into the existing row instead
# of creating a duplicate fetch.
_ =
if :hrrr in types do
{:ok, _} = HrrrPointEnqueuer.enqueue_for_contacts([contact])
end
# NARR jobs use Oban.insert_all/1 for batching. Oban OSS insert_all
# does not check uniqueness, but the worker's unique fields on
# (lat, lon, valid_time) de-duplicate at execution time.
_ = Oban.insert_all(jobs_by_type[:narr])
_ = mark_enrichment_statuses(contact, types, jobs_by_type)
:ok
end
defp build_jobs_by_type(contact, types) do
%{
weather: if(:weather in types, do: build_weather_jobs([contact]), else: []),
# hrrr jobs are routed to hrrr_fetch_tasks (not Oban) but the
# status-marking path still wants to know whether any HRRR work
# was created for this contact, so we mirror the list-shape.
hrrr: if(:hrrr in types, do: hrrr_placeholder_jobs([contact]), else: []),
terrain: if(:terrain in types, do: build_terrain_jobs([contact]), else: []),
iemre: if(:iemre in types, do: build_iemre_jobs([contact]), else: []),
narr: if(:narr in types, do: build_narr_jobs([contact]), else: []),
radar: if(:radar in types, do: build_radar_jobs([contact]), else: []),
mechanism: if(:mechanism in types, do: build_mechanism_jobs([contact]), else: [])
}
end
# Placeholder for the HRRR job-list signal. Returns a truthy singleton
# if the contact has at least one HRRR-eligible path point so
# mark_hrrr_status!/3 flips hrrr_queued on the contact row. Actual
# scheduling happens via HrrrPointEnqueuer.enqueue_for_contacts/1
# above. Returns [] for pre-coverage contacts (NARR's territory) so
# the status stays :unavailable, and [] for contacts whose HRRR
# profiles are already fully present in the DB so mark_hrrr_status!/3
# flips them directly to :complete — otherwise backfill re-flags the
# contact as :queued every cron cycle forever (HrrrPointEnqueuer
# sees 0 missing points, inserts nothing, but the status stays
# :queued and the contact is picked up again next tick).
defp hrrr_placeholder_jobs(contacts) do
# Collect all unique HRRR grid points across all contacts, query
# presence once, then map back per-contact — avoids N+1 per-contact
# queries through hrrr_data_fully_present?/1.
{point_map, present_set} = build_hrrr_present_set(contacts)
Enum.flat_map(contacts, fn contact ->
hrrr_placeholder_for_contact(contact, point_map, present_set)
end)
end
defp hrrr_placeholder_for_contact(contact, point_map, present_set) do
cond do
is_nil(contact.pos1) ->
[]
not Grid.contains?(contact.pos1) ->
[]
NarrClient.in_coverage?(contact.qso_timestamp) ->
[]
true ->
points = Map.get(point_map, contact.id, [])
if Enum.all?(points, &MapSet.member?(present_set, &1)), do: [], else: [contact.id]
end
end
defp build_hrrr_present_set(contacts) do
contacts
|> Enum.filter(&(&1.pos1 != nil and not NarrClient.in_coverage?(&1.qso_timestamp)))
|> Enum.reduce({%{}, []}, fn c, {pm, pts} ->
rounded = Weather.HrrrClient.nearest_hrrr_hour(c.qso_timestamp)
points =
c
|> Radio.contact_path_points()
|> Enum.map(fn {lat, lon} -> {Weather.round_to_hrrr_grid(lat, lon), rounded} end)
{Map.put(pm, c.id, points), points ++ pts}
end)
|> then(fn {pm, all_pts} ->
unique = Enum.uniq(all_pts)
present = Weather.hrrr_points_present_batch(unique)
{pm, present}
end)
end
defp mark_enrichment_statuses(contact, types, jobs_by_type) do
ids = [contact.id]
_ =
if contact.pos1 do
mark_pos1_statuses(contact, ids, types, jobs_by_type)
end
if contact.pos1 && contact.pos2 do
mark_path_statuses(contact, ids, types, jobs_by_type)
else
# Path enrichments (terrain/radar/mechanism) need both endpoints.
# Marking :unavailable stops BackfillEnqueueWorker from re-scanning
# these contacts every cron cycle. Radio.reset_enrichment_statuses/2
# will flip them back to :pending if pos2 is added later.
mark_missing_path_statuses(ids, types)
end
end
defp mark_missing_path_statuses(ids, types) do
_ = if :terrain in types, do: Radio.set_enrichment_status!(ids, :terrain_status, :unavailable)
_ = if :radar in types, do: Radio.set_enrichment_status!(ids, :radar_status, :unavailable)
if :mechanism in types,
do: Radio.set_enrichment_status!(ids, :mechanism_status, :unavailable)
end
defp mark_pos1_statuses(contact, ids, types, jobs_by_type) do
_ = if :weather in types, do: mark_status!(ids, :weather_status, jobs_by_type[:weather])
_ = if :hrrr in types, do: mark_hrrr_status!(contact, ids, jobs_by_type[:hrrr])
if :iemre in types, do: mark_status!(ids, :iemre_status, jobs_by_type[:iemre])
end
defp mark_path_statuses(contact, ids, types, jobs_by_type) do
_ = if :terrain in types, do: mark_status!(ids, :terrain_status, jobs_by_type[:terrain])
_ = if :radar in types, do: mark_radar_status!(contact, ids, jobs_by_type[:radar])
if :mechanism in types,
do: mark_status!(ids, :mechanism_status, jobs_by_type[:mechanism])
end
# IEM n0q composite reflectivity archive only has meaningful CONUS
# coverage post-2014 (pre that, the archive is spotty mosaics). For
# pre-2014 contacts we can't get radar, so pin to :unavailable rather
# than leaving it :pending forever.
defp mark_radar_status!(contact, ids, []) do
if NarrClient.in_coverage?(contact.qso_timestamp) do
Radio.set_enrichment_status!(ids, :radar_status, :unavailable)
else
Radio.set_enrichment_status!(ids, :radar_status, :queued)
end
end
defp mark_radar_status!(_contact, ids, [_ | _]), do: Radio.set_enrichment_status!(ids, :radar_status, :queued)
# Empty-job-list HRRR path has three terminal states:
#
# 1. pos1 outside the CONUS grid → :unavailable. hrrr_point_rs silently
# returns zero profiles for OCONUS points and the backfill cron
# would otherwise re-flag :queued every tick forever.
# 2. pre-2014 contacts (NarrClient.in_coverage?) → :unavailable so the
# :narr filter in BackfillEnqueueWorker picks them up.
# 3. modern CONUS contact whose profiles already exist → :complete.
defp mark_hrrr_status!(contact, ids, []) do
cond do
not Grid.contains?(contact.pos1) ->
Radio.set_enrichment_status!(ids, :hrrr_status, :unavailable)
NarrClient.in_coverage?(contact.qso_timestamp) ->
Radio.set_enrichment_status!(ids, :hrrr_status, :unavailable)
true ->
Radio.set_enrichment_status!(ids, :hrrr_status, :complete)
end
end
defp mark_hrrr_status!(_contact, ids, [_ | _]), do: Radio.set_enrichment_status!(ids, :hrrr_status, :queued)
@impl Oban.Pro.Worker
def process(%Oban.Job{}) do
enqueue_weather_jobs()
enqueue_hrrr_jobs()
enqueue_terrain_jobs()
enqueue_iemre_jobs()
# After enqueuing, sweep any :queued contacts whose data has
# already landed (from a prior run's ingestion) but whose
# <type>_status was never flipped. Without these, contacts stayed
# visible to the backfill scan forever.
Enum.each(
[
{"weather", &Weather.reconcile_weather_statuses/0},
{"iemre", &Weather.reconcile_iemre_statuses/0},
{"hrrr", &Weather.reconcile_hrrr_statuses/0}
],
fn {label, fun} ->
case fun.() do
{:ok, n} when n > 0 ->
Logger.info("ContactWeatherEnqueueWorker: reconciled #{n} #{label}_status → :complete")
_ ->
:ok
end
end
)
:ok
end
defp enqueue_weather_jobs do
case Radio.unprocessed_contacts() do
[] ->
:ok
contacts ->
_ = Radio.backfill_distances(contacts)
jobs = build_weather_jobs(contacts)
if jobs != [] do
insert_all_chunked(jobs)
end
ids = Enum.map(contacts, & &1.id)
_ = Radio.mark_weather_queued!(ids)
enqueue_weather_jobs()
end
end
defp enqueue_hrrr_jobs do
case Radio.unprocessed_hrrr_contacts() do
[] ->
:ok
contacts ->
# Rust Stream C path: route into hrrr_fetch_tasks instead of
# Oban. The enqueuer groups by valid_time and UPSERTs so a
# backfill cron sweep that rediscovers the same contacts
# simply refreshes the existing hourly row.
{:ok, _} = HrrrPointEnqueuer.enqueue_for_contacts(contacts)
ids = Enum.map(contacts, & &1.id)
_ = Radio.mark_hrrr_queued!(ids)
enqueue_hrrr_jobs()
end
end
defp enqueue_terrain_jobs do
case Radio.unprocessed_terrain_contacts() do
[] ->
:ok
contacts ->
jobs = build_terrain_jobs(contacts)
if jobs != [] do
insert_all_chunked(jobs)
end
ids = Enum.map(contacts, & &1.id)
_ = Radio.mark_terrain_queued!(ids)
enqueue_terrain_jobs()
end
end
@spec build_terrain_jobs([Contact.t()]) :: [Ecto.Changeset.t()]
def build_terrain_jobs(contacts) do
contacts
|> Enum.filter(fn contact -> contact.pos1 && contact.pos2 end)
|> Enum.map(fn contact ->
TerrainProfileWorker.new(%{"contact_id" => contact.id})
end)
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end
@spec build_radar_jobs([Contact.t()]) :: [Ecto.Changeset.t()]
def build_radar_jobs(contacts) do
# Group contacts by their 5-min NEXRAD frame so the worker can
# fetch+decode the ~5 MB PNG once per frame and iterate every
# contact in that frame against the in-memory pixel buffer.
# Historical backlog averages ~10 contacts per distinct frame,
# collapsing the per-job cost by roughly that factor.
contacts
|> Enum.filter(fn contact ->
contact.pos1 && contact.pos2 && not NarrClient.in_coverage?(contact.qso_timestamp)
end)
|> Enum.group_by(fn contact ->
contact.qso_timestamp
|> DateTime.from_naive!("Etc/UTC")
|> NexradClient.round_to_5min()
end)
|> Enum.map(fn {frame_ts, batch} ->
RadarFrameWorker.new(%{
"frame_ts" => DateTime.to_iso8601(frame_ts),
"contact_ids" => Enum.map(batch, & &1.id)
})
end)
end
@spec build_mechanism_jobs([Contact.t()]) :: [Ecto.Changeset.t()]
def build_mechanism_jobs(contacts) do
contacts
|> Enum.filter(fn contact -> contact.pos1 && contact.pos2 end)
|> Enum.map(fn contact ->
MechanismClassifyWorker.new(%{"contact_id" => contact.id})
end)
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end
@spec build_weather_jobs([Contact.t()]) :: [Ecto.Changeset.t()]
def build_weather_jobs(contacts) do
contacts
|> Enum.flat_map(&jobs_for_contact/1)
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end
@spec build_iemre_jobs([Contact.t()]) :: [Ecto.Changeset.t()]
def build_iemre_jobs(contacts) do
contacts
|> Enum.flat_map(&iemre_job_for_contact/1)
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end
@spec build_narr_jobs([Contact.t()]) :: [Ecto.Changeset.t()]
def build_narr_jobs(contacts) do
contacts
|> Enum.flat_map(&narr_jobs_for_contact/1)
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
end
defp narr_jobs_for_contact(%{pos1: nil}), do: []
defp narr_jobs_for_contact(contact) do
# NARR analyses are 3-hourly (00/03/06/09/12/15/18/21 UTC). NarrFetchWorker
# rejects anything else, so snap here in the caller.
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
# NCEI's NARR archive covers 1979-01-01 → 2014-10-02. Post-coverage contacts
# (hrrr_status = :unavailable but qso_timestamp >= 2014-10-02) are HRRR's
# responsibility — NARR would just 404.
if NarrClient.in_coverage?(valid_time) do
narr_jobs_for_path(contact, valid_time)
else
[]
end
end
defp narr_jobs_for_path(contact, valid_time) do
contact
|> Radio.contact_path_points()
|> Enum.flat_map(fn {lat, lon} ->
rlat = Float.round(lat * 4) / 4
rlon = Float.round(lon * 4) / 4
if Weather.find_nearest_narr(rlat, rlon, valid_time) do
[]
else
[
NarrFetchWorker.new(%{
"lat" => rlat,
"lon" => rlon,
"valid_time" => DateTime.to_iso8601(valid_time)
})
]
end
end)
end
defp jobs_for_contact(contact) do
contact
|> Radio.contact_path_points()
|> Enum.flat_map(fn {lat, lon} ->
build_asos_jobs(lat, lon, contact.qso_timestamp) ++
build_raob_jobs(lat, lon, contact.qso_timestamp)
end)
end
defp enqueue_iemre_jobs do
case Radio.unprocessed_iemre_contacts() do
[] ->
:ok
contacts ->
jobs = build_iemre_jobs(contacts)
if jobs != [] do
insert_all_chunked(jobs)
end
ids = Enum.map(contacts, & &1.id)
_ = Radio.mark_iemre_queued!(ids)
enqueue_iemre_jobs()
end
end
defp build_asos_jobs(lat, lon, timestamp) do
start_dt = DateTime.add(timestamp, -2 * 3600, :second)
end_dt = DateTime.add(timestamp, 2 * 3600, :second)
stations = Weather.nearby_stations(lat, lon, "asos", @asos_radius_km)
# A ±2h window around the QSO can straddle UTC midnight, so enumerate
# every date the window touches. Each (station, date) becomes one
# asos_day job — Oban's unique-args dedup collapses cross-QSO
# repeats (contact A's KDFW/2021-08-21 and contact B's KDFW/2021-08-21
# share args → one job). A full-day fetch returns ~24 hourly rows,
# amortizing the IemRateLimiter gap over 24× the previous payload.
dates = window_dates(start_dt, end_dt)
pairs =
for station <- stations, date <- dates do
{station, date}
end
covered =
pairs
|> Enum.map(fn {s, d} -> {s.id, d} end)
|> Weather.station_day_pairs_covered()
pairs
|> Enum.reject(fn {s, d} -> MapSet.member?(covered, {s.id, d}) end)
|> Enum.map(fn {station, date} ->
WeatherFetchWorker.new(%{
"fetch_type" => "asos_day",
"station_id" => station.id,
"station_code" => station.station_code,
"date" => Date.to_iso8601(date)
})
end)
end
defp window_dates(start_dt, end_dt) do
start_date = DateTime.to_date(start_dt)
end_date = DateTime.to_date(end_dt)
if Date.compare(start_date, end_date) == :eq do
[start_date]
else
[start_date, end_date]
end
end
defp iemre_job_for_contact(%{pos1: nil}), do: []
defp iemre_job_for_contact(contact) do
date = DateTime.to_date(contact.qso_timestamp)
contact
|> Radio.contact_path_points()
|> Enum.flat_map(fn {lat, lon} ->
{rlat, rlon} = Weather.round_to_iemre_grid(lat, lon)
if Weather.has_iemre_observation?(rlat, rlon, date) do
[]
else
[
IemreFetchWorker.new(%{
"lat" => rlat,
"lon" => rlon,
"date" => Date.to_iso8601(date)
})
]
end
end)
end
defp build_raob_jobs(lat, lon, timestamp) do
build_raob_jobs(lat, lon, timestamp, @sounding_radius_km, nil)
end
defp build_raob_jobs(lat, lon, timestamp, radius_km, contact_id) do
sounding_times = Weather.sounding_times_around(timestamp)
stations = Weather.nearby_stations(lat, lon, "sounding", radius_km)
station_ids = Enum.map(stations, & &1.id)
covered = Weather.station_ids_with_soundings(station_ids, sounding_times)
for station <- stations,
sounding_time <- sounding_times,
not MapSet.member?(covered, {station.id, sounding_time}) do
args =
Map.merge(
%{
"fetch_type" => "raob",
"station_id" => station.id,
"station_code" => station.station_code,
"sounding_time" => DateTime.to_iso8601(sounding_time)
},
if(contact_id, do: %{"contact_id" => contact_id}, else: %{})
)
WeatherFetchWorker.new(args)
end
end
@doc """
Enqueue RAOB fetch jobs for the given contact at a progressively wider
radius until the radius yields at least one sounding station. Passes
the contact id through so `WeatherFetchWorker` can PubSub the contact
detail page when each sounding lands.
Returns `{:ok, %{radius_km, jobs_enqueued}}`, or `{:error, :no_stations}`
when even 1000 km finds nothing.
"""
@spec enqueue_raob_fetch_with_widening(Contact.t()) ::
{:ok, %{radius_km: pos_integer(), jobs_enqueued: non_neg_integer()}}
| {:error, :no_stations}
| {:error, :no_coords}
def enqueue_raob_fetch_with_widening(contact) do
case Radio.contact_path_points(contact) do
[] ->
{:error, :no_coords}
[{lat, lon} | _] ->
do_widening_raob(lat, lon, contact)
end
end
defp do_widening_raob(lat, lon, contact) do
Enum.reduce_while([150, 300, 600, 1000], {:error, :no_stations}, fn radius, _acc ->
stations = Weather.nearby_stations(lat, lon, "sounding", radius)
if stations == [] do
{:cont, {:error, :no_stations}}
else
{:halt, enqueue_raob_at_radius(lat, lon, contact, radius)}
end
end)
end
defp enqueue_raob_at_radius(lat, lon, contact, radius) do
jobs = build_raob_jobs(lat, lon, contact.qso_timestamp, radius, contact.id)
if jobs != [], do: insert_all_chunked(jobs)
{:ok, %{radius_km: radius, jobs_enqueued: length(jobs)}}
end
defp insert_all_chunked(jobs) do
jobs
|> Enum.chunk_every(@insert_batch_size)
|> Enum.each(&Oban.insert_all/1)
end
defp mark_status!(ids, field, [_ | _]), do: Radio.set_enrichment_status!(ids, field, :queued)
defp mark_status!(ids, field, []), do: Radio.set_enrichment_status!(ids, field, :complete)
# :narr is a virtual type synthesized from hrrr_status = :unavailable;
# it has no column of its own, so there's nothing to "already be
# complete."
defp already_complete?(_contact, :narr), do: false
defp already_complete?(contact, type) do
Map.get(contact, String.to_existing_atom("#{type}_status")) == :complete
end
end