defmodule Microwaveprop.Workers.WeatherFetchWorker do @moduledoc false use Oban.Worker, queue: :weather, max_attempts: 20, unique: [period: :infinity, states: [:available, :scheduled, :executing, :retryable]] alias Microwaveprop.Repo alias Microwaveprop.Weather alias Microwaveprop.Weather.IemClient alias Microwaveprop.Weather.SoundingParams alias Microwaveprop.Weather.Station alias Microwaveprop.Weather.SurfaceObservation require Logger @impl Oban.Worker def backoff(%Oban.Job{attempt: attempt}) do min(120 * Integer.pow(2, attempt - 1), _six_hours = 21_600) end @impl Oban.Worker def perform(%Oban.Job{args: %{"fetch_type" => "asos"} = args}) do %{ "station_id" => station_id, "station_code" => station_code, "start_dt" => start_dt_str, "end_dt" => end_dt_str } = args {:ok, start_dt, _} = DateTime.from_iso8601(start_dt_str) {:ok, end_dt, _} = DateTime.from_iso8601(end_dt_str) case Repo.get(Station, station_id) do nil -> Logger.warning("WeatherFetch ASOS: station #{station_id} not found, skipping") :ok station -> fetch_asos(station, station_id, station_code, start_dt, end_dt, start_dt_str, end_dt_str) end end def perform(%Oban.Job{args: %{"fetch_type" => "raob"} = args}) do %{ "station_id" => station_id, "station_code" => station_code, "sounding_time" => sounding_time_str } = args contact_id = Map.get(args, "contact_id") {:ok, sounding_time, _} = DateTime.from_iso8601(sounding_time_str) case Repo.get(Station, station_id) do nil -> Logger.warning("WeatherFetch RAOB: station #{station_id} not found, skipping") :ok station -> fetch_raob(station, station_id, station_code, sounding_time, sounding_time_str, contact_id) end end # -- ASOS helpers -- defp fetch_asos(station, station_id, station_code, start_dt, end_dt, start_dt_str, end_dt_str) do if Weather.has_surface_observations?(station.id, start_dt, end_dt) do Logger.debug("WeatherFetch ASOS: #{station_code} already has obs for #{start_dt_str}..#{end_dt_str}") :ok else Logger.info("WeatherFetch ASOS: fetching #{station_code} for #{start_dt_str}..#{end_dt_str}") do_fetch_asos(station, station_id, station_code, start_dt, end_dt) end end defp do_fetch_asos(station, station_id, station_code, start_dt, end_dt) do case IemClient.fetch_asos(station_code, start_dt, end_dt) do {:ok, rows} -> ingest_asos_rows(station, station_id, station_code, rows, start_dt, end_dt) {:error, reason} -> handle_iem_error("ASOS", station_code, reason) end end # IEM upstream is routinely flaky — HTTP 429 rate-limits during a # backfill sweep, and transient connection resets (Req.TransportError # :closed / :timeout) on long-running fetches. Neither is a job # failure worth Oban.PerformError's stack-trace; snooze instead so # the retry is silent. Permanent errors (4xx ≠ 429, 5xx) fall through # to the warning + {:error, _} path so they count toward max_attempts. defp handle_iem_error(kind, station_code, "IEM " <> _ = reason) do if String.contains?(reason, "429") do Logger.debug("WeatherFetch #{kind}: #{station_code} rate-limited, snoozing") {:snooze, 300} else Logger.warning("WeatherFetch #{kind}: #{station_code} failed: #{reason}") {:error, reason} end end defp handle_iem_error(kind, station_code, %Req.TransportError{reason: tr_reason}) when tr_reason in [:closed, :timeout, :econnrefused, :nxdomain] do Logger.debug("WeatherFetch #{kind}: #{station_code} transport #{tr_reason}, snoozing") {:snooze, 60} end defp handle_iem_error(kind, station_code, reason) do Logger.warning("WeatherFetch #{kind}: #{station_code} failed: #{inspect(reason, printable_limit: 200)}") {:error, reason} end defp ingest_asos_rows(station, station_id, station_code, rows, start_dt, end_dt) do count = Enum.count(rows, & &1.observed_at) if count > 0 do Weather.upsert_surface_observations(station, rows) else store_asos_stub(station, start_dt, end_dt) end Logger.info("WeatherFetch ASOS: #{station_code} ingested #{count} observations") _ = Phoenix.PubSub.broadcast( Microwaveprop.PubSub, "contact_enrichment:weather", {:weather_ready, %{station_id: station_id}} ) :ok end defp store_asos_stub(station, start_dt, end_dt) do midpoint = DateTime.add(start_dt, div(DateTime.diff(end_dt, start_dt), 2)) %SurfaceObservation{} |> SurfaceObservation.changeset(%{station_id: station.id, observed_at: midpoint}) |> Repo.insert(on_conflict: :nothing, conflict_target: [:station_id, :observed_at]) end # -- RAOB helpers -- defp fetch_raob(station, station_id, station_code, sounding_time, sounding_time_str, contact_id) do if Weather.has_sounding?(station.id, sounding_time) do Logger.debug("WeatherFetch RAOB: #{station_code} already has sounding for #{sounding_time_str}") _ = broadcast_sounding_fetched(station_id, contact_id) :ok else Logger.info("WeatherFetch RAOB: fetching #{station_code} for #{sounding_time_str}") do_fetch_raob(station, station_id, station_code, sounding_time, sounding_time_str, contact_id) end end defp do_fetch_raob(station, station_id, station_code, sounding_time, sounding_time_str, contact_id) do case IemClient.fetch_raob(station_code, sounding_time) do {:ok, [parsed | _]} -> ingest_raob(station, station_id, station_code, parsed, contact_id) {:ok, []} -> _ = Weather.upsert_sounding(station, %{observed_at: sounding_time, profile: [], level_count: 0}) Logger.info("WeatherFetch RAOB: #{station_code} no data for #{sounding_time_str}, stored stub") _ = broadcast_sounding_fetched(station_id, contact_id) :ok {:error, reason} -> handle_iem_error("RAOB", station_code, reason) end end defp ingest_raob(station, station_id, station_code, parsed, contact_id) do sounding_attrs = build_sounding_attrs(parsed) _ = Weather.upsert_sounding(station, sounding_attrs) Logger.info("WeatherFetch RAOB: #{station_code} ingested sounding with #{length(parsed.profile)} levels") _ = broadcast_sounding_fetched(station_id, contact_id) :ok end defp broadcast_sounding_fetched(station_id, contact_id) do _ = Phoenix.PubSub.broadcast( Microwaveprop.PubSub, "contact_enrichment:weather", {:weather_ready, %{station_id: station_id}} ) if contact_id do Phoenix.PubSub.broadcast( Microwaveprop.PubSub, "contact_enrichment:#{contact_id}", {:sounding_fetched, %{station_id: station_id, contact_id: contact_id}} ) end end defp build_sounding_attrs(parsed) do case SoundingParams.derive(parsed.profile) do nil -> %{observed_at: parsed.observed_at, profile: parsed.profile, level_count: length(parsed.profile)} params -> %{ observed_at: parsed.observed_at, profile: parsed.profile, level_count: params.level_count, surface_pressure_mb: params.surface_pressure_mb, surface_temp_c: params.surface_temp_c, surface_dewpoint_c: params.surface_dewpoint_c, surface_refractivity: params.surface_refractivity, min_refractivity_gradient: params.min_refractivity_gradient, boundary_layer_depth_m: params.boundary_layer_depth_m, precipitable_water_mm: params.precipitable_water_mm, k_index: params.k_index, lifted_index: params.lifted_index, ducting_detected: params.ducting_detected, duct_characteristics: params.duct_characteristics } end end end