Graham McIntire
084e6b6224
Convert every context/worker upsert that used the catch-all
`:replace_all_except` form to the explicit `{:replace, [:col1, ...]}`
form. Reviewers can now see exactly which columns an upsert overwrites,
and adding a new column to a schema no longer silently opts it into the
update path.
Behaviour is preserved bit-for-bit: each new explicit list contains
every column the old `:replace_all_except` would have overwritten.
Touched:
- Microwaveprop.SpaceWeather (Kp / F10.7 / X-ray shared helper)
- Microwaveprop.Ionosphere (observation upsert)
- NexradWorker, HrrrNativeGridWorker (insert_all grids)
- CommonVolumeRadarWorker, RadarFrameWorker (per-contact stats)
Also pins the conflict behaviour for the ContactCommonVolumeRadar
upsert path in RadarFrameWorkerTest so a future column addition that
isn't reflected in the explicit list fails loudly.
99 lines
2.9 KiB
Elixir
99 lines
2.9 KiB
Elixir
defmodule Microwaveprop.Workers.NexradWorker do
|
|
@moduledoc """
|
|
Fetches one n0q NEXRAD composite frame and stores per-point
|
|
observations in `nexrad_observations`.
|
|
|
|
Jobs are unique on `{year, month, day, hour, minute}` so backfill
|
|
sweeps that enqueue duplicate timestamps collapse automatically.
|
|
"""
|
|
|
|
use Oban.Worker,
|
|
queue: :nexrad,
|
|
max_attempts: 3,
|
|
unique: [
|
|
period: :infinity,
|
|
states: [:available, :scheduled, :executing, :retryable],
|
|
keys: [:year, :month, :day, :hour, :minute]
|
|
]
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Radio.Contact
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Weather.NexradClient
|
|
alias Microwaveprop.Weather.NexradObservation
|
|
|
|
require Logger
|
|
|
|
@impl Oban.Worker
|
|
def perform(%Oban.Job{args: args}) do
|
|
%{"year" => year, "month" => month, "day" => day, "hour" => hour} = args
|
|
minute = Map.get(args, "minute", 0)
|
|
{:ok, date} = Date.new(year, month, day)
|
|
{:ok, time} = Time.new(hour, minute, 0)
|
|
{:ok, timestamp} = DateTime.new(date, time, "Etc/UTC")
|
|
|
|
points = points_of_interest_for_time(timestamp)
|
|
|
|
if points == [] do
|
|
Logger.info("NexradWorker: no points for #{timestamp}, skipping")
|
|
:ok
|
|
else
|
|
fetch_and_upsert(timestamp, points)
|
|
end
|
|
end
|
|
|
|
@doc false
|
|
@spec points_of_interest_for_time(DateTime.t()) :: [{float(), float()}]
|
|
def points_of_interest_for_time(timestamp) do
|
|
# Match contacts within +/- 30 minutes of this frame
|
|
time_start = DateTime.add(timestamp, -1800, :second)
|
|
time_end = DateTime.add(timestamp, 1800, :second)
|
|
|
|
Contact
|
|
|> where([c], not is_nil(c.pos1))
|
|
|> where([c], c.qso_timestamp >= ^time_start and c.qso_timestamp <= ^time_end)
|
|
|> select([c], c.pos1)
|
|
|> Repo.all()
|
|
|> Enum.flat_map(fn pos ->
|
|
case {pos["lat"], pos["lon"]} do
|
|
{lat, lon} when is_number(lat) and is_number(lon) -> [{snap(lat), snap(lon)}]
|
|
_ -> []
|
|
end
|
|
end)
|
|
|> Enum.uniq()
|
|
end
|
|
|
|
defp snap(x), do: Float.round(x * 1.0, 3)
|
|
|
|
defp fetch_and_upsert(timestamp, points) do
|
|
case NexradClient.fetch_frame(timestamp, points) do
|
|
{:ok, observations} ->
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
rows =
|
|
Enum.map(observations, fn obs ->
|
|
obs
|
|
|> Map.put(:id, Ecto.UUID.generate())
|
|
|> Map.put(:inserted_at, now)
|
|
|> Map.put(:updated_at, now)
|
|
end)
|
|
|
|
{inserted, _} =
|
|
Repo.insert_all(
|
|
NexradObservation,
|
|
rows,
|
|
on_conflict:
|
|
{:replace, ~w(mean_reflectivity_dbz max_reflectivity_dbz texture_variance pixel_count updated_at)a},
|
|
conflict_target: [:lat, :lon, :observed_at]
|
|
)
|
|
|
|
Logger.info("NexradWorker: upserted #{inserted} observations for #{timestamp}")
|
|
:ok
|
|
|
|
{:error, reason} ->
|
|
Logger.warning("NexradWorker failed for #{timestamp}: #{inspect(reason)}")
|
|
{:error, reason}
|
|
end
|
|
end
|
|
end
|