Graham McIntire
8a969e315c
57,186 prod contacts stored pos1/pos2 with 'lng'; 1,133 used 'lon'. Every Elixir caller carried a `pos["lon"] || pos["lng"]` fallback — which just caused a SQL widget to silently miscount 98% of contacts (count_narr_done used `pos1->>'lon'` directly, no fallback, so every lng-keyed row returned NULL and failed the coverage check). - Migration rewrites every pos1/pos2 JSONB in place, renaming 'lng' to 'lon' and dropping 'lng'. - Removes all 20+ `|| pos["lng"]` fallbacks across lib/, workers, scorer, weather, radio.ex, contact show view, and recalibrator. - lib_ml/propagation_analyze.ex SQL now reads pos1->>'lon' directly (was reading 'lng' only, which would have broken after migration). - priv/repo/import_contacts.exs one-time seed script now emits 'lon' with string keys, matching production shape. - Test fixtures in 4 test files normalized to 'lon'. - Two lng-characterization tests deleted — nonsensical post-normalize. - Updated notebook + old import_weather script to match. - JS hook contact_map_hook.ts TypeScript type narrowed to 'lon'.
97 lines
2.8 KiB
Elixir
97 lines
2.8 KiB
Elixir
defmodule Microwaveprop.Workers.NexradWorker do
|
|
@moduledoc """
|
|
Fetches one n0q NEXRAD composite frame and stores per-point
|
|
observations in `nexrad_observations`.
|
|
|
|
Jobs are unique on `{year, month, day, hour, minute}` so backfill
|
|
sweeps that enqueue duplicate timestamps collapse automatically.
|
|
"""
|
|
|
|
use Oban.Worker,
|
|
queue: :nexrad,
|
|
max_attempts: 3,
|
|
unique: [
|
|
period: :infinity,
|
|
states: [:available, :scheduled, :executing, :retryable],
|
|
keys: [:year, :month, :day, :hour, :minute]
|
|
]
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Radio.Contact
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Weather.NexradClient
|
|
alias Microwaveprop.Weather.NexradObservation
|
|
|
|
require Logger
|
|
|
|
@impl Oban.Worker
|
|
def perform(%Oban.Job{args: args}) do
|
|
%{"year" => year, "month" => month, "day" => day, "hour" => hour} = args
|
|
minute = Map.get(args, "minute", 0)
|
|
{:ok, date} = Date.new(year, month, day)
|
|
{:ok, time} = Time.new(hour, minute, 0)
|
|
{:ok, timestamp} = DateTime.new(date, time, "Etc/UTC")
|
|
|
|
points = points_of_interest_for_time(timestamp)
|
|
|
|
if points == [] do
|
|
Logger.info("NexradWorker: no points for #{timestamp}, skipping")
|
|
:ok
|
|
else
|
|
fetch_and_upsert(timestamp, points)
|
|
end
|
|
end
|
|
|
|
@doc false
|
|
def points_of_interest_for_time(timestamp) do
|
|
# Match contacts within +/- 30 minutes of this frame
|
|
time_start = DateTime.add(timestamp, -1800, :second)
|
|
time_end = DateTime.add(timestamp, 1800, :second)
|
|
|
|
Contact
|
|
|> where([c], not is_nil(c.pos1))
|
|
|> where([c], c.qso_timestamp >= ^time_start and c.qso_timestamp <= ^time_end)
|
|
|> select([c], c.pos1)
|
|
|> Repo.all()
|
|
|> Enum.flat_map(fn pos ->
|
|
case {pos["lat"], pos["lon"]} do
|
|
{lat, lon} when is_number(lat) and is_number(lon) -> [{snap(lat), snap(lon)}]
|
|
_ -> []
|
|
end
|
|
end)
|
|
|> Enum.uniq()
|
|
end
|
|
|
|
defp snap(x), do: Float.round(x * 1.0, 3)
|
|
|
|
defp fetch_and_upsert(timestamp, points) do
|
|
case NexradClient.fetch_frame(timestamp, points) do
|
|
{:ok, observations} ->
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
rows =
|
|
Enum.map(observations, fn obs ->
|
|
obs
|
|
|> Map.put(:id, Ecto.UUID.generate())
|
|
|> Map.put(:inserted_at, now)
|
|
|> Map.put(:updated_at, now)
|
|
end)
|
|
|
|
{inserted, _} =
|
|
Repo.insert_all(
|
|
NexradObservation,
|
|
rows,
|
|
on_conflict: {:replace_all_except, [:id, :inserted_at]},
|
|
conflict_target: [:lat, :lon, :observed_at]
|
|
)
|
|
|
|
Logger.info("NexradWorker: upserted #{inserted} observations for #{timestamp}")
|
|
:ok
|
|
|
|
{:error, reason} ->
|
|
Logger.warning("NexradWorker failed for #{timestamp}: #{inspect(reason)}")
|
|
{:error, reason}
|
|
end
|
|
end
|
|
end
|