Multi-point path enrichment for HRRR, IEMRE, and weather data
Enqueue worker now gathers atmospheric data at pos1, midpoint, and pos2 along each QSO path instead of only pos1. Existing has_* guards prevent duplicate fetches at each grid point. - Add Radio.qso_path_points/1 for path point extraction - Update hrrr_job_for_qso, iemre_job_for_qso, jobs_for_qso to iterate path points - Refactor Weather into find_nearest_hrrr/3 and find_nearest_iemre/3 - Add hrrr_profiles_for_path/1 and iemre_for_path/1 query functions - Add mix reset_enrichment task to trigger re-processing
This commit is contained in:
parent
ea439e56bd
commit
ded9c054ac
8 changed files with 432 additions and 72 deletions
|
|
@ -100,6 +100,33 @@ defmodule Microwaveprop.Radio do
|
||||||
|> Repo.update_all(set: [iemre_queued: true])
|
|> Repo.update_all(set: [iemre_queued: true])
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Returns a list of {lat, lon} points along the QSO path: pos1, midpoint, pos2.
|
||||||
|
Returns [pos1] if pos2 is nil, or [] if pos1 is nil.
|
||||||
|
"""
|
||||||
|
def qso_path_points(%{pos1: nil}), do: []
|
||||||
|
|
||||||
|
def qso_path_points(%{pos1: pos1, pos2: nil}) do
|
||||||
|
lat = pos1["lat"]
|
||||||
|
lon = pos1["lon"] || pos1["lng"]
|
||||||
|
if lat && lon, do: [{lat, lon}], else: []
|
||||||
|
end
|
||||||
|
|
||||||
|
def qso_path_points(%{pos1: pos1, pos2: pos2}) do
|
||||||
|
lat1 = pos1["lat"]
|
||||||
|
lon1 = pos1["lon"] || pos1["lng"]
|
||||||
|
lat2 = pos2["lat"]
|
||||||
|
lon2 = pos2["lon"] || pos2["lng"]
|
||||||
|
|
||||||
|
if lat1 && lon1 && lat2 && lon2 do
|
||||||
|
mid_lat = (lat1 + lat2) / 2
|
||||||
|
mid_lon = (lon1 + lon2) / 2
|
||||||
|
[{lat1, lon1}, {mid_lat, mid_lon}, {lat2, lon2}]
|
||||||
|
else
|
||||||
|
[{lat1, lon1}]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
@earth_radius_km 6371.0
|
@earth_radius_km 6371.0
|
||||||
|
|
||||||
def haversine_km(lat1, lon1, lat2, lon2) do
|
def haversine_km(lat1, lon1, lat2, lon2) do
|
||||||
|
|
|
||||||
|
|
@ -192,36 +192,48 @@ defmodule Microwaveprop.Weather do
|
||||||
lon = qso.pos1["lon"] || qso.pos1["lng"]
|
lon = qso.pos1["lon"] || qso.pos1["lng"]
|
||||||
|
|
||||||
if lat && lon do
|
if lat && lon do
|
||||||
# Search within ~0.05 degrees (~5km) and 1 hour
|
find_nearest_hrrr(lat, lon, qso.qso_timestamp)
|
||||||
dlat = 0.05
|
|
||||||
dlon = 0.05
|
|
||||||
time_start = DateTime.add(qso.qso_timestamp, -3600, :second)
|
|
||||||
time_end = DateTime.add(qso.qso_timestamp, 3600, :second)
|
|
||||||
|
|
||||||
HrrrProfile
|
|
||||||
|> where(
|
|
||||||
[h],
|
|
||||||
h.lat >= ^(lat - dlat) and h.lat <= ^(lat + dlat) and
|
|
||||||
h.lon >= ^(lon - dlon) and h.lon <= ^(lon + dlon) and
|
|
||||||
h.valid_time >= ^time_start and h.valid_time <= ^time_end
|
|
||||||
)
|
|
||||||
|> order_by([h],
|
|
||||||
asc:
|
|
||||||
fragment(
|
|
||||||
"ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))",
|
|
||||||
h.lat,
|
|
||||||
^lat,
|
|
||||||
h.lon,
|
|
||||||
^lon,
|
|
||||||
h.valid_time,
|
|
||||||
^qso.qso_timestamp
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|> limit(1)
|
|
||||||
|> Repo.one()
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def find_nearest_hrrr(lat, lon, timestamp) do
|
||||||
|
dlat = 0.05
|
||||||
|
dlon = 0.05
|
||||||
|
time_start = DateTime.add(timestamp, -3600, :second)
|
||||||
|
time_end = DateTime.add(timestamp, 3600, :second)
|
||||||
|
|
||||||
|
HrrrProfile
|
||||||
|
|> where(
|
||||||
|
[h],
|
||||||
|
h.lat >= ^(lat - dlat) and h.lat <= ^(lat + dlat) and
|
||||||
|
h.lon >= ^(lon - dlon) and h.lon <= ^(lon + dlon) and
|
||||||
|
h.valid_time >= ^time_start and h.valid_time <= ^time_end
|
||||||
|
)
|
||||||
|
|> order_by([h],
|
||||||
|
asc:
|
||||||
|
fragment(
|
||||||
|
"ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))",
|
||||||
|
h.lat,
|
||||||
|
^lat,
|
||||||
|
h.lon,
|
||||||
|
^lon,
|
||||||
|
h.valid_time,
|
||||||
|
^timestamp
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|> limit(1)
|
||||||
|
|> Repo.one()
|
||||||
|
end
|
||||||
|
|
||||||
|
def hrrr_profiles_for_path(%{pos1: nil}), do: []
|
||||||
|
|
||||||
|
def hrrr_profiles_for_path(qso) do
|
||||||
|
qso
|
||||||
|
|> Microwaveprop.Radio.qso_path_points()
|
||||||
|
|> Enum.map(fn {lat, lon} -> find_nearest_hrrr(lat, lon, qso.qso_timestamp) end)
|
||||||
|
|> Enum.reject(&is_nil/1)
|
||||||
|
end
|
||||||
|
|
||||||
def round_to_hrrr_grid(lat, lon) do
|
def round_to_hrrr_grid(lat, lon) do
|
||||||
{Float.round(lat / 1.0, 2), Float.round(lon / 1.0, 2)}
|
{Float.round(lat / 1.0, 2), Float.round(lon / 1.0, 2)}
|
||||||
end
|
end
|
||||||
|
|
@ -253,12 +265,25 @@ defmodule Microwaveprop.Weather do
|
||||||
lon = qso.pos1["lon"] || qso.pos1["lng"]
|
lon = qso.pos1["lon"] || qso.pos1["lng"]
|
||||||
|
|
||||||
if lat && lon do
|
if lat && lon do
|
||||||
{rlat, rlon} = round_to_iemre_grid(lat, lon)
|
find_nearest_iemre(lat, lon, qso.qso_timestamp)
|
||||||
date = DateTime.to_date(qso.qso_timestamp)
|
|
||||||
|
|
||||||
IemreObservation
|
|
||||||
|> where([i], i.lat == ^rlat and i.lon == ^rlon and i.date == ^date)
|
|
||||||
|> Repo.one()
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def find_nearest_iemre(lat, lon, timestamp) do
|
||||||
|
{rlat, rlon} = round_to_iemre_grid(lat, lon)
|
||||||
|
date = DateTime.to_date(timestamp)
|
||||||
|
|
||||||
|
IemreObservation
|
||||||
|
|> where([i], i.lat == ^rlat and i.lon == ^rlon and i.date == ^date)
|
||||||
|
|> Repo.one()
|
||||||
|
end
|
||||||
|
|
||||||
|
def iemre_for_path(%{pos1: nil}), do: []
|
||||||
|
|
||||||
|
def iemre_for_path(qso) do
|
||||||
|
qso
|
||||||
|
|> Microwaveprop.Radio.qso_path_points()
|
||||||
|
|> Enum.map(fn {lat, lon} -> find_nearest_iemre(lat, lon, qso.qso_timestamp) end)
|
||||||
|
|> Enum.reject(&is_nil/1)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
|
||||||
|
|
@ -112,13 +112,12 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorker do
|
||||||
end
|
end
|
||||||
|
|
||||||
defp jobs_for_qso(qso) do
|
defp jobs_for_qso(qso) do
|
||||||
lat = qso.pos1["lat"]
|
qso
|
||||||
lon = qso.pos1["lon"] || qso.pos1["lng"]
|
|> Radio.qso_path_points()
|
||||||
|
|> Enum.flat_map(fn {lat, lon} ->
|
||||||
asos_jobs = build_asos_jobs(lat, lon, qso.qso_timestamp)
|
build_asos_jobs(lat, lon, qso.qso_timestamp) ++
|
||||||
raob_jobs = build_raob_jobs(lat, lon, qso.qso_timestamp)
|
build_raob_jobs(lat, lon, qso.qso_timestamp)
|
||||||
|
end)
|
||||||
asos_jobs ++ raob_jobs
|
|
||||||
end
|
end
|
||||||
|
|
||||||
defp enqueue_iemre_jobs do
|
defp enqueue_iemre_jobs do
|
||||||
|
|
@ -143,12 +142,12 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorker do
|
||||||
defp hrrr_job_for_qso(%{pos1: nil}), do: []
|
defp hrrr_job_for_qso(%{pos1: nil}), do: []
|
||||||
|
|
||||||
defp hrrr_job_for_qso(qso) do
|
defp hrrr_job_for_qso(qso) do
|
||||||
lat = qso.pos1["lat"]
|
rounded_time = HrrrClient.nearest_hrrr_hour(qso.qso_timestamp)
|
||||||
lon = qso.pos1["lon"] || qso.pos1["lng"]
|
|
||||||
|
|
||||||
if lat && lon do
|
qso
|
||||||
|
|> Radio.qso_path_points()
|
||||||
|
|> Enum.flat_map(fn {lat, lon} ->
|
||||||
{rlat, rlon} = Weather.round_to_hrrr_grid(lat, lon)
|
{rlat, rlon} = Weather.round_to_hrrr_grid(lat, lon)
|
||||||
rounded_time = HrrrClient.nearest_hrrr_hour(qso.qso_timestamp)
|
|
||||||
|
|
||||||
if Weather.has_hrrr_profile?(rlat, rlon, rounded_time) do
|
if Weather.has_hrrr_profile?(rlat, rlon, rounded_time) do
|
||||||
[]
|
[]
|
||||||
|
|
@ -161,9 +160,7 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorker do
|
||||||
})
|
})
|
||||||
]
|
]
|
||||||
end
|
end
|
||||||
else
|
end)
|
||||||
[]
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
||||||
defp build_asos_jobs(lat, lon, timestamp) do
|
defp build_asos_jobs(lat, lon, timestamp) do
|
||||||
|
|
@ -189,12 +186,12 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorker do
|
||||||
defp iemre_job_for_qso(%{pos1: nil}), do: []
|
defp iemre_job_for_qso(%{pos1: nil}), do: []
|
||||||
|
|
||||||
defp iemre_job_for_qso(qso) do
|
defp iemre_job_for_qso(qso) do
|
||||||
lat = qso.pos1["lat"]
|
date = DateTime.to_date(qso.qso_timestamp)
|
||||||
lon = qso.pos1["lon"] || qso.pos1["lng"]
|
|
||||||
|
|
||||||
if lat && lon do
|
qso
|
||||||
|
|> Radio.qso_path_points()
|
||||||
|
|> Enum.flat_map(fn {lat, lon} ->
|
||||||
{rlat, rlon} = Weather.round_to_iemre_grid(lat, lon)
|
{rlat, rlon} = Weather.round_to_iemre_grid(lat, lon)
|
||||||
date = DateTime.to_date(qso.qso_timestamp)
|
|
||||||
|
|
||||||
if Weather.has_iemre_observation?(rlat, rlon, date) do
|
if Weather.has_iemre_observation?(rlat, rlon, date) do
|
||||||
[]
|
[]
|
||||||
|
|
@ -207,9 +204,7 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorker do
|
||||||
})
|
})
|
||||||
]
|
]
|
||||||
end
|
end
|
||||||
else
|
end)
|
||||||
[]
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
||||||
defp build_raob_jobs(lat, lon, timestamp) do
|
defp build_raob_jobs(lat, lon, timestamp) do
|
||||||
|
|
|
||||||
33
lib/mix/tasks/reset_enrichment.ex
Normal file
33
lib/mix/tasks/reset_enrichment.ex
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
defmodule Mix.Tasks.ResetEnrichment do
|
||||||
|
@shortdoc "Reset weather/HRRR/IEMRE enrichment flags for re-processing"
|
||||||
|
|
||||||
|
@moduledoc """
|
||||||
|
Resets weather, HRRR, and IEMRE enrichment flags on all QSOs so the
|
||||||
|
next enqueue worker run re-processes them with multi-point path data.
|
||||||
|
|
||||||
|
Does NOT reset terrain_queued since terrain profiles are path-based (pos1→pos2)
|
||||||
|
and don't change with multi-point enrichment.
|
||||||
|
|
||||||
|
mix reset_enrichment
|
||||||
|
"""
|
||||||
|
|
||||||
|
use Mix.Task
|
||||||
|
|
||||||
|
import Ecto.Query
|
||||||
|
|
||||||
|
alias Microwaveprop.Radio.Qso
|
||||||
|
alias Microwaveprop.Repo
|
||||||
|
|
||||||
|
@impl Mix.Task
|
||||||
|
def run(_args) do
|
||||||
|
Mix.Task.run("app.start")
|
||||||
|
|
||||||
|
{count, _} =
|
||||||
|
Qso
|
||||||
|
|> where([q], q.weather_queued == true or q.hrrr_queued == true or q.iemre_queued == true)
|
||||||
|
|> Repo.update_all(set: [weather_queued: false, hrrr_queued: false, iemre_queued: false])
|
||||||
|
|
||||||
|
Mix.shell().info("Reset enrichment flags on #{count} QSOs")
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
@ -346,6 +346,52 @@ defmodule Microwaveprop.RadioTest do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe "qso_path_points/1" do
|
||||||
|
test "returns pos1, midpoint, and pos2 when both positions exist" do
|
||||||
|
qso =
|
||||||
|
create_qso(%{
|
||||||
|
pos1: %{"lat" => 32.9, "lon" => -97.0},
|
||||||
|
pos2: %{"lat" => 30.3, "lon" => -97.7}
|
||||||
|
})
|
||||||
|
|
||||||
|
points = Radio.qso_path_points(qso)
|
||||||
|
|
||||||
|
assert length(points) == 3
|
||||||
|
[{lat1, lon1}, {mid_lat, mid_lon}, {lat2, lon2}] = points
|
||||||
|
assert lat1 == 32.9
|
||||||
|
assert lon1 == -97.0
|
||||||
|
assert_in_delta mid_lat, 31.6, 0.01
|
||||||
|
assert_in_delta mid_lon, -97.35, 0.01
|
||||||
|
assert lat2 == 30.3
|
||||||
|
assert lon2 == -97.7
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns only pos1 when pos2 is nil" do
|
||||||
|
qso = create_qso(%{pos2: nil})
|
||||||
|
|
||||||
|
points = Radio.qso_path_points(qso)
|
||||||
|
|
||||||
|
assert points == [{32.9, -97.0}]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns empty list when pos1 is nil" do
|
||||||
|
qso = create_qso(%{pos1: nil})
|
||||||
|
|
||||||
|
assert Radio.qso_path_points(qso) == []
|
||||||
|
end
|
||||||
|
|
||||||
|
test "handles lng key in position maps" do
|
||||||
|
qso =
|
||||||
|
create_qso(%{
|
||||||
|
pos1: %{"lat" => 32.9, "lng" => -97.0},
|
||||||
|
pos2: %{"lat" => 30.3, "lng" => -97.7}
|
||||||
|
})
|
||||||
|
|
||||||
|
points = Radio.qso_path_points(qso)
|
||||||
|
assert length(points) == 3
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
describe "change_qso/2" do
|
describe "change_qso/2" do
|
||||||
test "returns a submission changeset" do
|
test "returns a submission changeset" do
|
||||||
changeset = Radio.change_qso(%Qso{})
|
changeset = Radio.change_qso(%Qso{})
|
||||||
|
|
|
||||||
|
|
@ -578,6 +578,112 @@ defmodule Microwaveprop.WeatherTest do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe "hrrr_profiles_for_path/1" do
|
||||||
|
test "returns HRRR profiles for all path points" do
|
||||||
|
# Profile at pos1
|
||||||
|
Weather.upsert_hrrr_profile(@hrrr_profile_attrs)
|
||||||
|
|
||||||
|
# Profile near pos2
|
||||||
|
Weather.upsert_hrrr_profile(%{
|
||||||
|
@hrrr_profile_attrs
|
||||||
|
| lat: 30.30,
|
||||||
|
lon: -97.70
|
||||||
|
})
|
||||||
|
|
||||||
|
qso = %{
|
||||||
|
pos1: %{"lat" => 32.91, "lon" => -97.05},
|
||||||
|
pos2: %{"lat" => 30.31, "lon" => -97.71},
|
||||||
|
qso_timestamp: ~U[2026-03-28 18:30:00Z]
|
||||||
|
}
|
||||||
|
|
||||||
|
profiles = Weather.hrrr_profiles_for_path(qso)
|
||||||
|
assert length(profiles) >= 2
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns empty list when no HRRR profiles exist nearby" do
|
||||||
|
qso = %{
|
||||||
|
pos1: %{"lat" => 40.0, "lon" => -80.0},
|
||||||
|
pos2: %{"lat" => 41.0, "lon" => -81.0},
|
||||||
|
qso_timestamp: ~U[2026-03-28 18:00:00Z]
|
||||||
|
}
|
||||||
|
|
||||||
|
assert Weather.hrrr_profiles_for_path(qso) == []
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns profiles for pos1 only when pos2 is nil" do
|
||||||
|
Weather.upsert_hrrr_profile(@hrrr_profile_attrs)
|
||||||
|
|
||||||
|
qso = %{
|
||||||
|
pos1: %{"lat" => 32.91, "lon" => -97.05},
|
||||||
|
pos2: nil,
|
||||||
|
qso_timestamp: ~U[2026-03-28 18:30:00Z]
|
||||||
|
}
|
||||||
|
|
||||||
|
profiles = Weather.hrrr_profiles_for_path(qso)
|
||||||
|
assert length(profiles) == 1
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "iemre_for_path/1" do
|
||||||
|
test "returns IEMRE observations for all path points" do
|
||||||
|
Weather.upsert_iemre_observation(@iemre_attrs)
|
||||||
|
|
||||||
|
# Observation near pos2
|
||||||
|
Weather.upsert_iemre_observation(%{
|
||||||
|
lat: 30.25,
|
||||||
|
lon: -97.75,
|
||||||
|
date: ~D[2026-03-28],
|
||||||
|
hourly: [%{"hour" => 0, "p01m_mm" => 1.0}]
|
||||||
|
})
|
||||||
|
|
||||||
|
qso = %{
|
||||||
|
pos1: %{"lat" => 32.9, "lon" => -97.0},
|
||||||
|
pos2: %{"lat" => 30.3, "lon" => -97.7},
|
||||||
|
qso_timestamp: ~U[2026-03-28 18:00:00Z]
|
||||||
|
}
|
||||||
|
|
||||||
|
observations = Weather.iemre_for_path(qso)
|
||||||
|
assert length(observations) >= 2
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns empty list when no observations exist" do
|
||||||
|
qso = %{
|
||||||
|
pos1: %{"lat" => 40.0, "lon" => -80.0},
|
||||||
|
pos2: %{"lat" => 41.0, "lon" => -81.0},
|
||||||
|
qso_timestamp: ~U[2026-03-28 18:00:00Z]
|
||||||
|
}
|
||||||
|
|
||||||
|
assert Weather.iemre_for_path(qso) == []
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns observation for pos1 only when pos2 is nil" do
|
||||||
|
Weather.upsert_iemre_observation(@iemre_attrs)
|
||||||
|
|
||||||
|
qso = %{
|
||||||
|
pos1: %{"lat" => 32.9, "lon" => -97.0},
|
||||||
|
pos2: nil,
|
||||||
|
qso_timestamp: ~U[2026-03-28 18:00:00Z]
|
||||||
|
}
|
||||||
|
|
||||||
|
observations = Weather.iemre_for_path(qso)
|
||||||
|
assert length(observations) == 1
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "find_nearest_hrrr/3" do
|
||||||
|
test "returns nearest HRRR profile for given lat, lon, and timestamp" do
|
||||||
|
Weather.upsert_hrrr_profile(@hrrr_profile_attrs)
|
||||||
|
|
||||||
|
profile = Weather.find_nearest_hrrr(32.91, -97.05, ~U[2026-03-28 18:30:00Z])
|
||||||
|
assert profile
|
||||||
|
assert profile.lat == 32.90
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns nil when no HRRR profiles exist nearby" do
|
||||||
|
assert Weather.find_nearest_hrrr(40.0, -80.0, ~U[2026-03-28 18:00:00Z]) == nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
describe "iemre_for_qso/1" do
|
describe "iemre_for_qso/1" do
|
||||||
test "returns IEMRE observation matching QSO pos1 and date" do
|
test "returns IEMRE observation matching QSO pos1 and date" do
|
||||||
Weather.upsert_iemre_observation(@iemre_attrs)
|
Weather.upsert_iemre_observation(@iemre_attrs)
|
||||||
|
|
|
||||||
|
|
@ -113,8 +113,19 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
|
||||||
assert length(asos_jobs) == 1
|
assert length(asos_jobs) == 1
|
||||||
end
|
end
|
||||||
|
|
||||||
|
test "builds ASOS jobs for stations near pos2 and midpoint" do
|
||||||
|
# Station near pos2 (30.3, -97.7) but NOT near pos1 (32.9, -97.0)
|
||||||
|
station_near_pos2 = create_asos_station("KAUS", 30.20, -97.68)
|
||||||
|
qso = create_qso()
|
||||||
|
|
||||||
|
jobs = QsoWeatherEnqueueWorker.build_weather_jobs([qso])
|
||||||
|
|
||||||
|
station_ids = Enum.map(jobs, & &1.changes.args["station_id"])
|
||||||
|
assert station_near_pos2.id in station_ids
|
||||||
|
end
|
||||||
|
|
||||||
test "returns empty list when no stations nearby" do
|
test "returns empty list when no stations nearby" do
|
||||||
qso = create_qso(%{pos1: %{"lat" => 60.0, "lon" => -150.0}})
|
qso = create_qso(%{pos1: %{"lat" => 60.0, "lon" => -150.0}, pos2: nil})
|
||||||
|
|
||||||
assert QsoWeatherEnqueueWorker.build_weather_jobs([qso]) == []
|
assert QsoWeatherEnqueueWorker.build_weather_jobs([qso]) == []
|
||||||
end
|
end
|
||||||
|
|
@ -244,11 +255,22 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
|
||||||
end
|
end
|
||||||
|
|
||||||
describe "build_hrrr_jobs/1" do
|
describe "build_hrrr_jobs/1" do
|
||||||
test "builds one HRRR job per QSO with rounded lat/lon" do
|
test "builds HRRR jobs for all path points (pos1, midpoint, pos2)" do
|
||||||
qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}})
|
qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}})
|
||||||
|
|
||||||
jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso])
|
jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso])
|
||||||
|
|
||||||
|
# pos1, midpoint, pos2 = 3 distinct grid points
|
||||||
|
assert length(jobs) == 3
|
||||||
|
lats = jobs |> Enum.map(& &1.changes.args["lat"]) |> Enum.sort()
|
||||||
|
assert 32.91 in lats
|
||||||
|
end
|
||||||
|
|
||||||
|
test "builds only one HRRR job when pos2 is nil" do
|
||||||
|
qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}, pos2: nil})
|
||||||
|
|
||||||
|
jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso])
|
||||||
|
|
||||||
assert length(jobs) == 1
|
assert length(jobs) == 1
|
||||||
job = hd(jobs)
|
job = hd(jobs)
|
||||||
assert job.changes.args["lat"] == 32.91
|
assert job.changes.args["lat"] == 32.91
|
||||||
|
|
@ -256,7 +278,7 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
|
||||||
end
|
end
|
||||||
|
|
||||||
test "rounds valid_time to nearest hour" do
|
test "rounds valid_time to nearest hour" do
|
||||||
qso = create_qso(%{qso_timestamp: ~U[2026-03-28 18:45:00Z]})
|
qso = create_qso(%{qso_timestamp: ~U[2026-03-28 18:45:00Z], pos2: nil})
|
||||||
|
|
||||||
jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso])
|
jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso])
|
||||||
|
|
||||||
|
|
@ -264,19 +286,21 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
|
||||||
assert job.changes.args["valid_time"] == "2026-03-28T19:00:00Z"
|
assert job.changes.args["valid_time"] == "2026-03-28T19:00:00Z"
|
||||||
end
|
end
|
||||||
|
|
||||||
test "deduplicates jobs with identical args" do
|
test "deduplicates jobs across QSOs with identical path points" do
|
||||||
q1 =
|
q1 =
|
||||||
create_qso(%{
|
create_qso(%{
|
||||||
station1: "A1",
|
station1: "A1",
|
||||||
qso_timestamp: ~U[2026-03-28 18:10:00Z],
|
qso_timestamp: ~U[2026-03-28 18:10:00Z],
|
||||||
pos1: %{"lat" => 32.90, "lon" => -97.04}
|
pos1: %{"lat" => 32.90, "lon" => -97.04},
|
||||||
|
pos2: nil
|
||||||
})
|
})
|
||||||
|
|
||||||
q2 =
|
q2 =
|
||||||
create_qso(%{
|
create_qso(%{
|
||||||
station1: "A2",
|
station1: "A2",
|
||||||
qso_timestamp: ~U[2026-03-28 18:20:00Z],
|
qso_timestamp: ~U[2026-03-28 18:20:00Z],
|
||||||
pos1: %{"lat" => 32.90, "lon" => -97.04}
|
pos1: %{"lat" => 32.90, "lon" => -97.04},
|
||||||
|
pos2: nil
|
||||||
})
|
})
|
||||||
|
|
||||||
jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([q1, q2])
|
jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([q1, q2])
|
||||||
|
|
@ -285,16 +309,30 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
|
||||||
assert length(jobs) == 1
|
assert length(jobs) == 1
|
||||||
end
|
end
|
||||||
|
|
||||||
|
test "deduplicates path points that round to the same HRRR grid cell" do
|
||||||
|
# pos1 and pos2 only 0.001° apart — all 3 points round to same HRRR cell
|
||||||
|
qso =
|
||||||
|
create_qso(%{
|
||||||
|
pos1: %{"lat" => 32.901, "lon" => -97.041},
|
||||||
|
pos2: %{"lat" => 32.902, "lon" => -97.042}
|
||||||
|
})
|
||||||
|
|
||||||
|
jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso])
|
||||||
|
|
||||||
|
# All 3 path points round to {32.90, -97.04} → 1 job
|
||||||
|
assert length(jobs) == 1
|
||||||
|
end
|
||||||
|
|
||||||
test "skips QSOs without pos1" do
|
test "skips QSOs without pos1" do
|
||||||
qso = create_qso(%{pos1: nil})
|
qso = create_qso(%{pos1: nil})
|
||||||
|
|
||||||
assert QsoWeatherEnqueueWorker.build_hrrr_jobs([qso]) == []
|
assert QsoWeatherEnqueueWorker.build_hrrr_jobs([qso]) == []
|
||||||
end
|
end
|
||||||
|
|
||||||
test "skips HRRR job when profile already exists for that location and time" do
|
test "skips only path points where HRRR profile already exists" do
|
||||||
qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}})
|
qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}})
|
||||||
|
|
||||||
# Insert a profile at the rounded grid point and nearest hour
|
# Insert a profile at the pos1 rounded grid point only
|
||||||
Weather.upsert_hrrr_profile(%{
|
Weather.upsert_hrrr_profile(%{
|
||||||
valid_time: ~U[2026-03-28 18:00:00Z],
|
valid_time: ~U[2026-03-28 18:00:00Z],
|
||||||
lat: 32.91,
|
lat: 32.91,
|
||||||
|
|
@ -308,7 +346,12 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
|
||||||
surface_pressure_mb: 1013.0
|
surface_pressure_mb: 1013.0
|
||||||
})
|
})
|
||||||
|
|
||||||
assert QsoWeatherEnqueueWorker.build_hrrr_jobs([qso]) == []
|
jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso])
|
||||||
|
|
||||||
|
# pos1 skipped (exists), midpoint + pos2 still need fetching
|
||||||
|
assert length(jobs) == 2
|
||||||
|
lats = Enum.map(jobs, & &1.changes.args["lat"])
|
||||||
|
refute 32.91 in lats
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
@ -380,11 +423,22 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
|
||||||
end
|
end
|
||||||
|
|
||||||
describe "build_iemre_jobs/1" do
|
describe "build_iemre_jobs/1" do
|
||||||
test "builds one IEMRE job per QSO with rounded lat/lon" do
|
test "builds IEMRE jobs for all path points (pos1, midpoint, pos2)" do
|
||||||
qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}})
|
qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}})
|
||||||
|
|
||||||
jobs = QsoWeatherEnqueueWorker.build_iemre_jobs([qso])
|
jobs = QsoWeatherEnqueueWorker.build_iemre_jobs([qso])
|
||||||
|
|
||||||
|
# pos1, midpoint, pos2 may have distinct IEMRE grid cells
|
||||||
|
assert length(jobs) >= 2
|
||||||
|
lats = Enum.map(jobs, & &1.changes.args["lat"])
|
||||||
|
assert 32.875 in lats
|
||||||
|
end
|
||||||
|
|
||||||
|
test "builds only one IEMRE job when pos2 is nil" do
|
||||||
|
qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}, pos2: nil})
|
||||||
|
|
||||||
|
jobs = QsoWeatherEnqueueWorker.build_iemre_jobs([qso])
|
||||||
|
|
||||||
assert length(jobs) == 1
|
assert length(jobs) == 1
|
||||||
job = hd(jobs)
|
job = hd(jobs)
|
||||||
assert job.changes.args["lat"] == 32.875
|
assert job.changes.args["lat"] == 32.875
|
||||||
|
|
@ -392,19 +446,21 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
|
||||||
assert job.changes.args["date"] == "2026-03-28"
|
assert job.changes.args["date"] == "2026-03-28"
|
||||||
end
|
end
|
||||||
|
|
||||||
test "deduplicates jobs with identical args" do
|
test "deduplicates jobs across QSOs with identical path points" do
|
||||||
q1 =
|
q1 =
|
||||||
create_qso(%{
|
create_qso(%{
|
||||||
station1: "A1",
|
station1: "A1",
|
||||||
qso_timestamp: ~U[2026-03-28 18:10:00Z],
|
qso_timestamp: ~U[2026-03-28 18:10:00Z],
|
||||||
pos1: %{"lat" => 32.90, "lon" => -97.04}
|
pos1: %{"lat" => 32.90, "lon" => -97.04},
|
||||||
|
pos2: nil
|
||||||
})
|
})
|
||||||
|
|
||||||
q2 =
|
q2 =
|
||||||
create_qso(%{
|
create_qso(%{
|
||||||
station1: "A2",
|
station1: "A2",
|
||||||
qso_timestamp: ~U[2026-03-28 20:20:00Z],
|
qso_timestamp: ~U[2026-03-28 20:20:00Z],
|
||||||
pos1: %{"lat" => 32.90, "lon" => -97.04}
|
pos1: %{"lat" => 32.90, "lon" => -97.04},
|
||||||
|
pos2: nil
|
||||||
})
|
})
|
||||||
|
|
||||||
jobs = QsoWeatherEnqueueWorker.build_iemre_jobs([q1, q2])
|
jobs = QsoWeatherEnqueueWorker.build_iemre_jobs([q1, q2])
|
||||||
|
|
@ -418,14 +474,16 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
|
||||||
create_qso(%{
|
create_qso(%{
|
||||||
station1: "A1",
|
station1: "A1",
|
||||||
qso_timestamp: ~U[2026-03-28 23:00:00Z],
|
qso_timestamp: ~U[2026-03-28 23:00:00Z],
|
||||||
pos1: %{"lat" => 32.90, "lon" => -97.04}
|
pos1: %{"lat" => 32.90, "lon" => -97.04},
|
||||||
|
pos2: nil
|
||||||
})
|
})
|
||||||
|
|
||||||
q2 =
|
q2 =
|
||||||
create_qso(%{
|
create_qso(%{
|
||||||
station1: "A2",
|
station1: "A2",
|
||||||
qso_timestamp: ~U[2026-03-29 01:00:00Z],
|
qso_timestamp: ~U[2026-03-29 01:00:00Z],
|
||||||
pos1: %{"lat" => 32.90, "lon" => -97.04}
|
pos1: %{"lat" => 32.90, "lon" => -97.04},
|
||||||
|
pos2: nil
|
||||||
})
|
})
|
||||||
|
|
||||||
jobs = QsoWeatherEnqueueWorker.build_iemre_jobs([q1, q2])
|
jobs = QsoWeatherEnqueueWorker.build_iemre_jobs([q1, q2])
|
||||||
|
|
@ -437,10 +495,10 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
|
||||||
assert QsoWeatherEnqueueWorker.build_iemre_jobs([qso]) == []
|
assert QsoWeatherEnqueueWorker.build_iemre_jobs([qso]) == []
|
||||||
end
|
end
|
||||||
|
|
||||||
test "skips IEMRE job when observation already exists for that location and date" do
|
test "skips only path points where IEMRE observation already exists" do
|
||||||
qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}})
|
qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}})
|
||||||
|
|
||||||
# Insert an IEMRE observation at the rounded grid point and date
|
# Insert an IEMRE observation at the pos1 rounded grid point only
|
||||||
Weather.upsert_iemre_observation(%{
|
Weather.upsert_iemre_observation(%{
|
||||||
lat: 32.875,
|
lat: 32.875,
|
||||||
lon: -97.0,
|
lon: -97.0,
|
||||||
|
|
@ -448,7 +506,12 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do
|
||||||
hourly: [%{"hour" => 0, "p01m_mm" => 0.0}]
|
hourly: [%{"hour" => 0, "p01m_mm" => 0.0}]
|
||||||
})
|
})
|
||||||
|
|
||||||
assert QsoWeatherEnqueueWorker.build_iemre_jobs([qso]) == []
|
jobs = QsoWeatherEnqueueWorker.build_iemre_jobs([qso])
|
||||||
|
|
||||||
|
# pos1 skipped, midpoint + pos2 still need fetching
|
||||||
|
assert length(jobs) >= 1
|
||||||
|
lats = Enum.map(jobs, & &1.changes.args["lat"])
|
||||||
|
refute 32.875 in lats
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
||||||
65
test/mix/tasks/reset_enrichment_test.exs
Normal file
65
test/mix/tasks/reset_enrichment_test.exs
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
defmodule Mix.Tasks.ResetEnrichmentTest do
|
||||||
|
use Microwaveprop.DataCase, async: true
|
||||||
|
|
||||||
|
alias Microwaveprop.Radio.Qso
|
||||||
|
alias Mix.Tasks.ResetEnrichment
|
||||||
|
|
||||||
|
defp create_queued_qso do
|
||||||
|
import Ecto.Query
|
||||||
|
|
||||||
|
default = %{
|
||||||
|
station1: "W5XD",
|
||||||
|
station2: "K5TR",
|
||||||
|
qso_timestamp: ~U[2026-03-28 18:00:00Z],
|
||||||
|
mode: "CW",
|
||||||
|
band: Decimal.new("1296"),
|
||||||
|
grid1: "EM12",
|
||||||
|
grid2: "EM00",
|
||||||
|
pos1: %{"lat" => 32.9, "lon" => -97.0},
|
||||||
|
pos2: %{"lat" => 30.3, "lon" => -97.7},
|
||||||
|
distance_km: Decimal.new("295")
|
||||||
|
}
|
||||||
|
|
||||||
|
{:ok, qso} =
|
||||||
|
%Qso{}
|
||||||
|
|> Qso.changeset(default)
|
||||||
|
|> Repo.insert()
|
||||||
|
|
||||||
|
# Set all queued flags to true (not in changeset cast)
|
||||||
|
Qso
|
||||||
|
|> where([q], q.id == ^qso.id)
|
||||||
|
|> Repo.update_all(set: [weather_queued: true, hrrr_queued: true, iemre_queued: true, terrain_queued: true])
|
||||||
|
|
||||||
|
Repo.get!(Qso, qso.id)
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "run/1" do
|
||||||
|
test "resets weather_queued, hrrr_queued, and iemre_queued to false" do
|
||||||
|
qso = create_queued_qso()
|
||||||
|
|
||||||
|
assert qso.weather_queued == true
|
||||||
|
assert qso.hrrr_queued == true
|
||||||
|
assert qso.iemre_queued == true
|
||||||
|
|
||||||
|
ResetEnrichment.run([])
|
||||||
|
|
||||||
|
updated = Repo.get!(Qso, qso.id)
|
||||||
|
assert updated.weather_queued == false
|
||||||
|
assert updated.hrrr_queued == false
|
||||||
|
assert updated.iemre_queued == false
|
||||||
|
end
|
||||||
|
|
||||||
|
test "does not reset terrain_queued" do
|
||||||
|
qso = create_queued_qso()
|
||||||
|
|
||||||
|
ResetEnrichment.run([])
|
||||||
|
|
||||||
|
updated = Repo.get!(Qso, qso.id)
|
||||||
|
assert updated.terrain_queued == true
|
||||||
|
end
|
||||||
|
|
||||||
|
test "handles empty database" do
|
||||||
|
assert :ok = ResetEnrichment.run([])
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
Loading…
Add table
Reference in a new issue