diff --git a/lib/microwaveprop/radio.ex b/lib/microwaveprop/radio.ex index 2416840b..562d11ad 100644 --- a/lib/microwaveprop/radio.ex +++ b/lib/microwaveprop/radio.ex @@ -212,6 +212,54 @@ defmodule Microwaveprop.Radio do Contact.submission_changeset(contact, attrs) end + @doc """ + Computes pos1/pos2/distance_km from grid squares if missing. + Returns the updated contact, or the original if already computed or grids are missing. + """ + def ensure_positions!(contact) do + needs_pos1 = is_nil(contact.pos1) && contact.grid1 + needs_pos2 = is_nil(contact.pos2) && contact.grid2 + + if needs_pos1 || needs_pos2 do + pos1 = contact.pos1 || latlon_from_grid(contact.grid1) + pos2 = contact.pos2 || latlon_from_grid(contact.grid2) + + distance = + if pos1 && pos2 do + haversine_km(pos1["lat"], pos1["lon"], pos2["lat"], pos2["lon"]) + |> round() + |> Decimal.new() + else + contact.distance_km + end + + changes = + %{} + |> then(fn c -> if pos1 && is_nil(contact.pos1), do: Map.put(c, :pos1, pos1), else: c end) + |> then(fn c -> if pos2 && is_nil(contact.pos2), do: Map.put(c, :pos2, pos2), else: c end) + |> then(fn c -> if distance && is_nil(contact.distance_km), do: Map.put(c, :distance_km, distance), else: c end) + + if changes != %{} do + contact + |> Ecto.Changeset.change(changes) + |> Repo.update!() + else + contact + end + else + contact + end + end + + defp latlon_from_grid(nil), do: nil + + defp latlon_from_grid(grid) do + case Maidenhead.to_latlon(grid) do + {:ok, {lat, lon}} -> %{"lat" => lat, "lon" => lon} + _ -> nil + end + end + def create_contact(attrs) do changeset = Contact.submission_changeset(%Contact{}, attrs) diff --git a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex index b9f5635b..2e694acf 100644 --- a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex +++ b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex @@ -20,6 +20,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do Called directly from submission flow — no Oban indirection. """ def enqueue_for_contact(contact) do + contact = Radio.ensure_positions!(contact) weather_jobs = build_weather_jobs([contact]) hrrr_jobs = build_hrrr_jobs([contact]) terrain_jobs = build_terrain_jobs([contact]) diff --git a/lib/microwaveprop_web/live/backfill_live.ex b/lib/microwaveprop_web/live/backfill_live.ex index 0bcbaabe..2ca03d72 100644 --- a/lib/microwaveprop_web/live/backfill_live.ex +++ b/lib/microwaveprop_web/live/backfill_live.ex @@ -74,7 +74,7 @@ defmodule MicrowavepropWeb.BackfillLive do contacts = from(c in Contact, where: - not is_nil(c.pos1) and + (not is_nil(c.pos1) or not is_nil(c.grid1)) and (c.hrrr_status in ^@enrichable or c.weather_status in ^@enrichable or c.terrain_status in ^@enrichable or c.iemre_status in ^@enrichable), order_by: [desc: c.qso_timestamp], @@ -82,6 +82,7 @@ defmodule MicrowavepropWeb.BackfillLive do ) |> Repo.all() + # ensure_positions! is called inside enqueue_for_contact Enum.each(contacts, &ContactWeatherEnqueueWorker.enqueue_for_contact/1) length(contacts) end diff --git a/lib/microwaveprop_web/live/contact_live/show.ex b/lib/microwaveprop_web/live/contact_live/show.ex index 5ffe0ba4..bcd98ab4 100644 --- a/lib/microwaveprop_web/live/contact_live/show.ex +++ b/lib/microwaveprop_web/live/contact_live/show.ex @@ -20,7 +20,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do @impl true def mount(%{"id" => id}, _session, socket) do - contact = Radio.get_contact!(id) + contact = id |> Radio.get_contact!() |> Radio.ensure_positions!() if connected?(socket) do Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "contact_enrichment:#{contact.id}") diff --git a/priv/repo/migrations/20260402172332_fix_enrichment_status_from_actual_data.exs b/priv/repo/migrations/20260402172332_fix_enrichment_status_from_actual_data.exs new file mode 100644 index 00000000..4b07ec0c --- /dev/null +++ b/priv/repo/migrations/20260402172332_fix_enrichment_status_from_actual_data.exs @@ -0,0 +1,134 @@ +defmodule Microwaveprop.Repo.Migrations.FixEnrichmentStatusFromActualData do + use Ecto.Migration + + def up do + # First, backfill positions from grid squares where missing. + # Done in Elixir since Maidenhead decoding isn't available in SQL. + flush() + backfill_positions_from_grids() + + # Reset all "complete" statuses to "pending", then re-mark as "complete" + # only where actual data exists. This corrects the initial migration which + # assumed queued=true meant data exists. + + # Mark contacts without pos1 as unavailable for everything + execute """ + UPDATE qsos SET + hrrr_status = 'unavailable', + weather_status = 'unavailable', + terrain_status = 'unavailable', + iemre_status = 'unavailable' + WHERE pos1 IS NULL + """ + + # Mark contacts without pos2 as unavailable for terrain + execute """ + UPDATE qsos SET terrain_status = 'unavailable' + WHERE pos2 IS NULL AND pos1 IS NOT NULL + """ + + # Terrain: check actual terrain_profiles table + execute """ + UPDATE qsos SET terrain_status = 'pending' + WHERE terrain_status = 'complete' + AND id NOT IN (SELECT qso_id FROM terrain_profiles) + AND pos1 IS NOT NULL AND pos2 IS NOT NULL + """ + + # HRRR: reset to pending where no nearby profile exists + # (checking exact match is too slow for migration, reset all and let + # the backfill dashboard / page views re-enqueue as needed) + execute """ + UPDATE qsos SET hrrr_status = 'pending' + WHERE hrrr_status = 'complete' + AND pos1 IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM hrrr_profiles h + WHERE h.lat BETWEEN (qsos.pos1->>'lat')::float - 0.05 + AND (qsos.pos1->>'lat')::float + 0.05 + AND h.lon BETWEEN COALESCE(qsos.pos1->>'lon', qsos.pos1->>'lng')::float - 0.05 + AND COALESCE(qsos.pos1->>'lon', qsos.pos1->>'lng')::float + 0.05 + AND h.valid_time BETWEEN qsos.qso_timestamp - interval '1 hour' + AND qsos.qso_timestamp + interval '1 hour' + ) + """ + + # Weather: reset to pending where no surface observations exist nearby + execute """ + UPDATE qsos SET weather_status = 'pending' + WHERE weather_status = 'complete' + AND pos1 IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM surface_observations o + JOIN weather_stations ws ON ws.id = o.station_id + WHERE ws.lat BETWEEN (qsos.pos1->>'lat')::float - 3 + AND (qsos.pos1->>'lat')::float + 3 + AND ws.lon BETWEEN COALESCE(qsos.pos1->>'lon', qsos.pos1->>'lng')::float - 3 + AND COALESCE(qsos.pos1->>'lon', qsos.pos1->>'lng')::float + 3 + AND o.observed_at BETWEEN qsos.qso_timestamp - interval '6 hours' + AND qsos.qso_timestamp + interval '6 hours' + ) + """ + end + + def down do + # No rollback — statuses are now more accurate + :ok + end + + defp backfill_positions_from_grids do + import Ecto.Query + alias Microwaveprop.Radio.Maidenhead + + contacts = + from(q in "qsos", + where: is_nil(q.pos1) and not is_nil(q.grid1), + select: %{id: q.id, grid1: q.grid1, grid2: q.grid2} + ) + |> repo().all() + + for c <- contacts do + pos1 = grid_to_pos(c.grid1) + pos2 = grid_to_pos(c.grid2) + + if pos1 do + distance = + if pos1 && pos2 do + haversine(pos1["lat"], pos1["lon"], pos2["lat"], pos2["lon"]) + |> round() + end + + updates = + [pos1: pos1] + |> then(fn u -> if pos2, do: [{:pos2, pos2} | u], else: u end) + |> then(fn u -> if distance, do: [{:distance_km, distance} | u], else: u end) + |> Keyword.put(:updated_at, DateTime.truncate(DateTime.utc_now(), :second)) + + from(q in "qsos", where: q.id == ^c.id) + |> repo().update_all(set: updates) + end + end + end + + defp grid_to_pos(nil), do: nil + + defp grid_to_pos(grid) do + case Microwaveprop.Radio.Maidenhead.to_latlon(grid) do + {:ok, {lat, lon}} -> %{"lat" => lat, "lon" => lon} + _ -> nil + end + end + + defp haversine(lat1, lon1, lat2, lon2) do + dlat = (lat2 - lat1) * :math.pi() / 180 + dlon = (lon2 - lon1) * :math.pi() / 180 + rlat1 = lat1 * :math.pi() / 180 + rlat2 = lat2 * :math.pi() / 180 + + a = + :math.sin(dlat / 2) ** 2 + + :math.cos(rlat1) * :math.cos(rlat2) * :math.sin(dlon / 2) ** 2 + + 2 * 6371.0 * :math.asin(:math.sqrt(a)) + end +end diff --git a/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs b/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs index 309098f9..77cc3e6a 100644 --- a/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs +++ b/test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs @@ -607,9 +607,9 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do assert updated.iemre_status == :queued end - test "works for QSO without pos2 (marks terrain_queued false)" do + test "works for QSO without pos2 or grid2 (terrain stays pending)" do _station = create_asos_station("KDFW", 32.90, -97.04) - contact = create_contact(%{pos2: nil}) + contact = create_contact(%{pos2: nil, grid2: nil}) assert :ok = ContactWeatherEnqueueWorker.enqueue_for_contact(contact)