graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: normalize pos1/pos2 JSONB key to 'lon' everywhere

Browse source 
57,186 prod contacts stored pos1/pos2 with 'lng'; 1,133 used 'lon'.
Every Elixir caller carried a `pos["lon"] || pos["lng"]` fallback
— which just caused a SQL widget to silently miscount 98% of contacts
(count_narr_done used `pos1->>'lon'` directly, no fallback, so every
lng-keyed row returned NULL and failed the coverage check).

- Migration rewrites every pos1/pos2 JSONB in place, renaming 'lng' to
  'lon' and dropping 'lng'.
- Removes all 20+ `|| pos["lng"]` fallbacks across lib/, workers,
  scorer, weather, radio.ex, contact show view, and recalibrator.
- lib_ml/propagation_analyze.ex SQL now reads pos1->>'lon' directly
  (was reading 'lng' only, which would have broken after migration).
- priv/repo/import_contacts.exs one-time seed script now emits 'lon'
  with string keys, matching production shape.
- Test fixtures in 4 test files normalized to 'lon'.
- Two lng-characterization tests deleted — nonsensical post-normalize.
- Updated notebook + old import_weather script to match.
- JS hook contact_map_hook.ts TypeScript type narrowed to 'lon'.
This commit is contained in:
Graham McIntire 2026-04-17 09:10:20 -05:00
parent f252d4983b
commit 8a969e315c
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
19 changed files with 68 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
assets/js/contact_map_hook.ts
View file

@ -13,13 +13,13 @@ export const ContactMap = {
},
initMap(this: ContactMapHook) {
const pos1 = JSON.parse(this.el.dataset.pos1!) as { lat: number; lon?: number; lng?: number }
const pos2 = JSON.parse(this.el.dataset.pos2!) as { lat: number; lon?: number; lng?: number }
const pos1 = JSON.parse(this.el.dataset.pos1!) as { lat: number; lon: number }
const pos2 = JSON.parse(this.el.dataset.pos2!) as { lat: number; lon: number }
const lat1 = pos1.lat
const lon1 = (pos1.lon || pos1.lng)!
const lon1 = pos1.lon
const lat2 = pos2.lat
const lon2 = (pos2.lon || pos2.lng)!
const lon2 = pos2.lon
const bounds = L.latLngBounds([[lat1, lon1], [lat2, lon2]])

3
lib/microwaveprop/backtest.ex
View file

@ -384,11 +384,8 @@ defmodule Microwaveprop.Backtest do
defp eval_feature_for_contact(_feature, _contact), do: nil
# pos1 is sometimes stored with "lon" and sometimes with "lng".
defp pos_to_latlon(%{"lat" => lat, "lon" => lon}) when is_number(lat) and is_number(lon), do: {lat * 1.0, lon * 1.0}
defp pos_to_latlon(%{"lat" => lat, "lng" => lon}) when is_number(lat) and is_number(lon), do: {lat * 1.0, lon * 1.0}
defp pos_to_latlon(_), do: nil
# Matched random baseline: pick a random contact, perturb its

2
lib/microwaveprop/propagation/recalibrator.ex
View file

@ -159,7 +159,7 @@ defmodule Microwaveprop.Propagation.Recalibrator do
defp contact_to_factors(%Contact{pos1: pos, qso_timestamp: ts}) do
lat = pos["lat"]
lon = pos["lon"] || pos["lng"]
lon = pos["lon"]
if lat && lon do
case Weather.find_nearest_hrrr(lat, lon, ts) do

2
lib/microwaveprop/propagation/scorer.ex
View file

@ -532,7 +532,7 @@ defmodule Microwaveprop.Propagation.Scorer do
when dewpoints == [], do: nil
defp build_path_conditions({temps, dewpoints, pressures, gradients, bl_depths, pwats}, contact) do
lon = Kernel.||(contact.pos1["lon"] || contact.pos1["lng"], -97.0)
lon = contact.pos1["lon"] || -97.0
avg_temp_c = Enum.sum(temps) / length(temps)
avg_dewpoint_c = Enum.sum(dewpoints) / length(dewpoints)

20
lib/microwaveprop/radio.ex
View file

@ -77,9 +77,9 @@ defmodule Microwaveprop.Radio do
defp format_map_contact(c) do
lat1 = c.pos1["lat"]
lon1 = c.pos1["lon"] || c.pos1["lng"]
lon1 = c.pos1["lon"]
lat2 = c.pos2["lat"]
lon2 = c.pos2["lon"] || c.pos2["lng"]
lon2 = c.pos2["lon"]
if lat1 && lon1 && lat2 && lon2 do
[
@ -328,7 +328,7 @@ defmodule Microwaveprop.Radio do
def contact_path_points(%{pos1: pos1, pos2: nil}) do
lat = pos1["lat"]
lon = pos1["lon"] || pos1["lng"]
lon = pos1["lon"]
if lat && lon, do: [{lat, lon}], else: []
end
@ -337,7 +337,7 @@ defmodule Microwaveprop.Radio do
end
defp extract_latlon(%{"lat" => lat} = pos) when is_number(lat) do
lon = pos["lon"] || pos["lng"]
lon = pos["lon"]
if lon, do: {lat, lon}
end
@ -374,9 +374,9 @@ defmodule Microwaveprop.Radio do
for contact <- contacts,
is_nil(contact.distance_km) && contact.pos1 && contact.pos2,
lat1 = contact.pos1["lat"],
lon1 = contact.pos1["lon"] || contact.pos1["lng"],
lon1 = contact.pos1["lon"],
lat2 = contact.pos2["lat"],
lon2 = contact.pos2["lon"] || contact.pos2["lng"],
lon2 = contact.pos2["lon"],
lat1 && lon1 && lat2 && lon2 do
dist = lat1 |> haversine_km(lon1, lat2, lon2) |> round()
{contact.id, dist}
@ -451,8 +451,8 @@ defmodule Microwaveprop.Radio do
end
defp compute_distance(pos1, pos2, _existing_distance) when not is_nil(pos1) and not is_nil(pos2) do
lon1 = pos1["lon"] || pos1["lng"]
lon2 = pos2["lon"] || pos2["lng"]
lon1 = pos1["lon"]
lon2 = pos2["lon"]
pos1["lat"]
|> haversine_km(lon1, pos2["lat"], lon2)
@ -983,8 +983,8 @@ defmodule Microwaveprop.Radio do
end
defp compute_distance(pos1, pos2) when not is_nil(pos1) and not is_nil(pos2) do
lon1 = pos1["lon"] || pos1["lng"]
lon2 = pos2["lon"] || pos2["lng"]
lon1 = pos1["lon"]
lon2 = pos2["lon"]
pos1["lat"] |> haversine_km(lon1, pos2["lat"], lon2) |> round() |> Decimal.new()
end

6
lib/microwaveprop/weather.ex
View file

@ -788,7 +788,7 @@ defmodule Microwaveprop.Weather do
def hrrr_for_contact(contact) do
lat = contact.pos1["lat"]
lon = contact.pos1["lon"] || contact.pos1["lng"]
lon = contact.pos1["lon"]
if lat && lon do
find_nearest_hrrr(lat, lon, contact.qso_timestamp)
@ -893,7 +893,7 @@ defmodule Microwaveprop.Weather do
def narr_for_contact(contact) do
lat = contact.pos1["lat"]
lon = contact.pos1["lon"] || contact.pos1["lng"]
lon = contact.pos1["lon"]
if lat && lon do
find_nearest_narr(lat, lon, contact.qso_timestamp)
@ -1051,7 +1051,7 @@ defmodule Microwaveprop.Weather do
def iemre_for_contact(contact) do
lat = contact.pos1["lat"]
lon = contact.pos1["lon"] || contact.pos1["lng"]
lon = contact.pos1["lon"]
if lat && lon do
find_nearest_iemre(lat, lon, contact.qso_timestamp)

2
lib/microwaveprop/workers/hrrr_native_grid_worker.ex
View file

@ -67,7 +67,7 @@ defmodule Microwaveprop.Workers.HrrrNativeGridWorker do
|> select([c], c.pos1)
|> Repo.all()
|> Enum.flat_map(fn pos ->
case {pos["lat"], pos["lon"] || pos["lng"]} do
case {pos["lat"], pos["lon"]} do
{lat, lon} when is_number(lat) and is_number(lon) -> [{snap(lat), snap(lon)}]
_ -> []
end

2
lib/microwaveprop/workers/nexrad_worker.ex
View file

@ -55,7 +55,7 @@ defmodule Microwaveprop.Workers.NexradWorker do
|> select([c], c.pos1)
|> Repo.all()
|> Enum.flat_map(fn pos ->
case {pos["lat"], pos["lon"] || pos["lng"]} do
case {pos["lat"], pos["lon"]} do
{lat, lon} when is_number(lat) and is_number(lon) -> [{snap(lat), snap(lon)}]
_ -> []
end

4
lib/microwaveprop/workers/terrain_profile_worker.ex
View file

@ -30,9 +30,9 @@ defmodule Microwaveprop.Workers.TerrainProfileWorker do
contact = Radio.get_contact!(contact_id)
with %{"lat" => lat1} <- contact.pos1,
lon1 when is_number(lon1) <- contact.pos1["lon"] || contact.pos1["lng"],
lon1 when is_number(lon1) <- contact.pos1["lon"],
%{"lat" => lat2} <- contact.pos2,
lon2 when is_number(lon2) <- contact.pos2["lon"] || contact.pos2["lng"] do
lon2 when is_number(lon2) <- contact.pos2["lon"] do
fetch_and_store_profile(contact_id, contact, lat1, lon1, lat2, lon2)
else
_ ->

12
lib/microwaveprop_web/live/contact_live/show.ex
View file

@ -559,11 +559,11 @@ defmodule MicrowavepropWeb.ContactLive.Show do
defp extract_contact_coords(contact) do
lat1 = contact.pos1 && contact.pos1["lat"]
lon1 = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"])
lon1 = contact.pos1 && contact.pos1["lon"]
if lat1 && lon1 do
lat2 = contact.pos2 && contact.pos2["lat"]
lon2 = contact.pos2 && (contact.pos2["lon"] || contact.pos2["lng"])
lon2 = contact.pos2 && contact.pos2["lon"]
{:ok, lat1, lon1, lat2, lon2}
else
:error
@ -634,7 +634,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
defp maybe_enqueue_hrrr(nil, contact) do
lat = contact.pos1 && contact.pos1["lat"]
lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"])
lon = contact.pos1 && contact.pos1["lon"]
if lat && lon && contact.hrrr_status != :queued do
valid_time = HrrrClient.nearest_hrrr_hour(contact.qso_timestamp)
@ -668,7 +668,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
defp maybe_enqueue_narr(nil, %{hrrr_status: :unavailable} = contact) do
lat = contact.pos1 && contact.pos1["lat"]
lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"])
lon = contact.pos1 && contact.pos1["lon"]
valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)
if lat && lon && NarrClient.in_coverage?(valid_time) do
@ -1501,9 +1501,9 @@ defmodule MicrowavepropWeb.ContactLive.Show do
defp compute_elevation_profile(contact, hrrr_path, soundings) do
with %{"lat" => lat1} <- contact.pos1,
lon1 when is_number(lon1) <- contact.pos1["lon"] || contact.pos1["lng"],
lon1 when is_number(lon1) <- contact.pos1["lon"],
%{"lat" => lat2} <- contact.pos2,
lon2 when is_number(lon2) <- contact.pos2["lon"] || contact.pos2["lng"],
lon2 when is_number(lon2) <- contact.pos2["lon"],
{:ok, profile} <- safe_fetch_elevation(lat1, lon1, lat2, lon2) do
freq_ghz = band_to_ghz(contact.band)
dist_km = haversine_km(lat1, lon1, lat2, lon2)

6
lib_ml/propagation_analyze.ex
View file

@ -92,18 +92,18 @@ defmodule Mix.Tasks.PropagationAnalyze do
tp.verdict AS terrain_verdict,
-- Longitude for solar time
((q.pos1->>'lng')::float + (q.pos2->>'lng')::float) / 2.0 AS avg_longitude
((q.pos1->>'lon')::float + (q.pos2->>'lon')::float) / 2.0 AS avg_longitude
FROM qsos q
INNER JOIN hrrr_profiles h1
ON h1.lat = ROUND((q.pos1->>'lat')::numeric * 8) / 8
AND h1.lon = ROUND((q.pos1->>'lng')::numeric * 8) / 8
AND h1.lon = ROUND((q.pos1->>'lon')::numeric * 8) / 8
AND h1.valid_time = date_trunc('hour', q.qso_timestamp)
INNER JOIN hrrr_profiles h2
ON h2.lat = ROUND((q.pos2->>'lat')::numeric * 8) / 8
AND h2.lon = ROUND((q.pos2->>'lng')::numeric * 8) / 8
AND h2.lon = ROUND((q.pos2->>'lon')::numeric * 8) / 8
AND h2.valid_time = date_trunc('hour', q.qso_timestamp)
LEFT JOIN terrain_profiles tp ON tp.qso_id = q.id

2
notebooks/algorithm_analysis.livemd
View file

@ -325,7 +325,7 @@ scored =
utc_hour: contact.qso_timestamp.hour,
utc_minute: contact.qso_timestamp.minute,
month: contact.qso_timestamp.month,
longitude: (contact.pos1["lon"] || contact.pos1["lng"]) |> then(&(&1 || -97.0)),
longitude: contact.pos1["lon"] || -97.0,
temp_f: temp_f,
dewpoint_f: dewpoint_f,
min_refractivity_gradient: hrrr.min_refractivity_gradient,

4
priv/repo/import_contacts.exs
View file

@ -78,7 +78,7 @@ grid_coords =
fn grid ->
case Req.get("#{gridmap_base}#{grid}") do
{:ok, %{status: 200, body: %{"latitude" => lat, "longitude" => lng}}} ->
{grid, %{lat: lat, lng: lng}}
{grid, %{"lat" => lat, "lon" => lng}}
{:ok, resp} ->
IO.puts(" Warning: grid #{grid} returned status #{resp.status}")
@ -108,7 +108,7 @@ defmodule Haversine do
def distance_km(nil, _), do: nil
def distance_km(_, nil), do: nil
def distance_km(%{lat: lat1, lng: lng1}, %{lat: lat2, lng: lng2}) do
def distance_km(%{"lat" => lat1, "lon" => lng1}, %{"lat" => lat2, "lon" => lng2}) do
dlat = deg_to_rad(lat2 - lat1)
dlng = deg_to_rad(lng2 - lng1)
lat1_r = deg_to_rad(lat1)

2
priv/repo/import_weather.exs
View file

@ -198,7 +198,7 @@ Enum.with_index(qsos, 1)
if rem(idx, 500) == 0, do: IO.puts(" Scanned #{idx}/#{total} QSOs")
lat = qso.pos1["lat"] || qso.pos1["latitude"]
lon = qso.pos1["lng"] || qso.pos1["lon"] || qso.pos1["longitude"]
lon = qso.pos1["lon"] || qso.pos1["longitude"]
if lat && lon do
# Find nearby ASOS stations (within 150km)

27
priv/repo/migrations/20260417140525_normalize_pos_lng_to_lon.exs Normal file
View file

@ -0,0 +1,27 @@
defmodule Microwaveprop.Repo.Migrations.NormalizePosLngToLon do
use Ecto.Migration
def up do
execute("""
UPDATE contacts SET pos1 = jsonb_set(pos1 - 'lng', '{lon}', pos1->'lng')
WHERE pos1 ? 'lng'
""")
execute("""
UPDATE contacts SET pos2 = jsonb_set(pos2 - 'lng', '{lon}', pos2->'lng')
WHERE pos2 ? 'lng'
""")
end
def down do
execute("""
UPDATE contacts SET pos1 = jsonb_set(pos1 - 'lon', '{lng}', pos1->'lon')
WHERE pos1 ? 'lon' AND NOT (pos1 ? 'lng')
""")
execute("""
UPDATE contacts SET pos2 = jsonb_set(pos2 - 'lon', '{lng}', pos2->'lon')
WHERE pos2 ? 'lon' AND NOT (pos2 ? 'lng')
""")
end
end

4
test/microwaveprop/radio/contact_test.exs
View file

@ -9,8 +9,8 @@ defmodule Microwaveprop.Radio.ContactTest do
qso_timestamp: ~U[2026-03-28 12:00:00Z],
grid1: "FN31pr",
grid2: "EN91jm",
pos1: %{lat: 41.714, lng: -72.727},
pos2: %{lat: 41.049, lng: -80.166},
pos1: %{lat: 41.714, lon: -72.727},
pos2: %{lat: 41.049, lon: -80.166},
mode: "SSB",
band: Decimal.new("144.2"),
distance_km: Decimal.new("623.4")

6
test/microwaveprop/radio_test.exs
View file

@ -382,11 +382,11 @@ defmodule Microwaveprop.RadioTest do
assert Radio.contact_path_points(contact) == []
end
test "handles lng key in position maps" do
test "handles lon key in position maps" do
contact =
create_contact(%{
pos1: %{"lat" => 32.9, "lng" => -97.0},
pos2: %{"lat" => 30.3, "lng" => -97.7}
pos1: %{"lat" => 32.9, "lon" => -97.0},
pos2: %{"lat" => 30.3, "lon" => -97.7}
})
points = Radio.contact_path_points(contact)

7
test/microwaveprop/workers/hrrr_native_grid_worker_test.exs
View file

@ -52,13 +52,6 @@ defmodule Microwaveprop.Workers.HrrrNativeGridWorkerTest do
points = HrrrNativeGridWorker.points_of_interest_for_hour(~U[2026-03-28 18:00:00Z])
assert points == [{32.9, -97.0}]
end
test "handles contacts with pos1 stored as 'lng'" do
create_contact(%{pos1: %{"lat" => 32.9, "lng" => -97.0}})
points = HrrrNativeGridWorker.points_of_interest_for_hour(~U[2026-03-28 18:00:00Z])
assert points == [{32.9, -97.0}]
end
end
describe "perform/1" do

17
test/microwaveprop/workers/nexrad_worker_test.exs
View file

@ -259,23 +259,6 @@ defmodule Microwaveprop.Workers.NexradWorkerTest do
assert row.lat == 32.9
assert row.lon == -97.0
end
test ~s(accepts legacy pos1 maps that use the "lng" key instead of "lon") do
stub_png_success()
insert_contact!(%{
qso_timestamp: ~U[2022-08-20 14:10:00Z],
pos1: %{"lat" => 32.9, "lng" => -97.0}
})
args = %{"year" => 2022, "month" => 8, "day" => 20, "hour" => 14, "minute" => 0}
assert :ok = NexradWorker.perform(%Oban.Job{args: args})
assert [row] = Repo.all(NexradObservation)
assert row.lat == 32.9
assert row.lon == -97.0
end
end
describe "unique constraint" do