From b924cc2fc229272bb1ab84e83fc6295822db5dda Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 4 May 2026 12:42:46 -0500 Subject: [PATCH] feat(pskr): preserve full locator precision from MQTT spots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spots that include subsquare (EM12kl), extended-square (EM12kl37), or extended-extended-subsquare (EM12kl37ab) precision now keep that resolution end-to-end. Previously we truncated everything to 4 chars at parse time, which collapsed distinct paths into the same row whenever two spotters shared a field. The aggregation key uses the full grid as reported, so two spotters in different subsquares of `EM12` produce two rows — correct, since they are physically different paths. Any spot that fails Maidenhead validation (odd length, illegal char at position N) is still dropped via the same error path. Storage normalizes to uppercase so the unique index doesn't split on case. Maidenhead.valid?/1 already accepts both cases on input. --- lib/microwaveprop/pskr.ex | 32 ++++++++++++------ lib/microwaveprop/pskr/spot_hourly.ex | 7 ++-- test/microwaveprop/pskr/aggregator_test.exs | 4 +-- test/microwaveprop/pskr_test.exs | 37 ++++++++++++++++----- 4 files changed, 58 insertions(+), 22 deletions(-) diff --git a/lib/microwaveprop/pskr.ex b/lib/microwaveprop/pskr.ex index f58520de..e2238875 100644 --- a/lib/microwaveprop/pskr.ex +++ b/lib/microwaveprop/pskr.ex @@ -31,12 +31,19 @@ defmodule Microwaveprop.Pskr do ## What we discard The aggregator collapses every spot inside `(hour_utc, band, - sender_grid_4, receiver_grid_4)` into one `pskr_spots_hourly` - row. We keep counts + SNR envelope + the set of modes — the + sender_grid, receiver_grid)` into one `pskr_spots_hourly` row. + We keep counts + SNR envelope + the set of modes — the individual report identity, sequence number, and exact intra-hour timestamp are dropped on the floor. A 50-reporter QSO becomes a single bumped count, not 50 rows; that's the design, not a limitation. + + Locator precision is preserved as reported: a spotter who shares + `EM12kl` keeps subsquare resolution, while a `EM12kl37` reporter + keeps extended-square resolution. The bucket key uses the full + grid, so two spotters in different subsquares of the same field + produce two rows — that's the right behavior, since they're + literally different paths. """ alias Microwaveprop.Geo @@ -60,10 +67,10 @@ defmodule Microwaveprop.Pskr do @doc """ Decodes a single MQTT payload (binary JSON) into a normalized - spot map. Truncates locators to 4 characters since hourly - aggregation is at subsquare resolution. Returns `{:error, reason}` - for any payload missing the fields we need — the caller should - count and discard. + spot map. Locators are kept at the precision the spotter + reported (4/6/8/10 chars) and uppercased for consistent storage. + Returns `{:error, reason}` for any payload missing the fields we + need — the caller should count and discard. """ @spec parse_spot(binary()) :: {:ok, spot()} | {:error, term()} def parse_spot(payload) when is_binary(payload) do @@ -88,7 +95,7 @@ defmodule Microwaveprop.Pskr do end end - @doc "Bucket a spot to (top-of-hour, band, sender 4-char, receiver 4-char)." + @doc "Bucket a spot to (top-of-hour, band, sender_grid, receiver_grid) at full reported precision." @spec path_key(spot()) :: path_key() def path_key(%{transmit_time: t, band: band, sender_grid: snd, receiver_grid: rcv}) do {%{t | minute: 0, second: 0, microsecond: {0, 0}}, band, snd, rcv} @@ -145,11 +152,16 @@ defmodule Microwaveprop.Pskr do } end + # Keep locators at the precision the spotter reported. Maidenhead + # validity already enforces minimum 4 chars + even length + the + # right alphabet at each position, so anything that passes is a + # legal 4/6/8/10-char locator we can store as-is. Uppercase for + # consistent indexing — the unique key is case-sensitive. defp fetch_grid(json, key) do case Map.get(json, key) do - grid when is_binary(grid) and byte_size(grid) >= 4 -> - head = grid |> String.slice(0, 4) |> String.upcase() - if Maidenhead.valid?(head), do: {:ok, head}, else: {:error, {:bad_grid, key, grid}} + grid when is_binary(grid) -> + upper = String.upcase(grid) + if Maidenhead.valid?(upper), do: {:ok, upper}, else: {:error, {:bad_grid, key, grid}} other -> {:error, {:missing_grid, key, other}} diff --git a/lib/microwaveprop/pskr/spot_hourly.ex b/lib/microwaveprop/pskr/spot_hourly.ex index e38c48df..e2c828ce 100644 --- a/lib/microwaveprop/pskr/spot_hourly.ex +++ b/lib/microwaveprop/pskr/spot_hourly.ex @@ -1,7 +1,10 @@ defmodule Microwaveprop.Pskr.SpotHourly do @moduledoc """ - Hourly aggregate of PSK Reporter spots between two 4-character - Maidenhead grid squares on a single band. + Hourly aggregate of PSK Reporter spots between two Maidenhead + grid locators on a single band. Locator precision is preserved + from the source spot (4/6/8/10 chars), so two spotters in + different subsquares of the same field generate distinct rows + even though they share an `EM12` prefix. A single QSO often produces 50+ reception reports (one per monitoring station that decoded it). Storing each report verbatim diff --git a/test/microwaveprop/pskr/aggregator_test.exs b/test/microwaveprop/pskr/aggregator_test.exs index 7661954b..dd8e21f7 100644 --- a/test/microwaveprop/pskr/aggregator_test.exs +++ b/test/microwaveprop/pskr/aggregator_test.exs @@ -43,8 +43,8 @@ defmodule Microwaveprop.Pskr.AggregatorTest do assert row.min_snr_db == -12 assert Enum.sort(row.modes) == ["FT4", "FT8"] assert row.band == "2m" - assert row.sender_grid == "EM12" - assert row.receiver_grid == "DM43" + assert row.sender_grid == "EM12KL" + assert row.receiver_grid == "DM43ST" assert row.distance_km > 0 end diff --git a/test/microwaveprop/pskr_test.exs b/test/microwaveprop/pskr_test.exs index 864bec6c..1a04a880 100644 --- a/test/microwaveprop/pskr_test.exs +++ b/test/microwaveprop/pskr_test.exs @@ -20,7 +20,7 @@ defmodule Microwaveprop.PskrTest do } describe "parse_spot/1" do - test "extracts the fields needed for hourly aggregation" do + test "preserves full subsquare precision when reported" do json = Jason.encode!(@sample) assert {:ok, spot} = Pskr.parse_spot(json) @@ -28,14 +28,35 @@ defmodule Microwaveprop.PskrTest do assert spot.band == "2m" assert spot.mode == "FT8" assert spot.snr_db == -5 - assert spot.sender_grid == "EM12" - assert spot.receiver_grid == "DM43" + assert spot.sender_grid == "EM12KL" + assert spot.receiver_grid == "DM43ST" assert spot.sender_country == 291 assert spot.receiver_country == 291 assert %DateTime{} = spot.transmit_time assert DateTime.to_unix(spot.transmit_time) == 1_662_407_697 end + test "accepts 4-char grids verbatim" do + json = @sample |> Map.merge(%{"sl" => "EM12", "rl" => "DM43"}) |> Jason.encode!() + + assert {:ok, spot} = Pskr.parse_spot(json) + assert spot.sender_grid == "EM12" + assert spot.receiver_grid == "DM43" + end + + test "preserves extended-square (8-char) precision" do + json = @sample |> Map.merge(%{"sl" => "EM12kl37", "rl" => "DM43st99"}) |> Jason.encode!() + + assert {:ok, spot} = Pskr.parse_spot(json) + assert spot.sender_grid == "EM12KL37" + assert spot.receiver_grid == "DM43ST99" + end + + test "rejects odd-length grids (truncated 5-char)" do + json = @sample |> Map.put("sl", "EM12k") |> Jason.encode!() + assert {:error, _} = Pskr.parse_spot(json) + end + test "uses :t when :t_tx is missing" do json = @sample |> Map.delete("t_tx") |> Jason.encode!() @@ -54,13 +75,13 @@ defmodule Microwaveprop.PskrTest do end describe "path_key/1" do - test "buckets to the top of the hour and 4-char grid" do + test "buckets to the top of the hour using the full grid as reported" do {:ok, spot} = Pskr.parse_spot(Jason.encode!(@sample)) {hour, band, snd, rcv} = Pskr.path_key(spot) assert band == "2m" - assert snd == "EM12" - assert rcv == "DM43" + assert snd == "EM12KL" + assert rcv == "DM43ST" # 1_662_407_697 → 2022-09-05T19:54:57Z → bucket at 19:00:00Z assert hour == ~U[2022-09-05 19:00:00Z] end @@ -76,8 +97,8 @@ defmodule Microwaveprop.PskrTest do assert acc.max_snr_db == -5 assert acc.min_snr_db == -5 assert acc.modes == ["FT8"] - assert acc.sender_grid == "EM12" - assert acc.receiver_grid == "DM43" + assert acc.sender_grid == "EM12KL" + assert acc.receiver_grid == "DM43ST" assert is_float(acc.distance_km) assert acc.distance_km > 0 # Midpoint should land between the two grid centers