feat(pskr): preserve full locator precision from MQTT spots

Spots that include subsquare (EM12kl), extended-square (EM12kl37),
or extended-extended-subsquare (EM12kl37ab) precision now keep
that resolution end-to-end. Previously we truncated everything to
4 chars at parse time, which collapsed distinct paths into the same
row whenever two spotters shared a field.

The aggregation key uses the full grid as reported, so two
spotters in different subsquares of `EM12` produce two rows —
correct, since they are physically different paths. Any spot that
fails Maidenhead validation (odd length, illegal char at position
N) is still dropped via the same error path.

Storage normalizes to uppercase so the unique index doesn't split
on case. Maidenhead.valid?/1 already accepts both cases on input.
This commit is contained in:
Graham McIntire 2026-05-04 12:42:46 -05:00
parent 5521b0b153
commit b924cc2fc2
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
4 changed files with 58 additions and 22 deletions

View file

@ -31,12 +31,19 @@ defmodule Microwaveprop.Pskr do
## What we discard ## What we discard
The aggregator collapses every spot inside `(hour_utc, band, The aggregator collapses every spot inside `(hour_utc, band,
sender_grid_4, receiver_grid_4)` into one `pskr_spots_hourly` sender_grid, receiver_grid)` into one `pskr_spots_hourly` row.
row. We keep counts + SNR envelope + the set of modes the We keep counts + SNR envelope + the set of modes the
individual report identity, sequence number, and exact intra-hour individual report identity, sequence number, and exact intra-hour
timestamp are dropped on the floor. A 50-reporter QSO becomes a timestamp are dropped on the floor. A 50-reporter QSO becomes a
single bumped count, not 50 rows; that's the design, not a single bumped count, not 50 rows; that's the design, not a
limitation. limitation.
Locator precision is preserved as reported: a spotter who shares
`EM12kl` keeps subsquare resolution, while a `EM12kl37` reporter
keeps extended-square resolution. The bucket key uses the full
grid, so two spotters in different subsquares of the same field
produce two rows that's the right behavior, since they're
literally different paths.
""" """
alias Microwaveprop.Geo alias Microwaveprop.Geo
@ -60,10 +67,10 @@ defmodule Microwaveprop.Pskr do
@doc """ @doc """
Decodes a single MQTT payload (binary JSON) into a normalized Decodes a single MQTT payload (binary JSON) into a normalized
spot map. Truncates locators to 4 characters since hourly spot map. Locators are kept at the precision the spotter
aggregation is at subsquare resolution. Returns `{:error, reason}` reported (4/6/8/10 chars) and uppercased for consistent storage.
for any payload missing the fields we need the caller should Returns `{:error, reason}` for any payload missing the fields we
count and discard. need the caller should count and discard.
""" """
@spec parse_spot(binary()) :: {:ok, spot()} | {:error, term()} @spec parse_spot(binary()) :: {:ok, spot()} | {:error, term()}
def parse_spot(payload) when is_binary(payload) do def parse_spot(payload) when is_binary(payload) do
@ -88,7 +95,7 @@ defmodule Microwaveprop.Pskr do
end end
end end
@doc "Bucket a spot to (top-of-hour, band, sender 4-char, receiver 4-char)." @doc "Bucket a spot to (top-of-hour, band, sender_grid, receiver_grid) at full reported precision."
@spec path_key(spot()) :: path_key() @spec path_key(spot()) :: path_key()
def path_key(%{transmit_time: t, band: band, sender_grid: snd, receiver_grid: rcv}) do def path_key(%{transmit_time: t, band: band, sender_grid: snd, receiver_grid: rcv}) do
{%{t | minute: 0, second: 0, microsecond: {0, 0}}, band, snd, rcv} {%{t | minute: 0, second: 0, microsecond: {0, 0}}, band, snd, rcv}
@ -145,11 +152,16 @@ defmodule Microwaveprop.Pskr do
} }
end end
# Keep locators at the precision the spotter reported. Maidenhead
# validity already enforces minimum 4 chars + even length + the
# right alphabet at each position, so anything that passes is a
# legal 4/6/8/10-char locator we can store as-is. Uppercase for
# consistent indexing — the unique key is case-sensitive.
defp fetch_grid(json, key) do defp fetch_grid(json, key) do
case Map.get(json, key) do case Map.get(json, key) do
grid when is_binary(grid) and byte_size(grid) >= 4 -> grid when is_binary(grid) ->
head = grid |> String.slice(0, 4) |> String.upcase() upper = String.upcase(grid)
if Maidenhead.valid?(head), do: {:ok, head}, else: {:error, {:bad_grid, key, grid}} if Maidenhead.valid?(upper), do: {:ok, upper}, else: {:error, {:bad_grid, key, grid}}
other -> other ->
{:error, {:missing_grid, key, other}} {:error, {:missing_grid, key, other}}

View file

@ -1,7 +1,10 @@
defmodule Microwaveprop.Pskr.SpotHourly do defmodule Microwaveprop.Pskr.SpotHourly do
@moduledoc """ @moduledoc """
Hourly aggregate of PSK Reporter spots between two 4-character Hourly aggregate of PSK Reporter spots between two Maidenhead
Maidenhead grid squares on a single band. grid locators on a single band. Locator precision is preserved
from the source spot (4/6/8/10 chars), so two spotters in
different subsquares of the same field generate distinct rows
even though they share an `EM12` prefix.
A single QSO often produces 50+ reception reports (one per A single QSO often produces 50+ reception reports (one per
monitoring station that decoded it). Storing each report verbatim monitoring station that decoded it). Storing each report verbatim

View file

@ -43,8 +43,8 @@ defmodule Microwaveprop.Pskr.AggregatorTest do
assert row.min_snr_db == -12 assert row.min_snr_db == -12
assert Enum.sort(row.modes) == ["FT4", "FT8"] assert Enum.sort(row.modes) == ["FT4", "FT8"]
assert row.band == "2m" assert row.band == "2m"
assert row.sender_grid == "EM12" assert row.sender_grid == "EM12KL"
assert row.receiver_grid == "DM43" assert row.receiver_grid == "DM43ST"
assert row.distance_km > 0 assert row.distance_km > 0
end end

View file

@ -20,7 +20,7 @@ defmodule Microwaveprop.PskrTest do
} }
describe "parse_spot/1" do describe "parse_spot/1" do
test "extracts the fields needed for hourly aggregation" do test "preserves full subsquare precision when reported" do
json = Jason.encode!(@sample) json = Jason.encode!(@sample)
assert {:ok, spot} = Pskr.parse_spot(json) assert {:ok, spot} = Pskr.parse_spot(json)
@ -28,14 +28,35 @@ defmodule Microwaveprop.PskrTest do
assert spot.band == "2m" assert spot.band == "2m"
assert spot.mode == "FT8" assert spot.mode == "FT8"
assert spot.snr_db == -5 assert spot.snr_db == -5
assert spot.sender_grid == "EM12" assert spot.sender_grid == "EM12KL"
assert spot.receiver_grid == "DM43" assert spot.receiver_grid == "DM43ST"
assert spot.sender_country == 291 assert spot.sender_country == 291
assert spot.receiver_country == 291 assert spot.receiver_country == 291
assert %DateTime{} = spot.transmit_time assert %DateTime{} = spot.transmit_time
assert DateTime.to_unix(spot.transmit_time) == 1_662_407_697 assert DateTime.to_unix(spot.transmit_time) == 1_662_407_697
end end
test "accepts 4-char grids verbatim" do
json = @sample |> Map.merge(%{"sl" => "EM12", "rl" => "DM43"}) |> Jason.encode!()
assert {:ok, spot} = Pskr.parse_spot(json)
assert spot.sender_grid == "EM12"
assert spot.receiver_grid == "DM43"
end
test "preserves extended-square (8-char) precision" do
json = @sample |> Map.merge(%{"sl" => "EM12kl37", "rl" => "DM43st99"}) |> Jason.encode!()
assert {:ok, spot} = Pskr.parse_spot(json)
assert spot.sender_grid == "EM12KL37"
assert spot.receiver_grid == "DM43ST99"
end
test "rejects odd-length grids (truncated 5-char)" do
json = @sample |> Map.put("sl", "EM12k") |> Jason.encode!()
assert {:error, _} = Pskr.parse_spot(json)
end
test "uses :t when :t_tx is missing" do test "uses :t when :t_tx is missing" do
json = @sample |> Map.delete("t_tx") |> Jason.encode!() json = @sample |> Map.delete("t_tx") |> Jason.encode!()
@ -54,13 +75,13 @@ defmodule Microwaveprop.PskrTest do
end end
describe "path_key/1" do describe "path_key/1" do
test "buckets to the top of the hour and 4-char grid" do test "buckets to the top of the hour using the full grid as reported" do
{:ok, spot} = Pskr.parse_spot(Jason.encode!(@sample)) {:ok, spot} = Pskr.parse_spot(Jason.encode!(@sample))
{hour, band, snd, rcv} = Pskr.path_key(spot) {hour, band, snd, rcv} = Pskr.path_key(spot)
assert band == "2m" assert band == "2m"
assert snd == "EM12" assert snd == "EM12KL"
assert rcv == "DM43" assert rcv == "DM43ST"
# 1_662_407_697 → 2022-09-05T19:54:57Z → bucket at 19:00:00Z # 1_662_407_697 → 2022-09-05T19:54:57Z → bucket at 19:00:00Z
assert hour == ~U[2022-09-05 19:00:00Z] assert hour == ~U[2022-09-05 19:00:00Z]
end end
@ -76,8 +97,8 @@ defmodule Microwaveprop.PskrTest do
assert acc.max_snr_db == -5 assert acc.max_snr_db == -5
assert acc.min_snr_db == -5 assert acc.min_snr_db == -5
assert acc.modes == ["FT8"] assert acc.modes == ["FT8"]
assert acc.sender_grid == "EM12" assert acc.sender_grid == "EM12KL"
assert acc.receiver_grid == "DM43" assert acc.receiver_grid == "DM43ST"
assert is_float(acc.distance_km) assert is_float(acc.distance_km)
assert acc.distance_km > 0 assert acc.distance_km > 0
# Midpoint should land between the two grid centers # Midpoint should land between the two grid centers