feat(pskr): preserve full locator precision from MQTT spots

Spots that include subsquare (EM12kl), extended-square (EM12kl37),
or extended-extended-subsquare (EM12kl37ab) precision now keep
that resolution end-to-end. Previously we truncated everything to
4 chars at parse time, which collapsed distinct paths into the same
row whenever two spotters shared a field.

The aggregation key uses the full grid as reported, so two
spotters in different subsquares of `EM12` produce two rows —
correct, since they are physically different paths. Any spot that
fails Maidenhead validation (odd length, illegal char at position
N) is still dropped via the same error path.

Storage normalizes to uppercase so the unique index doesn't split
on case. Maidenhead.valid?/1 already accepts both cases on input.
This commit is contained in:
Graham McIntire 2026-05-04 12:42:46 -05:00
parent 5521b0b153
commit b924cc2fc2
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
4 changed files with 58 additions and 22 deletions

View file

@ -31,12 +31,19 @@ defmodule Microwaveprop.Pskr do
## What we discard
The aggregator collapses every spot inside `(hour_utc, band,
sender_grid_4, receiver_grid_4)` into one `pskr_spots_hourly`
row. We keep counts + SNR envelope + the set of modes the
sender_grid, receiver_grid)` into one `pskr_spots_hourly` row.
We keep counts + SNR envelope + the set of modes the
individual report identity, sequence number, and exact intra-hour
timestamp are dropped on the floor. A 50-reporter QSO becomes a
single bumped count, not 50 rows; that's the design, not a
limitation.
Locator precision is preserved as reported: a spotter who shares
`EM12kl` keeps subsquare resolution, while a `EM12kl37` reporter
keeps extended-square resolution. The bucket key uses the full
grid, so two spotters in different subsquares of the same field
produce two rows that's the right behavior, since they're
literally different paths.
"""
alias Microwaveprop.Geo
@ -60,10 +67,10 @@ defmodule Microwaveprop.Pskr do
@doc """
Decodes a single MQTT payload (binary JSON) into a normalized
spot map. Truncates locators to 4 characters since hourly
aggregation is at subsquare resolution. Returns `{:error, reason}`
for any payload missing the fields we need the caller should
count and discard.
spot map. Locators are kept at the precision the spotter
reported (4/6/8/10 chars) and uppercased for consistent storage.
Returns `{:error, reason}` for any payload missing the fields we
need the caller should count and discard.
"""
@spec parse_spot(binary()) :: {:ok, spot()} | {:error, term()}
def parse_spot(payload) when is_binary(payload) do
@ -88,7 +95,7 @@ defmodule Microwaveprop.Pskr do
end
end
@doc "Bucket a spot to (top-of-hour, band, sender 4-char, receiver 4-char)."
@doc "Bucket a spot to (top-of-hour, band, sender_grid, receiver_grid) at full reported precision."
@spec path_key(spot()) :: path_key()
def path_key(%{transmit_time: t, band: band, sender_grid: snd, receiver_grid: rcv}) do
{%{t | minute: 0, second: 0, microsecond: {0, 0}}, band, snd, rcv}
@ -145,11 +152,16 @@ defmodule Microwaveprop.Pskr do
}
end
# Keep locators at the precision the spotter reported. Maidenhead
# validity already enforces minimum 4 chars + even length + the
# right alphabet at each position, so anything that passes is a
# legal 4/6/8/10-char locator we can store as-is. Uppercase for
# consistent indexing — the unique key is case-sensitive.
defp fetch_grid(json, key) do
case Map.get(json, key) do
grid when is_binary(grid) and byte_size(grid) >= 4 ->
head = grid |> String.slice(0, 4) |> String.upcase()
if Maidenhead.valid?(head), do: {:ok, head}, else: {:error, {:bad_grid, key, grid}}
grid when is_binary(grid) ->
upper = String.upcase(grid)
if Maidenhead.valid?(upper), do: {:ok, upper}, else: {:error, {:bad_grid, key, grid}}
other ->
{:error, {:missing_grid, key, other}}

View file

@ -1,7 +1,10 @@
defmodule Microwaveprop.Pskr.SpotHourly do
@moduledoc """
Hourly aggregate of PSK Reporter spots between two 4-character
Maidenhead grid squares on a single band.
Hourly aggregate of PSK Reporter spots between two Maidenhead
grid locators on a single band. Locator precision is preserved
from the source spot (4/6/8/10 chars), so two spotters in
different subsquares of the same field generate distinct rows
even though they share an `EM12` prefix.
A single QSO often produces 50+ reception reports (one per
monitoring station that decoded it). Storing each report verbatim

View file

@ -43,8 +43,8 @@ defmodule Microwaveprop.Pskr.AggregatorTest do
assert row.min_snr_db == -12
assert Enum.sort(row.modes) == ["FT4", "FT8"]
assert row.band == "2m"
assert row.sender_grid == "EM12"
assert row.receiver_grid == "DM43"
assert row.sender_grid == "EM12KL"
assert row.receiver_grid == "DM43ST"
assert row.distance_km > 0
end

View file

@ -20,7 +20,7 @@ defmodule Microwaveprop.PskrTest do
}
describe "parse_spot/1" do
test "extracts the fields needed for hourly aggregation" do
test "preserves full subsquare precision when reported" do
json = Jason.encode!(@sample)
assert {:ok, spot} = Pskr.parse_spot(json)
@ -28,14 +28,35 @@ defmodule Microwaveprop.PskrTest do
assert spot.band == "2m"
assert spot.mode == "FT8"
assert spot.snr_db == -5
assert spot.sender_grid == "EM12"
assert spot.receiver_grid == "DM43"
assert spot.sender_grid == "EM12KL"
assert spot.receiver_grid == "DM43ST"
assert spot.sender_country == 291
assert spot.receiver_country == 291
assert %DateTime{} = spot.transmit_time
assert DateTime.to_unix(spot.transmit_time) == 1_662_407_697
end
test "accepts 4-char grids verbatim" do
json = @sample |> Map.merge(%{"sl" => "EM12", "rl" => "DM43"}) |> Jason.encode!()
assert {:ok, spot} = Pskr.parse_spot(json)
assert spot.sender_grid == "EM12"
assert spot.receiver_grid == "DM43"
end
test "preserves extended-square (8-char) precision" do
json = @sample |> Map.merge(%{"sl" => "EM12kl37", "rl" => "DM43st99"}) |> Jason.encode!()
assert {:ok, spot} = Pskr.parse_spot(json)
assert spot.sender_grid == "EM12KL37"
assert spot.receiver_grid == "DM43ST99"
end
test "rejects odd-length grids (truncated 5-char)" do
json = @sample |> Map.put("sl", "EM12k") |> Jason.encode!()
assert {:error, _} = Pskr.parse_spot(json)
end
test "uses :t when :t_tx is missing" do
json = @sample |> Map.delete("t_tx") |> Jason.encode!()
@ -54,13 +75,13 @@ defmodule Microwaveprop.PskrTest do
end
describe "path_key/1" do
test "buckets to the top of the hour and 4-char grid" do
test "buckets to the top of the hour using the full grid as reported" do
{:ok, spot} = Pskr.parse_spot(Jason.encode!(@sample))
{hour, band, snd, rcv} = Pskr.path_key(spot)
assert band == "2m"
assert snd == "EM12"
assert rcv == "DM43"
assert snd == "EM12KL"
assert rcv == "DM43ST"
# 1_662_407_697 → 2022-09-05T19:54:57Z → bucket at 19:00:00Z
assert hour == ~U[2022-09-05 19:00:00Z]
end
@ -76,8 +97,8 @@ defmodule Microwaveprop.PskrTest do
assert acc.max_snr_db == -5
assert acc.min_snr_db == -5
assert acc.modes == ["FT8"]
assert acc.sender_grid == "EM12"
assert acc.receiver_grid == "DM43"
assert acc.sender_grid == "EM12KL"
assert acc.receiver_grid == "DM43ST"
assert is_float(acc.distance_km)
assert acc.distance_km > 0
# Midpoint should land between the two grid centers