Graham McIntire
6d92973853
- Add `mix backtest --all` for consolidated pass/fail table across all features - Add Backtest.consolidated_report/2 and to_consolidated_markdown/1 - Add Features.all_features/0 to auto-discover backtestable features - Add `mix import_contest_logs` for bulk ARRL contest CSV import with dedup - Fix hrrr_climatology to batch by (month, hour) to avoid query timeout - Fix Repo.query! result pattern (Postgrex.Result, not tuple) - Backtest reports for all Phase 1-6 features
215 lines
6.8 KiB
Elixir
215 lines
6.8 KiB
Elixir
defmodule Microwaveprop.BacktestTest do
|
|
use Microwaveprop.DataCase, async: true
|
|
|
|
alias Microwaveprop.Backtest
|
|
alias Microwaveprop.Radio.Contact
|
|
|
|
defp create_contact(attrs \\ %{}) do
|
|
default = %{
|
|
station1: "W5XD",
|
|
station2: "K5TR",
|
|
qso_timestamp: ~U[2026-03-28 18:00:00Z],
|
|
mode: "CW",
|
|
band: Decimal.new("10000"),
|
|
grid1: "EM12",
|
|
grid2: "EM00",
|
|
pos1: %{"lat" => 32.9, "lon" => -97.0},
|
|
pos2: %{"lat" => 30.3, "lon" => -97.7},
|
|
distance_km: Decimal.new("295")
|
|
}
|
|
|
|
{:ok, contact} =
|
|
%Contact{}
|
|
|> Contact.changeset(Map.merge(default, attrs))
|
|
|> Repo.insert()
|
|
|
|
contact
|
|
end
|
|
|
|
describe "evaluate/2" do
|
|
test "returns counts and a qso distribution for a trivial feature" do
|
|
create_contact()
|
|
|
|
feature = fn _lat, _lon, _time -> 1.0 end
|
|
report = Backtest.evaluate(feature, sample_size: 10, baseline_size: 10)
|
|
|
|
assert report.qso_count == 1
|
|
assert report.baseline_count == 10
|
|
assert %Backtest.Distribution{} = report.qso_distribution
|
|
assert %Backtest.Distribution{} = report.baseline_distribution
|
|
assert report.qso_distribution.count == 1
|
|
assert report.qso_distribution.mean == 1.0
|
|
end
|
|
|
|
test "skips contacts with nil pos1" do
|
|
create_contact(%{pos1: nil, pos2: nil})
|
|
|
|
feature = fn _lat, _lon, _time -> 1.0 end
|
|
report = Backtest.evaluate(feature, sample_size: 10, baseline_size: 0)
|
|
|
|
assert report.qso_count == 0
|
|
end
|
|
|
|
test "excludes nil feature values from the distribution" do
|
|
create_contact(%{station1: "A"})
|
|
create_contact(%{station1: "B"})
|
|
|
|
feature = fn _lat, _lon, _time -> nil end
|
|
report = Backtest.evaluate(feature, sample_size: 10, baseline_size: 0)
|
|
|
|
assert report.qso_count == 2
|
|
assert report.qso_distribution.count == 0
|
|
end
|
|
end
|
|
|
|
describe "lift_by_distance/2" do
|
|
test "bins QSOs by distance_km and summarizes the feature per bin" do
|
|
create_contact(%{station1: "A", distance_km: Decimal.new("50"), pos1: %{"lat" => 1.0, "lon" => 0.0}})
|
|
create_contact(%{station1: "B", distance_km: Decimal.new("300"), pos1: %{"lat" => 5.0, "lon" => 0.0}})
|
|
create_contact(%{station1: "C", distance_km: Decimal.new("800"), pos1: %{"lat" => 9.0, "lon" => 0.0}})
|
|
|
|
f = fn lat, _lon, _time -> lat end
|
|
bins = Backtest.lift_by_distance(f, sample_size: 10)
|
|
|
|
assert bins["0-100"].count == 1
|
|
assert bins["0-100"].mean == 1.0
|
|
assert bins["100-250"].count == 0
|
|
assert bins["250-500"].count == 1
|
|
assert bins["250-500"].mean == 5.0
|
|
assert bins["500-1000"].count == 1
|
|
assert bins["500-1000"].mean == 9.0
|
|
assert bins["1000+"].count == 0
|
|
end
|
|
|
|
test "drops nil feature values from the bin stats" do
|
|
create_contact(%{station1: "A", distance_km: Decimal.new("50"), pos1: %{"lat" => 1.0, "lon" => 0.0}})
|
|
create_contact(%{station1: "B", distance_km: Decimal.new("60"), pos1: %{"lat" => 2.0, "lon" => 0.0}})
|
|
|
|
f = fn lat, _lon, _time -> if lat == 1.0, do: nil, else: 2.0 end
|
|
bins = Backtest.lift_by_distance(f, sample_size: 10)
|
|
|
|
assert bins["0-100"].count == 1
|
|
assert bins["0-100"].mean == 2.0
|
|
end
|
|
end
|
|
|
|
describe "random_baseline/2" do
|
|
test "generates N samples drawn from the contact locations" do
|
|
create_contact(%{
|
|
station1: "A",
|
|
qso_timestamp: ~U[2026-03-01 00:00:00Z],
|
|
pos1: %{"lat" => 1.0, "lon" => 10.0}
|
|
})
|
|
|
|
create_contact(%{
|
|
station1: "B",
|
|
qso_timestamp: ~U[2026-06-15 12:00:00Z],
|
|
pos1: %{"lat" => 2.0, "lon" => 20.0}
|
|
})
|
|
|
|
samples = Backtest.random_baseline(5, sample_size: 10)
|
|
assert length(samples) == 5
|
|
|
|
Enum.each(samples, fn {lat, lon, time} ->
|
|
assert lat in [1.0, 2.0]
|
|
assert lon in [10.0, 20.0]
|
|
assert %DateTime{} = time
|
|
end)
|
|
end
|
|
|
|
test "keeps timestamps within ±30 days of a source QSO" do
|
|
create_contact(%{
|
|
station1: "A",
|
|
qso_timestamp: ~U[2026-03-01 00:00:00Z],
|
|
pos1: %{"lat" => 1.0, "lon" => 10.0}
|
|
})
|
|
|
|
samples = Backtest.random_baseline(200, sample_size: 10)
|
|
source = ~U[2026-03-01 00:00:00Z]
|
|
|
|
Enum.each(samples, fn {_, _, time} ->
|
|
delta_seconds = abs(DateTime.diff(time, source))
|
|
assert delta_seconds <= 30 * 24 * 3600
|
|
end)
|
|
end
|
|
|
|
test "returns an empty list when there are no contacts with a position" do
|
|
assert Backtest.random_baseline(10) == []
|
|
end
|
|
end
|
|
|
|
describe "consolidated_report/1" do
|
|
test "runs all registered features and returns a summary per feature" do
|
|
create_contact(%{station1: "A", pos1: %{"lat" => 32.0, "lon" => -97.0}})
|
|
create_contact(%{station1: "B", pos1: %{"lat" => 33.0, "lon" => -96.0}})
|
|
|
|
features = %{
|
|
"always_one" => fn _lat, _lon, _time -> 1.0 end,
|
|
"always_nil" => fn _lat, _lon, _time -> nil end,
|
|
"uses_lat" => fn lat, _lon, _time -> lat end
|
|
}
|
|
|
|
results = Backtest.consolidated_report(features, sample_size: 10, baseline_size: 5)
|
|
|
|
assert is_list(results)
|
|
assert length(results) == 3
|
|
|
|
one = Enum.find(results, &(&1.feature_name == "always_one"))
|
|
assert one.qso_distribution.count == 2
|
|
assert one.qso_distribution.mean == 1.0
|
|
|
|
nil_feat = Enum.find(results, &(&1.feature_name == "always_nil"))
|
|
assert nil_feat.qso_distribution.count == 0
|
|
|
|
lat_feat = Enum.find(results, &(&1.feature_name == "uses_lat"))
|
|
assert lat_feat.qso_distribution.count == 2
|
|
end
|
|
|
|
test "to_consolidated_markdown renders a summary table" do
|
|
create_contact(%{station1: "A", pos1: %{"lat" => 32.0, "lon" => -97.0}})
|
|
|
|
features = %{
|
|
"feat_a" => fn _lat, _lon, _time -> 5.0 end,
|
|
"feat_b" => fn _lat, _lon, _time -> nil end
|
|
}
|
|
|
|
results = Backtest.consolidated_report(features, sample_size: 10, baseline_size: 5)
|
|
markdown = Backtest.to_consolidated_markdown(results)
|
|
|
|
assert markdown =~ "feat_a"
|
|
assert markdown =~ "feat_b"
|
|
assert markdown =~ "Consolidated"
|
|
assert markdown =~ "QSO N"
|
|
end
|
|
end
|
|
|
|
describe "lift_by_band/2" do
|
|
test "groups feature values by band and summarizes per band" do
|
|
create_contact(%{
|
|
station1: "A",
|
|
band: Decimal.new("10000"),
|
|
pos1: %{"lat" => 1.0, "lon" => 0.0}
|
|
})
|
|
|
|
create_contact(%{
|
|
station1: "B",
|
|
band: Decimal.new("24000"),
|
|
pos1: %{"lat" => 7.0, "lon" => 0.0}
|
|
})
|
|
|
|
create_contact(%{
|
|
station1: "C",
|
|
band: Decimal.new("24000"),
|
|
pos1: %{"lat" => 9.0, "lon" => 0.0}
|
|
})
|
|
|
|
f = fn lat, _lon, _time -> lat end
|
|
bands = Backtest.lift_by_band(f, sample_size: 10)
|
|
|
|
assert bands[Decimal.new("10000")].count == 1
|
|
assert bands[Decimal.new("10000")].mean == 1.0
|
|
assert bands[Decimal.new("24000")].count == 2
|
|
assert bands[Decimal.new("24000")].mean == 8.0
|
|
end
|
|
end
|
|
end
|