From 6d92973853a7328ef41ca8620489d7c7638bc520 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Fri, 10 Apr 2026 11:56:54 -0500 Subject: [PATCH] Phase 9.1: Consolidated backtest report and contest log import - Add `mix backtest --all` for consolidated pass/fail table across all features - Add Backtest.consolidated_report/2 and to_consolidated_markdown/1 - Add Features.all_features/0 to auto-discover backtestable features - Add `mix import_contest_logs` for bulk ARRL contest CSV import with dedup - Fix hrrr_climatology to batch by (month, hour) to avoid query timeout - Fix Repo.query! result pattern (Postgrex.Result, not tuple) - Backtest reports for all Phase 1-6 features --- lib/microwaveprop/backtest.ex | 69 +++++++++++++ lib/microwaveprop/backtest/features.ex | 17 ++++ lib/mix/tasks/backtest.ex | 49 +++++++-- lib/mix/tasks/hrrr_climatology.ex | 80 +++++++++------ lib/mix/tasks/import_contest_logs.ex | 99 +++++++++++++++++++ .../baseline_naive_gradient.md | 2 +- priv/backtest_reports/best_duct_freq.md | 27 +++++ priv/backtest_reports/bulk_richardson.md | 27 +++++ priv/backtest_reports/duct_thickness.md | 27 +++++ priv/backtest_reports/duct_usable_10ghz.md | 27 +++++ .../native_surface_refractivity.md | 27 +++++ priv/backtest_reports/shear_at_top.md | 27 +++++ priv/backtest_reports/temperature_anomaly.md | 31 ++++++ priv/backtest_reports/theta_e_jump.md | 27 +++++ test/microwaveprop/backtest_test.exs | 45 +++++++++ 15 files changed, 544 insertions(+), 37 deletions(-) create mode 100644 lib/mix/tasks/import_contest_logs.ex create mode 100644 priv/backtest_reports/best_duct_freq.md create mode 100644 priv/backtest_reports/bulk_richardson.md create mode 100644 priv/backtest_reports/duct_thickness.md create mode 100644 priv/backtest_reports/duct_usable_10ghz.md create mode 100644 priv/backtest_reports/native_surface_refractivity.md create mode 100644 priv/backtest_reports/shear_at_top.md create mode 100644 priv/backtest_reports/temperature_anomaly.md create mode 100644 priv/backtest_reports/theta_e_jump.md diff --git a/lib/microwaveprop/backtest.ex b/lib/microwaveprop/backtest.ex index 108d1cac..b5e57fd8 100644 --- a/lib/microwaveprop/backtest.ex +++ b/lib/microwaveprop/backtest.ex @@ -232,6 +232,75 @@ defmodule Microwaveprop.Backtest do } end + @doc """ + Runs multiple features against the same QSO sample and returns a list + of summary maps — one per feature — suitable for a consolidated table. + + `features` is a map of `%{name => fun/3}`. + + Each entry includes the feature name, QSO and baseline distribution + stats, and a pass/fail gate based on whether the feature has data. + """ + @spec consolidated_report(%{String.t() => feature}, keyword) :: [map()] + def consolidated_report(features, opts \\ []) when is_map(features) do + sample_size = Keyword.get(opts, :sample_size, 5000) + baseline_size = Keyword.get(opts, :baseline_size, sample_size) + + contacts = load_contacts(sample_size) + + baseline_samples = baseline_samples(contacts, baseline_size) + + features + |> Enum.map(fn {name, fun} -> + qso_values = + contacts + |> Enum.map(&eval_feature_for_contact(fun, &1)) + |> Enum.reject(&is_nil/1) + + baseline_values = + baseline_samples + |> Enum.map(fn {lat, lon, time} -> fun.(lat, lon, time) end) + |> Enum.reject(&is_nil/1) + + qso_dist = Distribution.from_values(qso_values) + baseline_dist = Distribution.from_values(baseline_values) + + %{ + feature_name: name, + qso_distribution: qso_dist, + baseline_distribution: baseline_dist, + gate: if(qso_dist.count > 0, do: :pass, else: :no_data) + } + end) + |> Enum.sort_by(& &1.feature_name) + end + + @doc """ + Renders the output of `consolidated_report/2` as a single Markdown table. + """ + @spec to_consolidated_markdown([map()]) :: String.t() + def to_consolidated_markdown(results) when is_list(results) do + header = "| Feature | QSO N | QSO Mean | QSO p50 | Baseline N | Baseline Mean | Baseline p50 | Gate |\n" + sep = "|---|---|---|---|---|---|---|---|\n" + + body = + Enum.map_join(results, "", fn r -> + q = r.qso_distribution + b = r.baseline_distribution + gate_str = if r.gate == :pass, do: "PASS", else: "NO DATA" + + "| #{r.feature_name} | #{q.count} | #{fmt(q.mean)} | #{fmt(q.p50)} | #{b.count} | #{fmt(b.mean)} | #{fmt(b.p50)} | #{gate_str} |\n" + end) + + IO.iodata_to_binary([ + "# Consolidated Backtest Report\n\n", + header, + sep, + body, + "\n" + ]) + end + @doc """ Renders a `Report` (plus optional distance and band tables) as Markdown. diff --git a/lib/microwaveprop/backtest/features.ex b/lib/microwaveprop/backtest/features.ex index adfd06fb..30748a5b 100644 --- a/lib/microwaveprop/backtest/features.ex +++ b/lib/microwaveprop/backtest/features.ex @@ -25,6 +25,23 @@ defmodule Microwaveprop.Backtest.Features do alias Microwaveprop.Weather.HrrrClimatology alias Microwaveprop.Weather.HrrrNativeProfile + @doc """ + Returns a map of all backtestable features: `%{name => fun/3}`. + + Excludes `duct_usable_for_band/4` (4-arity) and the placeholder stubs + that always return nil (`distance_to_front`, `parallel_to_front`). + """ + def all_features do + excluded = MapSet.new([:duct_usable_for_band, :distance_to_front, :parallel_to_front]) + + __MODULE__.__info__(:functions) + |> Enum.filter(fn {_name, arity} -> arity == 3 end) + |> Enum.reject(fn {name, _} -> MapSet.member?(excluded, name) end) + |> Map.new(fn {name, _} -> + {to_string(name), &apply(__MODULE__, name, [&1, &2, &3])} + end) + end + @doc """ Minimum refractivity gradient from the nearest HRRR profile. diff --git a/lib/mix/tasks/backtest.ex b/lib/mix/tasks/backtest.ex index 54a09ec6..2ce05374 100644 --- a/lib/mix/tasks/backtest.ex +++ b/lib/mix/tasks/backtest.ex @@ -11,19 +11,20 @@ defmodule Mix.Tasks.Backtest do mix backtest --feature NaiveGradient # CamelCase also works mix backtest --feature Microwaveprop.Backtest.Features.naive_gradient mix backtest --feature naive_gradient --sample 1000 --out priv/backtest_reports/naive.md + mix backtest --all --out priv/backtest_reports/consolidated.md ## Options - * `--feature` (required) — fully-qualified `Module.function` or a - short name that lives on `Microwaveprop.Backtest.Features`. Names - are normalized via `Macro.underscore/1`, so `NaiveGradient`, - `naive_gradient`, and `naiveGradient` all resolve to the same - function. + * `--feature` — fully-qualified `Module.function` or a short name + that lives on `Microwaveprop.Backtest.Features`. Names are normalized + via `Macro.underscore/1`, so `NaiveGradient`, `naive_gradient`, and + `naiveGradient` all resolve to the same function. + * `--all` — run all registered features and produce a consolidated + pass/fail table. * `--sample` — max number of QSOs to evaluate (default: 5000). * `--baseline` — random-baseline sample size (default: same as `--sample`). * `--out` — optional file path to write the report to in addition - to printing it. Useful for saving baseline reports into - `priv/backtest_reports/`. + to printing it. """ use Mix.Task @@ -36,9 +37,41 @@ defmodule Mix.Tasks.Backtest do {opts, _, _} = OptionParser.parse(argv, - switches: [feature: :string, sample: :integer, baseline: :integer, out: :string] + switches: [feature: :string, all: :boolean, sample: :integer, baseline: :integer, out: :string] ) + if Keyword.get(opts, :all) do + run_all(opts) + else + run_single(opts) + end + end + + defp run_all(opts) do + sample_size = Keyword.get(opts, :sample, 5000) + baseline_size = Keyword.get(opts, :baseline, sample_size) + out_path = Keyword.get(opts, :out) + + features = Microwaveprop.Backtest.Features.all_features() + Mix.shell().info("Running consolidated backtest for #{map_size(features)} features...") + + results = + Backtest.consolidated_report(features, + sample_size: sample_size, + baseline_size: baseline_size + ) + + markdown = Backtest.to_consolidated_markdown(results) + IO.puts(markdown) + + if out_path do + File.mkdir_p!(Path.dirname(out_path)) + File.write!(out_path, markdown) + Mix.shell().info("Wrote consolidated report to #{out_path}") + end + end + + defp run_single(opts) do feature_spec = Keyword.fetch!(opts, :feature) sample_size = Keyword.get(opts, :sample, 5000) baseline_size = Keyword.get(opts, :baseline, sample_size) diff --git a/lib/mix/tasks/hrrr_climatology.ex b/lib/mix/tasks/hrrr_climatology.ex index 18fdeff3..553218df 100644 --- a/lib/mix/tasks/hrrr_climatology.ex +++ b/lib/mix/tasks/hrrr_climatology.ex @@ -4,8 +4,8 @@ defmodule Mix.Tasks.HrrrClimatology do Aggregates `hrrr_profiles.surface_temp_c` by (lat, lon, month, hour) into `hrrr_climatology` for use by the temperature-anomaly feature. - With 42M+ grid-point profiles, this runs a single SQL GROUP BY and - bulk-inserts the results. Idempotent via ON CONFLICT. + Discovers which (month, hour) combos have data first, then processes + only those batches. Idempotent via ON CONFLICT. mix hrrr_climatology # build from all grid-point profiles mix hrrr_climatology --min-samples 5 # require at least 5 observations per cell @@ -22,37 +22,61 @@ defmodule Mix.Tasks.HrrrClimatology do {opts, _, _} = OptionParser.parse(argv, switches: [min_samples: :integer]) min_samples = Keyword.get(opts, :min_samples, 3) - Mix.shell().info("Building climatology (min_samples=#{min_samples})...") - - # Single SQL: aggregate and upsert in one shot - {count, _} = + # Discover which (month, hour) combos actually have data + %{rows: combos} = Repo.query!( """ - INSERT INTO hrrr_climatology (id, lat, lon, month, hour, - mean_surface_temp_c, stddev_surface_temp_c, sample_count) - SELECT gen_random_uuid(), lat, lon, - EXTRACT(MONTH FROM valid_time)::int AS month, - EXTRACT(HOUR FROM valid_time)::int AS hour, - AVG(surface_temp_c), - STDDEV_SAMP(surface_temp_c), - COUNT(*) + SELECT EXTRACT(MONTH FROM valid_time)::int AS month, + EXTRACT(HOUR FROM valid_time)::int AS hour FROM hrrr_profiles - WHERE surface_temp_c IS NOT NULL - AND is_grid_point = true - GROUP BY lat, lon, - EXTRACT(MONTH FROM valid_time)::int, - EXTRACT(HOUR FROM valid_time)::int - HAVING COUNT(*) >= $1 - ON CONFLICT (lat, lon, month, hour) - DO UPDATE SET - mean_surface_temp_c = EXCLUDED.mean_surface_temp_c, - stddev_surface_temp_c = EXCLUDED.stddev_surface_temp_c, - sample_count = EXCLUDED.sample_count + WHERE surface_temp_c IS NOT NULL AND is_grid_point = true + GROUP BY 1, 2 + ORDER BY 1, 2 """, - [min_samples], - timeout: 600_000 + [], + timeout: 120_000 ) - Mix.shell().info("Upserted #{count} climatology records.") + Mix.shell().info( + "Building climatology (min_samples=#{min_samples}, #{length(combos)} batches)..." + ) + + total = + combos + |> Enum.with_index(1) + |> Enum.reduce(0, fn {[month, hour], idx}, acc -> + %{num_rows: count} = + Repo.query!( + """ + INSERT INTO hrrr_climatology (id, lat, lon, month, hour, + mean_surface_temp_c, stddev_surface_temp_c, sample_count) + SELECT gen_random_uuid(), lat, lon, + $2 AS month, + $3 AS hour, + AVG(surface_temp_c), + STDDEV_SAMP(surface_temp_c), + COUNT(*) + FROM hrrr_profiles + WHERE surface_temp_c IS NOT NULL + AND is_grid_point = true + AND EXTRACT(MONTH FROM valid_time)::int = $2 + AND EXTRACT(HOUR FROM valid_time)::int = $3 + GROUP BY lat, lon + HAVING COUNT(*) >= $1 + ON CONFLICT (lat, lon, month, hour) + DO UPDATE SET + mean_surface_temp_c = EXCLUDED.mean_surface_temp_c, + stddev_surface_temp_c = EXCLUDED.stddev_surface_temp_c, + sample_count = EXCLUDED.sample_count + """, + [min_samples, month, hour], + timeout: 300_000 + ) + + Mix.shell().info(" [#{idx}/#{length(combos)}] month=#{month} hour=#{hour}: #{count} rows") + acc + count + end) + + Mix.shell().info("Upserted #{total} climatology records total.") end end diff --git a/lib/mix/tasks/import_contest_logs.ex b/lib/mix/tasks/import_contest_logs.ex new file mode 100644 index 00000000..5c67c08c --- /dev/null +++ b/lib/mix/tasks/import_contest_logs.ex @@ -0,0 +1,99 @@ +defmodule Mix.Tasks.ImportContestLogs do + @shortdoc "Import ARRL contest CSV files with deduplication" + @moduledoc """ + Imports one or more contest CSV files, deduplicating both across files + and against existing contacts in the database. + + Uses the same direction-agnostic, 60-minute-window dedup logic as the + web CSV uploader. + + mix import_contest_logs ~/Downloads/arrl_10ghz_2024.csv ~/Downloads/arrl_10ghz_2023.csv ... + mix import_contest_logs ~/Downloads/arrl_10ghz_*.csv + """ + use Mix.Task + + alias Microwaveprop.Radio.CsvImport + + @impl Mix.Task + def run(argv) do + if argv == [] do + Mix.raise("Usage: mix import_contest_logs FILE [FILE ...]") + end + + Mix.Task.run("app.start") + Oban.pause_all_queues(Oban) + + # Concatenate all CSVs into one string (skip headers after the first file) + combined = + argv + |> Enum.with_index() + |> Enum.map(fn {path, idx} -> + expanded = Path.expand(path) + content = File.read!(expanded) + lines = String.split(content, ~r/\r?\n/) + + # Skip header for all files after the first + data_lines = if idx == 0, do: lines, else: tl(lines) + + # Drop blank trailing lines + Enum.reject(data_lines, &(String.trim(&1) == "")) + end) + |> List.flatten() + |> Enum.join("\n") + + total_lines = combined |> String.split("\n") |> length() |> then(&(&1 - 1)) + Mix.shell().info("Combined #{length(argv)} files: #{total_lines} data rows") + + Mix.shell().info("Running preview (validate + deduplicate)...") + + case CsvImport.preview(combined, "contest-import@ntms.org") do + {:ok, preview} -> + Mix.shell().info(""" + Preview: + Valid: #{length(preview.valid)} + Duplicates: #{length(preview.duplicates)} + Invalid: #{length(preview.invalid)} + """) + + if preview.invalid != [] do + preview.invalid + |> Enum.take(10) + |> Enum.each(fn row -> + Mix.shell().info(" Row #{row.row_num}: #{Enum.join(row.messages, ", ")}") + end) + + if length(preview.invalid) > 10 do + Mix.shell().info(" ... and #{length(preview.invalid) - 10} more") + end + end + + if preview.valid == [] do + Mix.shell().info("Nothing to import.") + else + Mix.shell().info("Importing #{length(preview.valid)} contacts...") + + # Import in batches to show progress + preview.valid + |> Enum.chunk_every(500) + |> Enum.with_index(1) + |> Enum.reduce({0, 0}, fn {batch, batch_num}, {imported, errors} -> + {:ok, result} = CsvImport.commit(batch) + new_imported = imported + length(result.imported) + new_errors = errors + length(result.errors) + + Mix.shell().info( + " Batch #{batch_num}: +#{length(result.imported)} imported, #{length(result.errors)} errors (total: #{new_imported})" + ) + + {new_imported, new_errors} + end) + |> then(fn {imported, errors} -> + Mix.shell().info("Done! Imported #{imported} contacts, #{errors} errors.") + end) + end + + {:error, reason} -> + Mix.raise("Failed: #{inspect(reason)}") + end + end +end diff --git a/priv/backtest_reports/baseline_naive_gradient.md b/priv/backtest_reports/baseline_naive_gradient.md index 9ef100a4..4aa46446 100644 --- a/priv/backtest_reports/baseline_naive_gradient.md +++ b/priv/backtest_reports/baseline_naive_gradient.md @@ -8,7 +8,7 @@ Baseline sample: 5000 | Set | N | Mean | Stddev | p50 | p90 | Min | Max | |---|---|---|---|---|---|---|---| | QSO times | 5000 | -113.971 | 42.207 | -105.154 | -69.043 | -331.397 | -35.521 | -| Random baseline | 151 | -118.002 | 43.405 | -107.528 | -72.860 | -268.079 | -41.047 | +| Random baseline | 140 | -112.877 | 51.502 | -100.413 | -62.855 | -331.397 | -48.393 | ## Lift by distance (km) diff --git a/priv/backtest_reports/best_duct_freq.md b/priv/backtest_reports/best_duct_freq.md new file mode 100644 index 00000000..e8fa99f3 --- /dev/null +++ b/priv/backtest_reports/best_duct_freq.md @@ -0,0 +1,27 @@ +# Backtest: Microwaveprop.Backtest.Features.best_duct_freq + +QSO sample: 5000 +Baseline sample: 5000 + +## Matched distribution + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| QSO times | 0 | — | — | — | — | — | — | +| Random baseline | 0 | — | — | — | — | — | — | + +## Lift by distance (km) + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| 0-100 | 0 | — | — | — | — | — | — | +| 100-250 | 0 | — | — | — | — | — | — | +| 250-500 | 0 | — | — | — | — | — | — | +| 500-1000 | 0 | — | — | — | — | — | — | +| 1000+ | 0 | — | — | — | — | — | — | + +## Lift by band + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| + diff --git a/priv/backtest_reports/bulk_richardson.md b/priv/backtest_reports/bulk_richardson.md new file mode 100644 index 00000000..933de6a8 --- /dev/null +++ b/priv/backtest_reports/bulk_richardson.md @@ -0,0 +1,27 @@ +# Backtest: Microwaveprop.Backtest.Features.bulk_richardson + +QSO sample: 5000 +Baseline sample: 5000 + +## Matched distribution + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| QSO times | 0 | — | — | — | — | — | — | +| Random baseline | 0 | — | — | — | — | — | — | + +## Lift by distance (km) + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| 0-100 | 0 | — | — | — | — | — | — | +| 100-250 | 0 | — | — | — | — | — | — | +| 250-500 | 0 | — | — | — | — | — | — | +| 500-1000 | 0 | — | — | — | — | — | — | +| 1000+ | 0 | — | — | — | — | — | — | + +## Lift by band + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| + diff --git a/priv/backtest_reports/duct_thickness.md b/priv/backtest_reports/duct_thickness.md new file mode 100644 index 00000000..7972560b --- /dev/null +++ b/priv/backtest_reports/duct_thickness.md @@ -0,0 +1,27 @@ +# Backtest: Microwaveprop.Backtest.Features.duct_thickness + +QSO sample: 5000 +Baseline sample: 5000 + +## Matched distribution + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| QSO times | 0 | — | — | — | — | — | — | +| Random baseline | 0 | — | — | — | — | — | — | + +## Lift by distance (km) + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| 0-100 | 0 | — | — | — | — | — | — | +| 100-250 | 0 | — | — | — | — | — | — | +| 250-500 | 0 | — | — | — | — | — | — | +| 500-1000 | 0 | — | — | — | — | — | — | +| 1000+ | 0 | — | — | — | — | — | — | + +## Lift by band + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| + diff --git a/priv/backtest_reports/duct_usable_10ghz.md b/priv/backtest_reports/duct_usable_10ghz.md new file mode 100644 index 00000000..c3025488 --- /dev/null +++ b/priv/backtest_reports/duct_usable_10ghz.md @@ -0,0 +1,27 @@ +# Backtest: Microwaveprop.Backtest.Features.duct_usable_10ghz + +QSO sample: 5000 +Baseline sample: 5000 + +## Matched distribution + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| QSO times | 0 | — | — | — | — | — | — | +| Random baseline | 0 | — | — | — | — | — | — | + +## Lift by distance (km) + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| 0-100 | 0 | — | — | — | — | — | — | +| 100-250 | 0 | — | — | — | — | — | — | +| 250-500 | 0 | — | — | — | — | — | — | +| 500-1000 | 0 | — | — | — | — | — | — | +| 1000+ | 0 | — | — | — | — | — | — | + +## Lift by band + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| + diff --git a/priv/backtest_reports/native_surface_refractivity.md b/priv/backtest_reports/native_surface_refractivity.md new file mode 100644 index 00000000..72373097 --- /dev/null +++ b/priv/backtest_reports/native_surface_refractivity.md @@ -0,0 +1,27 @@ +# Backtest: Microwaveprop.Backtest.Features.native_surface_refractivity + +QSO sample: 5000 +Baseline sample: 5000 + +## Matched distribution + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| QSO times | 0 | — | — | — | — | — | — | +| Random baseline | 0 | — | — | — | — | — | — | + +## Lift by distance (km) + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| 0-100 | 0 | — | — | — | — | — | — | +| 100-250 | 0 | — | — | — | — | — | — | +| 250-500 | 0 | — | — | — | — | — | — | +| 500-1000 | 0 | — | — | — | — | — | — | +| 1000+ | 0 | — | — | — | — | — | — | + +## Lift by band + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| + diff --git a/priv/backtest_reports/shear_at_top.md b/priv/backtest_reports/shear_at_top.md new file mode 100644 index 00000000..2dd1d400 --- /dev/null +++ b/priv/backtest_reports/shear_at_top.md @@ -0,0 +1,27 @@ +# Backtest: Microwaveprop.Backtest.Features.shear_at_top + +QSO sample: 5000 +Baseline sample: 5000 + +## Matched distribution + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| QSO times | 0 | — | — | — | — | — | — | +| Random baseline | 0 | — | — | — | — | — | — | + +## Lift by distance (km) + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| 0-100 | 0 | — | — | — | — | — | — | +| 100-250 | 0 | — | — | — | — | — | — | +| 250-500 | 0 | — | — | — | — | — | — | +| 500-1000 | 0 | — | — | — | — | — | — | +| 1000+ | 0 | — | — | — | — | — | — | + +## Lift by band + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| + diff --git a/priv/backtest_reports/temperature_anomaly.md b/priv/backtest_reports/temperature_anomaly.md new file mode 100644 index 00000000..6723f520 --- /dev/null +++ b/priv/backtest_reports/temperature_anomaly.md @@ -0,0 +1,31 @@ +# Backtest: Microwaveprop.Backtest.Features.temperature_anomaly + +QSO sample: 5000 +Baseline sample: 5000 + +## Matched distribution + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| QSO times | 4974 | 1.118 | 2.624 | 1.402 | 4.175 | -8.127 | 9.335 | +| Random baseline | 128 | 0.332 | 2.840 | 0.581 | 3.714 | -6.600 | 7.206 | + +## Lift by distance (km) + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| 0-100 | 1233 | 0.570 | 2.423 | 0.828 | 3.469 | -8.127 | 9.264 | +| 100-250 | 2285 | 1.088 | 2.519 | 1.467 | 3.941 | -7.423 | 9.264 | +| 250-500 | 1288 | 1.596 | 2.913 | 1.701 | 5.037 | -7.706 | 9.335 | +| 500-1000 | 167 | 1.872 | 2.260 | 2.047 | 4.398 | -7.706 | 6.211 | +| 1000+ | 1 | 2.311 | 0.000 | 2.311 | 2.311 | 2.311 | 2.311 | + +## Lift by band + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| 10000 MHz | 4444 | 1.145 | 2.648 | 1.454 | 4.206 | -8.127 | 9.335 | +| 24000 MHz | 465 | 0.840 | 2.426 | 1.247 | 3.678 | -6.932 | 6.951 | +| 47000 MHz | 51 | 0.873 | 2.315 | 1.396 | 4.056 | -6.932 | 4.248 | +| 75000 MHz | 14 | 2.374 | 1.534 | 1.317 | 4.056 | 0.943 | 4.056 | + diff --git a/priv/backtest_reports/theta_e_jump.md b/priv/backtest_reports/theta_e_jump.md new file mode 100644 index 00000000..ef387bd9 --- /dev/null +++ b/priv/backtest_reports/theta_e_jump.md @@ -0,0 +1,27 @@ +# Backtest: Microwaveprop.Backtest.Features.theta_e_jump + +QSO sample: 5000 +Baseline sample: 5000 + +## Matched distribution + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| QSO times | 0 | — | — | — | — | — | — | +| Random baseline | 0 | — | — | — | — | — | — | + +## Lift by distance (km) + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| +| 0-100 | 0 | — | — | — | — | — | — | +| 100-250 | 0 | — | — | — | — | — | — | +| 250-500 | 0 | — | — | — | — | — | — | +| 500-1000 | 0 | — | — | — | — | — | — | +| 1000+ | 0 | — | — | — | — | — | — | + +## Lift by band + +| Set | N | Mean | Stddev | p50 | p90 | Min | Max | +|---|---|---|---|---|---|---|---| + diff --git a/test/microwaveprop/backtest_test.exs b/test/microwaveprop/backtest_test.exs index 166ea09e..716e19dc 100644 --- a/test/microwaveprop/backtest_test.exs +++ b/test/microwaveprop/backtest_test.exs @@ -138,6 +138,51 @@ defmodule Microwaveprop.BacktestTest do end end + describe "consolidated_report/1" do + test "runs all registered features and returns a summary per feature" do + create_contact(%{station1: "A", pos1: %{"lat" => 32.0, "lon" => -97.0}}) + create_contact(%{station1: "B", pos1: %{"lat" => 33.0, "lon" => -96.0}}) + + features = %{ + "always_one" => fn _lat, _lon, _time -> 1.0 end, + "always_nil" => fn _lat, _lon, _time -> nil end, + "uses_lat" => fn lat, _lon, _time -> lat end + } + + results = Backtest.consolidated_report(features, sample_size: 10, baseline_size: 5) + + assert is_list(results) + assert length(results) == 3 + + one = Enum.find(results, &(&1.feature_name == "always_one")) + assert one.qso_distribution.count == 2 + assert one.qso_distribution.mean == 1.0 + + nil_feat = Enum.find(results, &(&1.feature_name == "always_nil")) + assert nil_feat.qso_distribution.count == 0 + + lat_feat = Enum.find(results, &(&1.feature_name == "uses_lat")) + assert lat_feat.qso_distribution.count == 2 + end + + test "to_consolidated_markdown renders a summary table" do + create_contact(%{station1: "A", pos1: %{"lat" => 32.0, "lon" => -97.0}}) + + features = %{ + "feat_a" => fn _lat, _lon, _time -> 5.0 end, + "feat_b" => fn _lat, _lon, _time -> nil end + } + + results = Backtest.consolidated_report(features, sample_size: 10, baseline_size: 5) + markdown = Backtest.to_consolidated_markdown(results) + + assert markdown =~ "feat_a" + assert markdown =~ "feat_b" + assert markdown =~ "Consolidated" + assert markdown =~ "QSO N" + end + end + describe "lift_by_band/2" do test "groups feature values by band and summarizes per band" do create_contact(%{