Graham McIntire
ab04cb9168
Add Microwaveprop.Backtest: a feature-evaluation framework that runs a (lat, lon, valid_time) -> float function over the historical QSO corpus and a matched random-time baseline, reporting distribution statistics, distance-binned lift, and band-stratified lift. Adds four baseline feature wrappers around the current scorer inputs (naive_gradient, td_depression, time_of_day, pressure), a mix backtest CLI, and the first set of baseline reports under priv/backtest_reports so downstream phases have a frozen reference point to compare against.
81 lines
2.6 KiB
Elixir
81 lines
2.6 KiB
Elixir
defmodule Mix.Tasks.Backtest do
|
|
@shortdoc "Evaluate a propagation feature against the QSO corpus"
|
|
@moduledoc """
|
|
Runs `Microwaveprop.Backtest.evaluate/2` (plus the distance and band
|
|
breakdowns) for a named feature function and prints a Markdown report
|
|
to stdout.
|
|
|
|
## Usage
|
|
|
|
mix backtest --feature Microwaveprop.Backtest.Features.naive_gradient
|
|
mix backtest --feature naive_gradient
|
|
mix backtest --feature naive_gradient --sample 1000 --out priv/backtest_reports/naive.md
|
|
|
|
## Options
|
|
|
|
* `--feature` (required) — fully-qualified `Module.function` or a
|
|
short name that lives on `Microwaveprop.Backtest.Features`.
|
|
* `--sample` — max number of QSOs to evaluate (default: 5000).
|
|
* `--baseline` — random-baseline sample size (default: same as `--sample`).
|
|
* `--out` — optional file path to write the report to in addition
|
|
to printing it. Useful for saving baseline reports into
|
|
`priv/backtest_reports/`.
|
|
"""
|
|
use Mix.Task
|
|
|
|
alias Microwaveprop.Backtest
|
|
|
|
@impl Mix.Task
|
|
def run(argv) do
|
|
Mix.Task.run("app.start")
|
|
|
|
{opts, _, _} =
|
|
OptionParser.parse(argv,
|
|
switches: [feature: :string, sample: :integer, baseline: :integer, out: :string]
|
|
)
|
|
|
|
feature_spec = Keyword.fetch!(opts, :feature)
|
|
sample_size = Keyword.get(opts, :sample, 5000)
|
|
baseline_size = Keyword.get(opts, :baseline, sample_size)
|
|
out_path = Keyword.get(opts, :out)
|
|
|
|
{feature_fun, feature_name} = resolve_feature(feature_spec)
|
|
|
|
report =
|
|
Backtest.evaluate(feature_fun,
|
|
sample_size: sample_size,
|
|
baseline_size: baseline_size,
|
|
feature_name: feature_name
|
|
)
|
|
|
|
distance_bins = Backtest.lift_by_distance(feature_fun, sample_size: sample_size)
|
|
band_stats = Backtest.lift_by_band(feature_fun, sample_size: sample_size)
|
|
|
|
markdown =
|
|
Backtest.to_markdown(report, distance_bins: distance_bins, band_stats: band_stats)
|
|
|
|
IO.puts(markdown)
|
|
|
|
if out_path do
|
|
File.mkdir_p!(Path.dirname(out_path))
|
|
File.write!(out_path, markdown)
|
|
Mix.shell().info("Wrote report to #{out_path}")
|
|
end
|
|
end
|
|
|
|
defp resolve_feature(spec) do
|
|
case String.split(spec, ".") do
|
|
[name] ->
|
|
fun = String.to_atom(name)
|
|
feature_fun = &apply(Microwaveprop.Backtest.Features, fun, [&1, &2, &3])
|
|
{feature_fun, "Microwaveprop.Backtest.Features.#{name}"}
|
|
|
|
parts ->
|
|
{fun_name, mod_parts} = List.pop_at(parts, -1)
|
|
module = Module.concat(mod_parts)
|
|
fun = String.to_atom(fun_name)
|
|
feature_fun = &apply(module, fun, [&1, &2, &3])
|
|
{feature_fun, Enum.join(parts, ".")}
|
|
end
|
|
end
|
|
end
|