Graham McIntire
7630934bcb
mix backtest --feature NaiveGradient now resolves to the same function as --feature naive_gradient. Names are normalized via Macro.underscore, so any casing works. Also prints the available functions when an unknown feature is requested so typos don't produce an opaque UndefinedFunctionError deep in the Enum.map stack.
105 lines
3.5 KiB
Elixir
105 lines
3.5 KiB
Elixir
defmodule Mix.Tasks.Backtest do
|
|
@shortdoc "Evaluate a propagation feature against the QSO corpus"
|
|
@moduledoc """
|
|
Runs `Microwaveprop.Backtest.evaluate/2` (plus the distance and band
|
|
breakdowns) for a named feature function and prints a Markdown report
|
|
to stdout.
|
|
|
|
## Usage
|
|
|
|
mix backtest --feature naive_gradient
|
|
mix backtest --feature NaiveGradient # CamelCase also works
|
|
mix backtest --feature Microwaveprop.Backtest.Features.naive_gradient
|
|
mix backtest --feature naive_gradient --sample 1000 --out priv/backtest_reports/naive.md
|
|
|
|
## Options
|
|
|
|
* `--feature` (required) — fully-qualified `Module.function` or a
|
|
short name that lives on `Microwaveprop.Backtest.Features`. Names
|
|
are normalized via `Macro.underscore/1`, so `NaiveGradient`,
|
|
`naive_gradient`, and `naiveGradient` all resolve to the same
|
|
function.
|
|
* `--sample` — max number of QSOs to evaluate (default: 5000).
|
|
* `--baseline` — random-baseline sample size (default: same as `--sample`).
|
|
* `--out` — optional file path to write the report to in addition
|
|
to printing it. Useful for saving baseline reports into
|
|
`priv/backtest_reports/`.
|
|
"""
|
|
use Mix.Task
|
|
|
|
alias Microwaveprop.Backtest
|
|
|
|
@impl Mix.Task
|
|
def run(argv) do
|
|
Mix.Task.run("app.start")
|
|
|
|
{opts, _, _} =
|
|
OptionParser.parse(argv,
|
|
switches: [feature: :string, sample: :integer, baseline: :integer, out: :string]
|
|
)
|
|
|
|
feature_spec = Keyword.fetch!(opts, :feature)
|
|
sample_size = Keyword.get(opts, :sample, 5000)
|
|
baseline_size = Keyword.get(opts, :baseline, sample_size)
|
|
out_path = Keyword.get(opts, :out)
|
|
|
|
{feature_fun, feature_name} = resolve_feature(feature_spec)
|
|
|
|
report =
|
|
Backtest.evaluate(feature_fun,
|
|
sample_size: sample_size,
|
|
baseline_size: baseline_size,
|
|
feature_name: feature_name
|
|
)
|
|
|
|
distance_bins = Backtest.lift_by_distance(feature_fun, sample_size: sample_size)
|
|
band_stats = Backtest.lift_by_band(feature_fun, sample_size: sample_size)
|
|
|
|
markdown =
|
|
Backtest.to_markdown(report, distance_bins: distance_bins, band_stats: band_stats)
|
|
|
|
IO.puts(markdown)
|
|
|
|
if out_path do
|
|
File.mkdir_p!(Path.dirname(out_path))
|
|
File.write!(out_path, markdown)
|
|
Mix.shell().info("Wrote report to #{out_path}")
|
|
end
|
|
end
|
|
|
|
defp resolve_feature(spec) do
|
|
case String.split(spec, ".") do
|
|
[name] ->
|
|
fun = resolve_function_atom!(Microwaveprop.Backtest.Features, name)
|
|
feature_fun = &apply(Microwaveprop.Backtest.Features, fun, [&1, &2, &3])
|
|
{feature_fun, "Microwaveprop.Backtest.Features.#{fun}"}
|
|
|
|
parts ->
|
|
{fun_name, mod_parts} = List.pop_at(parts, -1)
|
|
module = Module.concat(mod_parts)
|
|
fun = resolve_function_atom!(module, fun_name)
|
|
feature_fun = &apply(module, fun, [&1, &2, &3])
|
|
{feature_fun, "#{inspect(module)}.#{fun}"}
|
|
end
|
|
end
|
|
|
|
# Accept both `naive_gradient` and `NaiveGradient` and anything in between.
|
|
defp resolve_function_atom!(module, name) do
|
|
Code.ensure_loaded!(module)
|
|
normalized = Macro.underscore(name)
|
|
fun = String.to_atom(normalized)
|
|
|
|
if function_exported?(module, fun, 3) do
|
|
fun
|
|
else
|
|
exported =
|
|
module.__info__(:functions)
|
|
|> Enum.filter(fn {_f, arity} -> arity == 3 end)
|
|
|> Enum.map_join(", ", fn {f, _} -> to_string(f) end)
|
|
|
|
Mix.raise(
|
|
"Feature #{inspect(module)}.#{normalized}/3 is not defined.\nAvailable 3-arity functions: #{exported}"
|
|
)
|
|
end
|
|
end
|
|
end
|