Graham McIntire
65e3159a85
Mix tasks that call app.start were also booting Oban's cron scheduler, causing PropagationGridWorker and other cron jobs to fire during backfills. Add Oban.pause_all_queues(Oban) immediately after app.start in every mix task that only needs Repo access.
106 lines
3.5 KiB
Elixir
106 lines
3.5 KiB
Elixir
defmodule Mix.Tasks.Backtest do
|
|
@shortdoc "Evaluate a propagation feature against the QSO corpus"
|
|
@moduledoc """
|
|
Runs `Microwaveprop.Backtest.evaluate/2` (plus the distance and band
|
|
breakdowns) for a named feature function and prints a Markdown report
|
|
to stdout.
|
|
|
|
## Usage
|
|
|
|
mix backtest --feature naive_gradient
|
|
mix backtest --feature NaiveGradient # CamelCase also works
|
|
mix backtest --feature Microwaveprop.Backtest.Features.naive_gradient
|
|
mix backtest --feature naive_gradient --sample 1000 --out priv/backtest_reports/naive.md
|
|
|
|
## Options
|
|
|
|
* `--feature` (required) — fully-qualified `Module.function` or a
|
|
short name that lives on `Microwaveprop.Backtest.Features`. Names
|
|
are normalized via `Macro.underscore/1`, so `NaiveGradient`,
|
|
`naive_gradient`, and `naiveGradient` all resolve to the same
|
|
function.
|
|
* `--sample` — max number of QSOs to evaluate (default: 5000).
|
|
* `--baseline` — random-baseline sample size (default: same as `--sample`).
|
|
* `--out` — optional file path to write the report to in addition
|
|
to printing it. Useful for saving baseline reports into
|
|
`priv/backtest_reports/`.
|
|
"""
|
|
use Mix.Task
|
|
|
|
alias Microwaveprop.Backtest
|
|
|
|
@impl Mix.Task
|
|
def run(argv) do
|
|
Mix.Task.run("app.start")
|
|
Oban.pause_all_queues(Oban)
|
|
|
|
{opts, _, _} =
|
|
OptionParser.parse(argv,
|
|
switches: [feature: :string, sample: :integer, baseline: :integer, out: :string]
|
|
)
|
|
|
|
feature_spec = Keyword.fetch!(opts, :feature)
|
|
sample_size = Keyword.get(opts, :sample, 5000)
|
|
baseline_size = Keyword.get(opts, :baseline, sample_size)
|
|
out_path = Keyword.get(opts, :out)
|
|
|
|
{feature_fun, feature_name} = resolve_feature(feature_spec)
|
|
|
|
report =
|
|
Backtest.evaluate(feature_fun,
|
|
sample_size: sample_size,
|
|
baseline_size: baseline_size,
|
|
feature_name: feature_name
|
|
)
|
|
|
|
distance_bins = Backtest.lift_by_distance(feature_fun, sample_size: sample_size)
|
|
band_stats = Backtest.lift_by_band(feature_fun, sample_size: sample_size)
|
|
|
|
markdown =
|
|
Backtest.to_markdown(report, distance_bins: distance_bins, band_stats: band_stats)
|
|
|
|
IO.puts(markdown)
|
|
|
|
if out_path do
|
|
File.mkdir_p!(Path.dirname(out_path))
|
|
File.write!(out_path, markdown)
|
|
Mix.shell().info("Wrote report to #{out_path}")
|
|
end
|
|
end
|
|
|
|
defp resolve_feature(spec) do
|
|
case String.split(spec, ".") do
|
|
[name] ->
|
|
fun = resolve_function_atom!(Microwaveprop.Backtest.Features, name)
|
|
feature_fun = &apply(Microwaveprop.Backtest.Features, fun, [&1, &2, &3])
|
|
{feature_fun, "Microwaveprop.Backtest.Features.#{fun}"}
|
|
|
|
parts ->
|
|
{fun_name, mod_parts} = List.pop_at(parts, -1)
|
|
module = Module.concat(mod_parts)
|
|
fun = resolve_function_atom!(module, fun_name)
|
|
feature_fun = &apply(module, fun, [&1, &2, &3])
|
|
{feature_fun, "#{inspect(module)}.#{fun}"}
|
|
end
|
|
end
|
|
|
|
# Accept both `naive_gradient` and `NaiveGradient` and anything in between.
|
|
defp resolve_function_atom!(module, name) do
|
|
Code.ensure_loaded!(module)
|
|
normalized = Macro.underscore(name)
|
|
fun = String.to_atom(normalized)
|
|
|
|
if function_exported?(module, fun, 3) do
|
|
fun
|
|
else
|
|
exported =
|
|
module.__info__(:functions)
|
|
|> Enum.filter(fn {_f, arity} -> arity == 3 end)
|
|
|> Enum.map_join(", ", fn {f, _} -> to_string(f) end)
|
|
|
|
Mix.raise(
|
|
"Feature #{inspect(module)}.#{normalized}/3 is not defined.\nAvailable 3-arity functions: #{exported}"
|
|
)
|
|
end
|
|
end
|
|
end
|