prop/lib/mix/tasks/backtest.ex

defmodule Mix.Tasks.Backtest do
  @shortdoc "Evaluate a propagation feature against the QSO corpus"
  @moduledoc """
  Runs `Microwaveprop.Backtest.evaluate/2` (plus the distance and band
  breakdowns) for a named feature function and prints a Markdown report
  to stdout.

  ## Usage

      mix backtest --feature naive_gradient
      mix backtest --feature NaiveGradient            # CamelCase also works
      mix backtest --feature Microwaveprop.Backtest.Features.naive_gradient
      mix backtest --feature naive_gradient --sample 1000 --out priv/backtest_reports/naive.md
      mix backtest --all --out priv/backtest_reports/consolidated.md

  ## Options

    * `--feature` — fully-qualified `Module.function` or a short name
      that lives on `Microwaveprop.Backtest.Features`. Names are normalized
      via `Macro.underscore/1`, so `NaiveGradient`, `naive_gradient`, and
      `naiveGradient` all resolve to the same function.
    * `--all` — run all registered features and produce a consolidated
      pass/fail table.
    * `--sample` — max number of QSOs to evaluate (default: 5000).
    * `--baseline` — random-baseline sample size (default: same as `--sample`).
    * `--out` — optional file path to write the report to in addition
      to printing it.
  """
  use Mix.Task

  alias Microwaveprop.Backtest
  alias Microwaveprop.Backtest.Features

  @impl Mix.Task
  def run(argv) do
    Mix.Task.run("app.start")
    Oban.pause_all_queues(Oban)

    {opts, _, _} =
      OptionParser.parse(argv,
        switches: [feature: :string, all: :boolean, sample: :integer, baseline: :integer, out: :string]
      )

    if Keyword.get(opts, :all) do
      run_all(opts)
    else
      run_single(opts)
    end
  end

  defp run_all(opts) do
    sample_size = Keyword.get(opts, :sample, 5000)
    baseline_size = Keyword.get(opts, :baseline, sample_size)
    out_path = Keyword.get(opts, :out)

    features = Features.all_features()
    Mix.shell().info("Running consolidated backtest for #{map_size(features)} features...")

    results =
      Backtest.consolidated_report(features,
        sample_size: sample_size,
        baseline_size: baseline_size
      )

    markdown = Backtest.to_consolidated_markdown(results)
    IO.puts(markdown)

    if out_path do
      File.mkdir_p!(Path.dirname(out_path))
      File.write!(out_path, markdown)
      Mix.shell().info("Wrote consolidated report to #{out_path}")
    end
  end

  defp run_single(opts) do
    feature_spec = Keyword.fetch!(opts, :feature)
    sample_size = Keyword.get(opts, :sample, 5000)
    baseline_size = Keyword.get(opts, :baseline, sample_size)
    out_path = Keyword.get(opts, :out)

    {feature_fun, feature_name} = resolve_feature(feature_spec)

    report =
      Backtest.evaluate(feature_fun,
        sample_size: sample_size,
        baseline_size: baseline_size,
        feature_name: feature_name
      )

    distance_bins = Backtest.lift_by_distance(feature_fun, sample_size: sample_size)
    band_stats = Backtest.lift_by_band(feature_fun, sample_size: sample_size)

    markdown =
      Backtest.to_markdown(report, distance_bins: distance_bins, band_stats: band_stats)

    IO.puts(markdown)

    if out_path do
      File.mkdir_p!(Path.dirname(out_path))
      File.write!(out_path, markdown)
      Mix.shell().info("Wrote report to #{out_path}")
    end
  end

  defp resolve_feature(spec) do
    case String.split(spec, ".") do
      [name] ->
        fun = resolve_function_atom!(Features, name)
        feature_fun = &apply(Features, fun, [&1, &2, &3])
        {feature_fun, "Microwaveprop.Backtest.Features.#{fun}"}

      parts ->
        {fun_name, mod_parts} = List.pop_at(parts, -1)
        module = Module.concat(mod_parts)
        fun = resolve_function_atom!(module, fun_name)
        feature_fun = &apply(module, fun, [&1, &2, &3])
        {feature_fun, "#{inspect(module)}.#{fun}"}
    end
  end

  # Accept both `naive_gradient` and `NaiveGradient` and anything in between.
  defp resolve_function_atom!(module, name) do
    Code.ensure_loaded!(module)
    normalized = Macro.underscore(name)
    fun = String.to_atom(normalized)

    if function_exported?(module, fun, 3) do
      fun
    else
      exported =
        :functions
        |> module.__info__()
        |> Enum.filter(fn {_f, arity} -> arity == 3 end)
        |> Enum.map_join(", ", fn {f, _} -> to_string(f) end)

      Mix.raise("Feature #{inspect(module)}.#{normalized}/3 is not defined.\nAvailable 3-arity functions: #{exported}")
    end
  end
end