prop/lib/microwaveprop/pskr/recalibrator.ex

defmodule Microwaveprop.Pskr.Recalibrator do
  @moduledoc """
  PSKR-driven analysis pipeline. Reads `pskr_calibration_samples`,
  bins each sample by every scoring feature × band, and writes per-
  (run, band, feature, bin) spot-density stats to
  `pskr_feature_bins`. The operator queries the bin tables to see
  whether each feature actually discriminates spot density at the
  threshold granularity the scorer uses.

  Auto-applies *nothing* to the live algorithm. Weight changes still
  go through human review of `BandConfig.@band_configs` and a code
  commit. The recalibrator's job is to surface evidence; it stays
  out of the production scoring path.

  ## Why bins, not regression

  `BandConfig` already encodes scoring as discrete thresholds (e.g.
  refractivity gradient at -500 / -300 / -200 / -100). The bin
  output matches that shape so an operator can read "PWAT 25-40 mm
  on 10 GHz had n=4,200 cells with avg 18 spots, vs PWAT >55 mm
  with n=620 and avg 4 spots" and decide directly whether a
  threshold needs to move. A continuous regression would obscure
  the threshold structure that the scorer needs to consume.

  ## Self-healing

  `run/0` is idempotent at the run level — every fire creates a
  fresh `pskr_recalibration_runs` row. Fresh runs replace stale
  ones at query time (operators look at the latest). Mid-run
  failure logs a `failed` status and surfaces the exception in
  `notes` so the next fire can retry without manual cleanup.
  """

  import Ecto.Query

  alias Microwaveprop.Pskr.CalibrationSample
  alias Microwaveprop.Pskr.FeatureBin
  alias Microwaveprop.Pskr.RecalibrationRun
  alias Microwaveprop.Repo

  require Logger

  # Minimum corpus size to attempt analysis. Below this the bin
  # statistics are so noisy they're misleading; better to log a
  # `skipped_insufficient_data` run and wait. ~1000 samples ≈
  # 10 cells × 100 hours of CONUS PSKR activity, achievable in 4-5
  # days even with sparse traffic.
  @min_total_samples 1_000

  # Per-band threshold for emitting bins for that band. With < this
  # many samples on a band, the per-bin counts collapse to single
  # digits and the analysis tells you nothing.
  @min_band_samples 100

  # Feature → bin definitions. Each list is `{label, min, max}` with
  # `min`/`max` as inclusive lower / exclusive upper. nil sides
  # mean unbounded. Bins are ordered for human-readable output but
  # SQL returns them set-style; the worker re-sorts on read if
  # needed.
  @feature_bins %{
    pwat_mm: [
      {"<15", nil, 15.0},
      {"15-25", 15.0, 25.0},
      {"25-40", 25.0, 40.0},
      {"40-55", 40.0, 55.0},
      {">55", 55.0, nil}
    ],
    hpbl_m: [
      {"<250", nil, 250.0},
      {"250-500", 250.0, 500.0},
      {"500-1000", 500.0, 1000.0},
      {"1000-1500", 1000.0, 1500.0},
      {">1500", 1500.0, nil}
    ],
    min_refractivity_gradient: [
      {"<-200", nil, -200.0},
      {"-200..-100", -200.0, -100.0},
      {"-100..-50", -100.0, -50.0},
      {">-50", -50.0, nil}
    ],
    surface_pressure_mb: [
      {"<1010", nil, 1010.0},
      {"1010-1015", 1010.0, 1015.0},
      {"1015-1020", 1015.0, 1020.0},
      {">1020", 1020.0, nil}
    ],
    kp_index: [
      {"0-2 quiet", nil, 3.0},
      {"3-4 unsettled", 3.0, 5.0},
      {"5+ storm", 5.0, nil}
    ]
  }

  @doc """
  Run the analysis end-to-end. Returns the `RecalibrationRun` row
  written. Safe to call from a worker, an iex shell, or a test —
  always inserts a run record so the audit trail is complete.
  """
  @spec run() :: RecalibrationRun.t()
  def run do
    now = DateTime.truncate(DateTime.utc_now(), :second)
    stats = corpus_stats()

    if stats.sample_count < @min_total_samples do
      insert_skipped_run(now, stats, "corpus has #{stats.sample_count} samples, need ≥ #{@min_total_samples}")
    else
      run_analysis(now, stats)
    end
  end

  # ── corpus stats ────────────────────────────────────────────────

  defp corpus_stats do
    %{rows: [[total, bands, earliest, latest, avg_spots]]} =
      Repo.query!(
        """
        SELECT
          count(*)::int,
          count(DISTINCT band)::int,
          min(hour_utc),
          max(hour_utc),
          avg(spot_count)::float
        FROM pskr_calibration_samples
        """,
        []
      )

    %{
      sample_count: total || 0,
      band_count: bands || 0,
      earliest_sample: cast_to_utc(earliest),
      latest_sample: cast_to_utc(latest),
      avg_spots_per_sample: avg_spots
    }
  end

  defp cast_to_utc(%NaiveDateTime{} = ndt), do: DateTime.from_naive!(ndt, "Etc/UTC")
  defp cast_to_utc(nil), do: nil
  defp cast_to_utc(%DateTime{} = dt), do: dt

  # ── skipped run ────────────────────────────────────────────────

  defp insert_skipped_run(now, stats, note) do
    Logger.info("Pskr.Recalibrator: skipping — #{note}")

    {:ok, run} =
      %RecalibrationRun{}
      |> RecalibrationRun.changeset(
        Map.merge(stats, %{
          run_at: now,
          status: "skipped_insufficient_data",
          notes: note
        })
      )
      |> Repo.insert()

    run
  end

  # ── analysis run ───────────────────────────────────────────────

  defp run_analysis(now, stats) do
    Logger.info("Pskr.Recalibrator: starting analysis on #{stats.sample_count} samples across #{stats.band_count} bands")

    {:ok, run} =
      %RecalibrationRun{}
      |> RecalibrationRun.changeset(
        Map.merge(stats, %{
          run_at: now,
          status: "completed"
        })
      )
      |> Repo.insert()

    bin_rows = build_bin_rows(run.id)

    if bin_rows != [] do
      now_seconds = DateTime.truncate(DateTime.utc_now(), :second)

      timestamped =
        Enum.map(bin_rows, &Map.merge(&1, %{inserted_at: now_seconds, updated_at: now_seconds}))

      {count, _} = Repo.insert_all(FeatureBin, timestamped)
      Logger.info("Pskr.Recalibrator: wrote #{count} feature_bins for run #{run.id}")
    end

    Repo.preload(run, :feature_bins)
  end

  # ── bin generation ─────────────────────────────────────────────

  defp build_bin_rows(run_id) do
    eligible_bands = eligible_bands_for_run()

    for band <- eligible_bands,
        {feature, defs} <- @feature_bins,
        {label, lo, hi} <- defs,
        row = bin_for(run_id, band, feature, label, lo, hi),
        not is_nil(row) do
      row
    end
  end

  defp eligible_bands_for_run do
    Repo.all(
      from(s in CalibrationSample,
        group_by: s.band,
        having: count(s.id) >= ^@min_band_samples,
        select: s.band
      )
    )
  end

  defp bin_for(run_id, band, feature, label, lo, hi) do
    %{rows: rows} =
      Repo.query!(bin_sql(feature), bin_params(band, lo, hi))

    [[count, total, avg, p50, p90]] = rows

    if count == 0 do
      nil
    else
      %{
        id: Ecto.UUID.generate(),
        run_id: run_id,
        band: band,
        feature: Atom.to_string(feature),
        bin_label: label,
        bin_min: lo,
        bin_max: hi,
        sample_count: count,
        spot_count_total: total || 0,
        spot_count_avg: avg,
        spot_count_p50: p50,
        spot_count_p90: p90
      }
    end
  end

  # Per-feature SQL: count samples that have a non-null value of the
  # feature within the bin range, plus spot-count statistics. Built
  # at compile time per feature so we don't take an Ecto fragment
  # interpolation hit on the hot path.
  for feature <- Map.keys(@feature_bins) do
    col = Atom.to_string(feature)

    defp bin_sql(unquote(feature)) do
      """
      SELECT
        count(*)::int,
        sum(spot_count)::bigint,
        avg(spot_count)::float,
        percentile_cont(0.5) WITHIN GROUP (ORDER BY spot_count)::float,
        percentile_cont(0.9) WITHIN GROUP (ORDER BY spot_count)::float
      FROM pskr_calibration_samples
      WHERE band = $1
        AND #{unquote(col)} IS NOT NULL
        AND ($2::float IS NULL OR #{unquote(col)} >= $2)
        AND ($3::float IS NULL OR #{unquote(col)} < $3)
      """
    end
  end

  defp bin_params(band, lo, hi), do: [band, lo, hi]
end