prop/lib/microwaveprop/workers/pskr_calibration_worker.ex

defmodule Microwaveprop.Workers.PskrCalibrationWorker do
  @moduledoc """
  Cron worker that builds `Pskr.CalibrationSample` rows by joining
  PSKR spot density with HRRR atmospheric state and SWPC space
  weather.

  ## Why frequent + rolling window

  The naive design — one cron fire per hour, processing only the
  previous hour — has two failure modes:

  1. The whole hour's worth of cells is built in a single batch.
     During peaks (50k+ spots/hour) one batch holds the DB pool
     long enough to look like a backlog.
  2. A pod restart between fires drops a full hour because the
     next cron only catches the most-recent-prev hour.

  Instead we fire every 5 minutes and process the **current hour
  plus one prior hour** by default. Idempotent UPSERT on
  `(hour_utc, band, midpoint_lat, midpoint_lon)` means each pass
  merges new spots into existing rows; the work spreads across the
  hour as spots arrive, and the boundary always has at least one
  cron pass covering both sides without waiting for an hourly fire.

  ## Manual reruns

  - `hour_utc` (ISO 8601) — process exactly that one hour. Uniqueness
    is keyed on args, so manual reruns slot in alongside cron fires.
  - `lookback_hours` (integer) — override the default window size.
    Useful after longer outages: `%{"lookback_hours" => 24}` will
    sweep the last day.

  ## Self-healing follow-ups

  When the sampler enqueues `hrrr_fetch_tasks` (cells without HRRR
  data), this worker schedules a follow-up of itself for that exact
  hour ~10 min in the future via the `hour_utc` args path. The
  follow-up's job is to UPSERT the now-landed HRRR data onto the
  same calibration_samples rows.

  The chain terminates naturally: once HRRR data is fully landed for
  an hour, no fetch tasks are enqueued and no follow-up is scheduled.
  Args carry a `_follow_up: true` sentinel on scheduled re-runs so a
  follow-up that *itself* still finds missing data won't infinitely
  schedule another (the Rust worker sometimes fails on HRRR publish-
  lag for the current hour); the rolling-window cron remains the
  safety net.
  """
  use Oban.Worker,
    queue: :backfill_enqueue,
    max_attempts: 3,
    unique: [
      # Cron is every 5 min (300 s); 240 s window means the unique
      # slot has expired by the time the next fire arrives. Manual
      # reruns with explicit args use a different args fingerprint
      # and bypass this check entirely.
      period: 240,
      states: [:available, :scheduled, :executing, :retryable]
    ]

  alias Microwaveprop.Pskr.CalibrationSampler

  require Logger

  # Default rolling window: current hour + 1 prior. Wide enough to
  # cover a 5–10 min cron interval crossing an hour boundary; not
  # so wide that every fire burns DB time on hours that are already
  # complete. Use `lookback_hours` to widen for catch-up.
  @default_lookback_hours 1

  # Delay before the self-scheduled follow-up sampler pass fires. 10
  # min covers the typical Rust hrrr-point-worker drain latency for
  # ~1k point batches.
  @follow_up_delay_seconds 600

  @impl Oban.Worker
  def perform(%Oban.Job{args: %{"hour_utc" => iso} = args}) when is_binary(iso) do
    {:ok, hour, _} = DateTime.from_iso8601(iso)
    process_hour(hour, args)
    :ok
  end

  def perform(%Oban.Job{args: args}) do
    lookback = lookback_hours(args)
    now = truncate_to_hour(DateTime.utc_now())

    Enum.each(0..lookback, fn offset ->
      hour = DateTime.add(now, -offset * 3600, :second)
      process_hour(hour, args)
    end)

    :ok
  end

  defp lookback_hours(%{"lookback_hours" => n}) when is_integer(n) and n >= 0, do: n
  defp lookback_hours(_), do: @default_lookback_hours

  defp process_hour(hour, args) do
    %{upserted: upserted, missing_hrrr_cells: missing} = CalibrationSampler.build_for_hour(hour)

    if upserted > 0 do
      Logger.info("PskrCalibrationWorker: upserted #{upserted} samples for #{hour}")
    end

    if missing > 0, do: maybe_schedule_follow_up(hour, args)
  end

  # Schedule one follow-up for this hour ~10 min out unless we're
  # already on a follow-up. The `_follow_up: true` sentinel + the
  # `hour_utc`-keyed args together make a unique fingerprint for the
  # Oban unique constraint, so duplicate follow-ups for the same
  # hour within 240 s are deduped automatically.
  defp maybe_schedule_follow_up(_hour, %{"_follow_up" => true}), do: :ok

  defp maybe_schedule_follow_up(hour, _args) do
    %{"hour_utc" => DateTime.to_iso8601(hour), "_follow_up" => true}
    |> __MODULE__.new(schedule_in: @follow_up_delay_seconds)
    |> Oban.insert()
    |> case do
      {:ok, _job} ->
        :ok

      {:error, reason} ->
        # Unique-conflict (same hour follow-up already pending) or
        # transient DB error — either way the rolling-window cron is
        # the safety net, so log and move on.
        Logger.warning("PskrCalibrationWorker: failed to schedule follow-up for #{hour}: #{inspect(reason)}")

        :ok
    end
  end

  defp truncate_to_hour(dt), do: %{dt | minute: 0, second: 0, microsecond: {0, 0}}
end