defmodule Microwaveprop.Workers.AdminTaskWorker do @moduledoc """ Oban worker for long-running admin tasks (backtests, climatology, etc). Enqueued via `Microwaveprop.Release` so that `eval` returns immediately and the work runs in the normal Oban pipeline without blocking. Each task type is dispatched by the `"task"` arg: Microwaveprop.Workers.AdminTaskWorker.new(%{task: "backtest_all"}) Microwaveprop.Workers.AdminTaskWorker.new(%{task: "backtest", feature: "naive_gradient"}) Microwaveprop.Workers.AdminTaskWorker.new(%{task: "climatology", min_samples: 3}) Microwaveprop.Workers.AdminTaskWorker.new(%{task: "native_derive", limit: 10000}) """ use Oban.Worker, queue: :admin, max_attempts: 1, unique: [period: 60] import Ecto.Query alias Microwaveprop.Backtest alias Microwaveprop.Backtest.Features alias Microwaveprop.Propagation.Duct alias Microwaveprop.Propagation.Inversion alias Microwaveprop.Propagation.Recalibrator alias Microwaveprop.Repo alias Microwaveprop.Weather.HrrrNativeProfile alias Microwaveprop.Weather.ThetaE require Logger @impl Oban.Worker def perform(%Oban.Job{args: %{"task" => "backtest_all"} = args}) do sample_size = Map.get(args, "sample_size", 5000) features = Features.all_features() Logger.info("AdminTask: running consolidated backtest for #{map_size(features)} features") results = Backtest.consolidated_report(features, sample_size: sample_size) markdown = Backtest.to_consolidated_markdown(results) path = "priv/backtest_reports/consolidated.md" File.mkdir_p!(Path.dirname(path)) File.write!(path, markdown) Logger.info("AdminTask: consolidated backtest complete, wrote #{path}") :ok end def perform(%Oban.Job{args: %{"task" => "backtest", "feature" => feature_name} = args}) do sample_size = Map.get(args, "sample_size", 5000) fun_atom = String.to_atom(feature_name) if !function_exported?(Features, fun_atom, 3) do Logger.error("AdminTask: unknown feature #{feature_name}") {:error, "unknown feature: #{feature_name}"} end feature_fun = &apply(Features, fun_atom, [&1, &2, &3]) name = "Microwaveprop.Backtest.Features.#{feature_name}" Logger.info("AdminTask: running backtest for #{feature_name}") report = Backtest.evaluate(feature_fun, sample_size: sample_size, feature_name: name) distance_bins = Backtest.lift_by_distance(feature_fun, sample_size: sample_size) band_stats = Backtest.lift_by_band(feature_fun, sample_size: sample_size) markdown = Backtest.to_markdown(report, distance_bins: distance_bins, band_stats: band_stats) path = "priv/backtest_reports/#{feature_name}.md" File.mkdir_p!(Path.dirname(path)) File.write!(path, markdown) Logger.info("AdminTask: backtest for #{feature_name} complete, wrote #{path}") :ok end def perform(%Oban.Job{args: %{"task" => "climatology"} = args}) do min_samples = Map.get(args, "min_samples", 3) %{rows: combos} = Repo.query!( """ SELECT EXTRACT(MONTH FROM valid_time)::int AS month, EXTRACT(HOUR FROM valid_time)::int AS hour FROM hrrr_profiles WHERE surface_temp_c IS NOT NULL AND is_grid_point = true GROUP BY 1, 2 ORDER BY 1, 2 """, [], timeout: 120_000 ) Logger.info("AdminTask: building climatology (min_samples=#{min_samples}, #{length(combos)} batches)") total = combos |> Enum.with_index(1) |> Enum.reduce(0, fn {[month, hour], idx}, acc -> %{num_rows: count} = Repo.query!( """ INSERT INTO hrrr_climatology (id, lat, lon, month, hour, mean_surface_temp_c, stddev_surface_temp_c, sample_count) SELECT gen_random_uuid(), lat, lon, $2 AS month, $3 AS hour, AVG(surface_temp_c), STDDEV_SAMP(surface_temp_c), COUNT(*) FROM hrrr_profiles WHERE surface_temp_c IS NOT NULL AND is_grid_point = true AND EXTRACT(MONTH FROM valid_time)::int = $2 AND EXTRACT(HOUR FROM valid_time)::int = $3 GROUP BY lat, lon HAVING COUNT(*) >= $1 ON CONFLICT (lat, lon, month, hour) DO UPDATE SET mean_surface_temp_c = EXCLUDED.mean_surface_temp_c, stddev_surface_temp_c = EXCLUDED.stddev_surface_temp_c, sample_count = EXCLUDED.sample_count """, [min_samples, month, hour], timeout: 300_000 ) if rem(idx, 10) == 0, do: Logger.info("AdminTask: climatology [#{idx}/#{length(combos)}]") acc + count end) Logger.info("AdminTask: climatology complete, upserted #{total} records") :ok end def perform(%Oban.Job{args: %{"task" => "native_derive"} = args}) do limit = Map.get(args, "limit", 10_000) profiles = HrrrNativeProfile |> where([p], is_nil(p.bulk_richardson) and p.level_count > 2) |> limit(^limit) |> Repo.all() Logger.info("AdminTask: deriving fields for #{length(profiles)} native profiles") count = Enum.count(profiles, fn profile -> p = %{ heights_m: profile.heights_m, temp_k: profile.temp_k, spfh: profile.spfh, pressure_pa: profile.pressure_pa, u_wind_ms: profile.u_wind_ms, v_wind_ms: profile.v_wind_ms, tke_m2s2: profile.tke_m2s2, level_count: profile.level_count } inversion_fields = case Inversion.find_inversion_top(p) do {:ok, %{height_m: top_h, level_idx: top_idx, base_idx: base_idx}} -> [ inversion_top_m: top_h, bulk_richardson: Inversion.bulk_richardson(p, base_idx, top_idx), shear_at_top_ms: Inversion.shear_magnitude(p, base_idx, top_idx), theta_e_jump_k: ThetaE.theta_e_jump(p, base_idx, top_idx) ] :none -> [inversion_top_m: nil, bulk_richardson: nil, shear_at_top_ms: nil, theta_e_jump_k: nil] end duct_result = Duct.analyze(p) fields = inversion_fields ++ [ducts: duct_result.ducts, best_duct_band_ghz: duct_result.best_duct_band_ghz] {1, _} = HrrrNativeProfile |> where([pr], pr.id == ^profile.id) |> Repo.update_all(set: fields) true end) Logger.info("AdminTask: derived fields for #{count} profiles") :ok end def perform(%Oban.Job{args: %{"task" => "recalibrate"} = args}) do sample_size = Map.get(args, "sample_size", 5000) epochs = Map.get(args, "epochs", 2000) learning_rate = Map.get(args, "learning_rate", 0.01) Logger.info("AdminTask: running weight recalibration (sample=#{sample_size}, epochs=#{epochs}, lr=#{learning_rate})") result = Recalibrator.fit( sample_size: sample_size, epochs: epochs, learning_rate: learning_rate ) Logger.info("AdminTask: recalibration complete") Logger.info(" train_loss=#{result.train_loss}, val_loss=#{result.val_loss}, initial_loss=#{result.initial_loss}") for {factor, weight} <- Enum.sort(result.weights) do Logger.info(" #{factor}: #{weight}") end :ok end def perform(%Oban.Job{args: %{"task" => "scorer_diff"}}) do # scorer_diff was a calibration tool that read the per-cell # factors JSONB from the propagation_scores table. That table is # gone — factors are no longer persisted across the CONUS grid — # so the diff has nothing to compute against. Left as a no-op so # any in-flight Oban rows from the pre-cutover era drain cleanly. Logger.warning("AdminTask: scorer_diff is disabled — propagation_scores table + factors storage have been dropped") :ok end def perform(%Oban.Job{args: %{"task" => task}}) do Logger.error("AdminTask: unknown task #{task}") {:error, "unknown task: #{task}"} end end