prop/lib/microwaveprop/release.ex
Graham McIntire 811251dd15
Drop propagation_scores table, ScoresFile is the only store
Full cutover: the propagation_scores Postgres table is gone, and
the binary files under /data/scores are the sole source of truth
for the map render path. Three stacked changes:

1. New migration drops the propagation_scores table and its indexes
   (the earlier tuning migrations for it were already applied and
   are now no-ops against a missing table, which is fine — Ecto
   just runs them on fresh environments).

2. Propagation context is gutted of every GridScore reference.
   replace_scores/2 writes files only. upsert_scores/2 is deleted.
   load_scores_from_db, available_valid_times_from_db,
   point_detail_from_db, point_forecast_from_db, fetch_factors,
   coalesce_factors, the Postgres side of prune_old_scores, and
   the Postgres fallbacks in latest/earliest_valid_time are all
   removed. point_detail always returns an empty factors map now
   since factor breakdowns were retired with the table.

3. Deleted modules:
   - Propagation.GridScore (the schema)
   - Propagation.ScorerDiff (read factors from the table)
   - Propagation.AsosNudge (helper for AsosAdjustmentWorker)
   - Workers.AsosAdjustmentWorker (its cron was already disabled)
   - Mix.Tasks.ScorerDiff (wrapper around the deleted module)
   And their tests. AdminTaskWorker's scorer_diff task is a
   logging no-op so any queued Oban rows drain cleanly.
   Release.scorer_diff stays as a stub that tells the operator.

propagation_prune_worker_test rewritten to exercise ScoresFile
pruning. propagation_test.exs rewritten to use replace_scores +
ScoresFile throughout (no more GridScore / upsert_scores paths).
2026-04-14 15:13:01 -05:00

123 lines
3.9 KiB
Elixir

defmodule Microwaveprop.Release do
@moduledoc """
Release tasks that run in production without Mix installed.
Long-running tasks are enqueued as Oban jobs so `eval` returns
immediately. Check Oban Web at /admin/oban for progress.
Usage via `bin/microwaveprop eval`:
bin/microwaveprop eval "Microwaveprop.Release.migrate"
bin/microwaveprop eval "Microwaveprop.Release.backtest_all"
bin/microwaveprop eval "Microwaveprop.Release.backtest(\"naive_gradient\")"
bin/microwaveprop eval "Microwaveprop.Release.climatology"
bin/microwaveprop eval "Microwaveprop.Release.native_backfill(500)"
bin/microwaveprop eval "Microwaveprop.Release.native_derive"
bin/microwaveprop eval "Microwaveprop.Release.recalibrate"
"""
alias Microwaveprop.Workers.AdminTaskWorker
alias Microwaveprop.Workers.HrrrNativeGridWorker
@app :microwaveprop
def migrate do
load_app()
for repo <- repos() do
{:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true))
end
end
def rollback(repo, version) do
load_app()
{:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version))
end
def backtest(feature_name) when is_binary(feature_name) do
start_app()
{:ok, job} = Oban.insert(AdminTaskWorker.new(%{task: "backtest", feature: feature_name}))
IO.puts("Enqueued backtest for #{feature_name} (job #{job.id})")
end
def backtest_all do
start_app()
{:ok, job} = Oban.insert(AdminTaskWorker.new(%{task: "backtest_all"}))
IO.puts("Enqueued consolidated backtest (job #{job.id})")
end
def climatology(min_samples \\ 3) do
start_app()
{:ok, job} = Oban.insert(AdminTaskWorker.new(%{task: "climatology", min_samples: min_samples}))
IO.puts("Enqueued climatology build (job #{job.id})")
end
def native_backfill(limit \\ 500) do
start_app()
%{rows: rows} =
Microwaveprop.Repo.query!(
"""
SELECT EXTRACT(YEAR FROM qso_timestamp)::int AS year,
EXTRACT(MONTH FROM qso_timestamp)::int AS month,
EXTRACT(DAY FROM qso_timestamp)::int AS day,
EXTRACT(HOUR FROM qso_timestamp)::int AS hour,
COUNT(*) AS cnt
FROM contacts
WHERE pos1 IS NOT NULL
AND qso_timestamp >= '2019-01-01'
GROUP BY 1, 2, 3, 4
ORDER BY cnt DESC
LIMIT $1
""",
[limit],
timeout: 60_000
)
IO.puts("Enqueuing #{length(rows)} native backfill jobs...")
for [year, month, day, hour, cnt] <- rows do
args = %{year: year, month: month, day: day, hour: hour}
case Oban.insert(HrrrNativeGridWorker.new(args, unique: [period: :infinity])) do
{:ok, _} -> IO.puts(" #{year}-#{month}-#{day} #{hour}Z (#{cnt} contacts)")
{:error, _} -> IO.puts(" #{year}-#{month}-#{day} #{hour}Z (skipped)")
end
end
IO.puts("Done.")
end
def recalibrate do
start_app()
{:ok, job} = Oban.insert(AdminTaskWorker.new(%{task: "recalibrate"}))
IO.puts("Enqueued recalibration (job #{job.id})")
end
# scorer_diff removed — the tool depended on per-cell factors in
# propagation_scores, which are gone now that scores live as binary
# files on disk. Left as a stub that tells the operator what
# happened if they shell into a running release from muscle memory.
def scorer_diff(_new_weights_json) do
start_app()
IO.puts("scorer_diff is disabled: propagation_scores table + factors storage have been dropped.")
end
def native_derive(limit \\ 10_000) do
start_app()
{:ok, job} = Oban.insert(AdminTaskWorker.new(%{task: "native_derive", limit: limit}))
IO.puts("Enqueued native derive (job #{job.id})")
end
defp repos do
Application.fetch_env!(@app, :ecto_repos)
end
defp load_app do
Application.ensure_all_started(:ssl)
Application.ensure_loaded(@app)
end
defp start_app do
Application.ensure_all_started(@app)
end
end