prop/lib/mix/tasks/rust.golden.ex
Graham McIntire b80878056d
feat(grid-rs): Rust worker for HRRR f01..f18 propagation chain
Extracts the memory-hostile HRRR fetch → decode → score pipeline for
forecast hours 1–18 into a separate Rust service (`prop-grid-rs`).
Elixir retains f00 with its native-duct + NEXRAD + commercial-link
enrichment; Rust handles the other 18 steps per hourly chain.

Hand-off via a new `grid_tasks` table (`FOR UPDATE SKIP LOCKED` claim
from Rust, Postgrex `NOTIFY propagation_ready` back to Elixir). Rust
writes the same on-disk score-grid format Elixir already uses, to the
same `/data/scores` NFS tree. Phase 1 ships in shadow mode with
PROP_SCORES_DIR=/data/scores_shadow.

Rust crate layout at `rust/prop_grid_rs/` (1:1 module parity with the
Elixir source it ports):
  - grid, region, band_config, scores_file, sounding_params
  - scorer: all 10 factors + composite. Matches the Elixir scorer
    byte-for-byte across 115 golden-fixture samples (5 scenarios
    × 23 bands).
  - decoder: wgrib2 subprocess + Fortran-record lola-binary parser
  - fetcher: HRRR URL derivation + idx cache (1h TTL) + 8-way
    parallel byte-range downloads with 429/5xx retry
  - pipeline: end-to-end chain step, f00 rejected at the boundary
  - db: sqlx grid_tasks claim/complete + propagation_ready NOTIFY
  - bin/worker: tokio main loop, JSON logs, SIGTERM-safe

91 Rust tests + clippy -D warnings clean. 2,158 Elixir tests green.

Elixir additions:
  - `GridTaskEnqueuer` seeds fh=1..18 rows from
    `PropagationGridWorker.seed_chain/0`
  - `NotifyListener` LISTEN → warm `ScoreCache` → PubSub fan-out
  - `ShadowComparator` diffs prod vs shadow `.ntms` bodies
  - `mix rust.golden` task writes the Rust-side golden fixture

k8s: new `deployment-grid-rs.yaml` pinned to talos5 (32 GB NUC) via
`prop-grid-rs=primary` nodeSelector + `workload=grid-rs:NoSchedule`
toleration, 512 Mi limit, sharing the existing NFS `/data` mount.

The plan document is at `plans/vivid-hatching-quail.md` (local to my
workstation); phases 2 (cutover) and 3 (talos5 concurrency tuning)
follow after 72h of shadow-mode parity.
2026-04-19 15:42:49 -05:00

214 lines
5.7 KiB
Elixir

defmodule Mix.Tasks.Rust.Golden do
@shortdoc "Generate golden test fixtures for the Rust scorer port"
@moduledoc """
Dump Elixir-scorer output for a sample of points to
`priv/rust_golden/scores.bincode`. The Rust crate loads this fixture
in its scorer tests to assert byte-for-byte parity with the Elixir
implementation.
The fixture format is a little-endian binary:
n_samples : u32
per-sample:
abs_humidity : f64
temp_f : f64
dewpoint_f : f64
wind_speed_kts : f64 (NaN = None)
sky_cover_pct : f64 (NaN = None)
utc_hour : u8
utc_minute : u8
month : u8
longitude : f64
latitude : f64 (NaN = None)
pressure_mb : f64 (NaN = None)
prev_pressure_mb : f64 (NaN = None)
rain_rate_mmhr : f64 (NaN = None)
min_refractivity_gradient : f64 (NaN = None)
bl_depth_m : f64 (NaN = None)
pwat_mm : f64 (NaN = None)
best_duct_band_ghz: f64 (NaN = None)
bulk_richardson : f64 (NaN = None)
band_mhz : u32
expected_score : i32
No Rust dependency on bincode's stable format — we write fixed-width
fields in a fixed order and the Rust loader reads them in the same
order. Matches the Rust `Conditions` struct field-for-field.
mix rust.golden # writes priv/rust_golden/scores.bincode
mix rust.golden --print 5 # pretty-print first 5 samples
"""
use Mix.Task
alias Microwaveprop.Propagation.BandConfig
alias Microwaveprop.Propagation.Scorer
@impl true
def run(args) do
{opts, _, _} = OptionParser.parse(args, strict: [print: :integer])
samples = build_samples()
path = Path.join(["priv", "rust_golden", "scores.bincode"])
File.mkdir_p!(Path.dirname(path))
File.write!(path, encode(samples))
Mix.shell().info("wrote #{length(samples)} samples to #{path}")
if n = opts[:print] do
samples |> Enum.take(n) |> Enum.each(&IO.inspect/1)
end
end
defp build_samples do
bands = BandConfig.all_bands()
conditions_set = [
summer_peak(),
winter_dry_cold(),
stormy_afternoon(),
dawn_duct(),
humid_gulf()
]
for {c, idx} <- Enum.with_index(conditions_set), band <- bands do
result = Scorer.composite_score(c, band)
{idx, c, band.freq_mhz, result.score}
end
end
defp summer_peak do
%{
abs_humidity: 10.0,
temp_f: 80,
dewpoint_f: 70,
wind_speed_kts: 5,
sky_cover_pct: 20,
utc_hour: 11,
utc_minute: 15,
month: 6,
longitude: -75.0,
pressure_mb: 1020,
prev_pressure_mb: 1018,
rain_rate_mmhr: 0,
min_refractivity_gradient: -350,
bl_depth_m: 400,
pwat_mm: 25.0
}
end
defp winter_dry_cold do
%{
abs_humidity: 2.5,
temp_f: 28,
dewpoint_f: 14,
wind_speed_kts: 18,
sky_cover_pct: 10,
utc_hour: 14,
utc_minute: 0,
month: 1,
longitude: -97.0,
pressure_mb: 1028,
prev_pressure_mb: 1026,
rain_rate_mmhr: 0,
min_refractivity_gradient: -60,
bl_depth_m: 1800,
pwat_mm: 5.0
}
end
defp stormy_afternoon do
%{
abs_humidity: 18.0,
temp_f: 85,
dewpoint_f: 74,
wind_speed_kts: 22,
sky_cover_pct: 95,
utc_hour: 21,
utc_minute: 0,
month: 7,
longitude: -97.0,
pressure_mb: 1002,
prev_pressure_mb: 1008,
rain_rate_mmhr: 8.0,
min_refractivity_gradient: -80,
bl_depth_m: 2200,
pwat_mm: 38.0
}
end
defp dawn_duct do
%{
abs_humidity: 14.0,
temp_f: 72,
dewpoint_f: 70,
wind_speed_kts: 3,
sky_cover_pct: 5,
utc_hour: 11,
utc_minute: 30,
month: 5,
longitude: -95.0,
pressure_mb: 1018,
prev_pressure_mb: 1017,
rain_rate_mmhr: 0,
min_refractivity_gradient: -220,
bl_depth_m: 140,
pwat_mm: 30.0
}
end
defp humid_gulf do
%{
abs_humidity: 22.0,
temp_f: 88,
dewpoint_f: 77,
wind_speed_kts: 8,
sky_cover_pct: 55,
utc_hour: 18,
utc_minute: 0,
month: 8,
longitude: -93.0,
latitude: 29.5,
pressure_mb: 1014,
prev_pressure_mb: 1014,
rain_rate_mmhr: 0,
min_refractivity_gradient: -110,
bl_depth_m: 700,
pwat_mm: 52.0
}
end
defp encode(samples) do
header = <<length(samples)::unsigned-little-32>>
body = for {_idx, c, band_mhz, score} <- samples, into: <<>>, do: encode_sample(c, band_mhz, score)
header <> body
end
defp encode_sample(c, band_mhz, score) do
f(c.abs_humidity) <>
f(c.temp_f) <>
f(c.dewpoint_f) <>
opt(c[:wind_speed_kts]) <>
opt(c[:sky_cover_pct]) <>
<<c.utc_hour::unsigned-8, c.utc_minute::unsigned-8, c.month::unsigned-8>> <>
f(c.longitude) <>
opt(c[:latitude]) <>
opt(c[:pressure_mb]) <>
opt(c[:prev_pressure_mb]) <>
opt(c[:rain_rate_mmhr]) <>
opt(c[:min_refractivity_gradient]) <>
opt(c[:bl_depth_m]) <>
opt(c[:pwat_mm]) <>
opt(c[:best_duct_band_ghz]) <>
opt(c[:bulk_richardson]) <>
<<band_mhz::unsigned-little-32, score::signed-little-32>>
end
# IEEE 754 quiet-NaN bit pattern, little-endian. Used as the sentinel
# for `None` on the Rust side — `f64::is_nan` → `Option::None`.
@nan_bytes <<0x7FF8_0000_0000_0000::unsigned-little-64>>
defp f(n) when is_float(n), do: <<n::float-little-64>>
defp f(n) when is_integer(n), do: <<n * 1.0::float-little-64>>
defp opt(nil), do: @nan_bytes
defp opt(v), do: f(v)
end