Extracts the memory-hostile HRRR fetch → decode → score pipeline for
forecast hours 1–18 into a separate Rust service (`prop-grid-rs`).
Elixir retains f00 with its native-duct + NEXRAD + commercial-link
enrichment; Rust handles the other 18 steps per hourly chain.
Hand-off via a new `grid_tasks` table (`FOR UPDATE SKIP LOCKED` claim
from Rust, Postgrex `NOTIFY propagation_ready` back to Elixir). Rust
writes the same on-disk score-grid format Elixir already uses, to the
same `/data/scores` NFS tree. Phase 1 ships in shadow mode with
PROP_SCORES_DIR=/data/scores_shadow.
Rust crate layout at `rust/prop_grid_rs/` (1:1 module parity with the
Elixir source it ports):
- grid, region, band_config, scores_file, sounding_params
- scorer: all 10 factors + composite. Matches the Elixir scorer
byte-for-byte across 115 golden-fixture samples (5 scenarios
× 23 bands).
- decoder: wgrib2 subprocess + Fortran-record lola-binary parser
- fetcher: HRRR URL derivation + idx cache (1h TTL) + 8-way
parallel byte-range downloads with 429/5xx retry
- pipeline: end-to-end chain step, f00 rejected at the boundary
- db: sqlx grid_tasks claim/complete + propagation_ready NOTIFY
- bin/worker: tokio main loop, JSON logs, SIGTERM-safe
91 Rust tests + clippy -D warnings clean. 2,158 Elixir tests green.
Elixir additions:
- `GridTaskEnqueuer` seeds fh=1..18 rows from
`PropagationGridWorker.seed_chain/0`
- `NotifyListener` LISTEN → warm `ScoreCache` → PubSub fan-out
- `ShadowComparator` diffs prod vs shadow `.ntms` bodies
- `mix rust.golden` task writes the Rust-side golden fixture
k8s: new `deployment-grid-rs.yaml` pinned to talos5 (32 GB NUC) via
`prop-grid-rs=primary` nodeSelector + `workload=grid-rs:NoSchedule`
toleration, 512 Mi limit, sharing the existing NFS `/data` mount.
The plan document is at `plans/vivid-hatching-quail.md` (local to my
workstation); phases 2 (cutover) and 3 (talos5 concurrency tuning)
follow after 72h of shadow-mode parity.
214 lines
5.7 KiB
Elixir
214 lines
5.7 KiB
Elixir
defmodule Mix.Tasks.Rust.Golden do
|
|
@shortdoc "Generate golden test fixtures for the Rust scorer port"
|
|
@moduledoc """
|
|
Dump Elixir-scorer output for a sample of points to
|
|
`priv/rust_golden/scores.bincode`. The Rust crate loads this fixture
|
|
in its scorer tests to assert byte-for-byte parity with the Elixir
|
|
implementation.
|
|
|
|
The fixture format is a little-endian binary:
|
|
|
|
n_samples : u32
|
|
per-sample:
|
|
abs_humidity : f64
|
|
temp_f : f64
|
|
dewpoint_f : f64
|
|
wind_speed_kts : f64 (NaN = None)
|
|
sky_cover_pct : f64 (NaN = None)
|
|
utc_hour : u8
|
|
utc_minute : u8
|
|
month : u8
|
|
longitude : f64
|
|
latitude : f64 (NaN = None)
|
|
pressure_mb : f64 (NaN = None)
|
|
prev_pressure_mb : f64 (NaN = None)
|
|
rain_rate_mmhr : f64 (NaN = None)
|
|
min_refractivity_gradient : f64 (NaN = None)
|
|
bl_depth_m : f64 (NaN = None)
|
|
pwat_mm : f64 (NaN = None)
|
|
best_duct_band_ghz: f64 (NaN = None)
|
|
bulk_richardson : f64 (NaN = None)
|
|
band_mhz : u32
|
|
expected_score : i32
|
|
|
|
No Rust dependency on bincode's stable format — we write fixed-width
|
|
fields in a fixed order and the Rust loader reads them in the same
|
|
order. Matches the Rust `Conditions` struct field-for-field.
|
|
|
|
mix rust.golden # writes priv/rust_golden/scores.bincode
|
|
mix rust.golden --print 5 # pretty-print first 5 samples
|
|
|
|
"""
|
|
use Mix.Task
|
|
|
|
alias Microwaveprop.Propagation.BandConfig
|
|
alias Microwaveprop.Propagation.Scorer
|
|
|
|
@impl true
|
|
def run(args) do
|
|
{opts, _, _} = OptionParser.parse(args, strict: [print: :integer])
|
|
samples = build_samples()
|
|
|
|
path = Path.join(["priv", "rust_golden", "scores.bincode"])
|
|
File.mkdir_p!(Path.dirname(path))
|
|
File.write!(path, encode(samples))
|
|
Mix.shell().info("wrote #{length(samples)} samples to #{path}")
|
|
|
|
if n = opts[:print] do
|
|
samples |> Enum.take(n) |> Enum.each(&IO.inspect/1)
|
|
end
|
|
end
|
|
|
|
defp build_samples do
|
|
bands = BandConfig.all_bands()
|
|
|
|
conditions_set = [
|
|
summer_peak(),
|
|
winter_dry_cold(),
|
|
stormy_afternoon(),
|
|
dawn_duct(),
|
|
humid_gulf()
|
|
]
|
|
|
|
for {c, idx} <- Enum.with_index(conditions_set), band <- bands do
|
|
result = Scorer.composite_score(c, band)
|
|
{idx, c, band.freq_mhz, result.score}
|
|
end
|
|
end
|
|
|
|
defp summer_peak do
|
|
%{
|
|
abs_humidity: 10.0,
|
|
temp_f: 80,
|
|
dewpoint_f: 70,
|
|
wind_speed_kts: 5,
|
|
sky_cover_pct: 20,
|
|
utc_hour: 11,
|
|
utc_minute: 15,
|
|
month: 6,
|
|
longitude: -75.0,
|
|
pressure_mb: 1020,
|
|
prev_pressure_mb: 1018,
|
|
rain_rate_mmhr: 0,
|
|
min_refractivity_gradient: -350,
|
|
bl_depth_m: 400,
|
|
pwat_mm: 25.0
|
|
}
|
|
end
|
|
|
|
defp winter_dry_cold do
|
|
%{
|
|
abs_humidity: 2.5,
|
|
temp_f: 28,
|
|
dewpoint_f: 14,
|
|
wind_speed_kts: 18,
|
|
sky_cover_pct: 10,
|
|
utc_hour: 14,
|
|
utc_minute: 0,
|
|
month: 1,
|
|
longitude: -97.0,
|
|
pressure_mb: 1028,
|
|
prev_pressure_mb: 1026,
|
|
rain_rate_mmhr: 0,
|
|
min_refractivity_gradient: -60,
|
|
bl_depth_m: 1800,
|
|
pwat_mm: 5.0
|
|
}
|
|
end
|
|
|
|
defp stormy_afternoon do
|
|
%{
|
|
abs_humidity: 18.0,
|
|
temp_f: 85,
|
|
dewpoint_f: 74,
|
|
wind_speed_kts: 22,
|
|
sky_cover_pct: 95,
|
|
utc_hour: 21,
|
|
utc_minute: 0,
|
|
month: 7,
|
|
longitude: -97.0,
|
|
pressure_mb: 1002,
|
|
prev_pressure_mb: 1008,
|
|
rain_rate_mmhr: 8.0,
|
|
min_refractivity_gradient: -80,
|
|
bl_depth_m: 2200,
|
|
pwat_mm: 38.0
|
|
}
|
|
end
|
|
|
|
defp dawn_duct do
|
|
%{
|
|
abs_humidity: 14.0,
|
|
temp_f: 72,
|
|
dewpoint_f: 70,
|
|
wind_speed_kts: 3,
|
|
sky_cover_pct: 5,
|
|
utc_hour: 11,
|
|
utc_minute: 30,
|
|
month: 5,
|
|
longitude: -95.0,
|
|
pressure_mb: 1018,
|
|
prev_pressure_mb: 1017,
|
|
rain_rate_mmhr: 0,
|
|
min_refractivity_gradient: -220,
|
|
bl_depth_m: 140,
|
|
pwat_mm: 30.0
|
|
}
|
|
end
|
|
|
|
defp humid_gulf do
|
|
%{
|
|
abs_humidity: 22.0,
|
|
temp_f: 88,
|
|
dewpoint_f: 77,
|
|
wind_speed_kts: 8,
|
|
sky_cover_pct: 55,
|
|
utc_hour: 18,
|
|
utc_minute: 0,
|
|
month: 8,
|
|
longitude: -93.0,
|
|
latitude: 29.5,
|
|
pressure_mb: 1014,
|
|
prev_pressure_mb: 1014,
|
|
rain_rate_mmhr: 0,
|
|
min_refractivity_gradient: -110,
|
|
bl_depth_m: 700,
|
|
pwat_mm: 52.0
|
|
}
|
|
end
|
|
|
|
defp encode(samples) do
|
|
header = <<length(samples)::unsigned-little-32>>
|
|
body = for {_idx, c, band_mhz, score} <- samples, into: <<>>, do: encode_sample(c, band_mhz, score)
|
|
header <> body
|
|
end
|
|
|
|
defp encode_sample(c, band_mhz, score) do
|
|
f(c.abs_humidity) <>
|
|
f(c.temp_f) <>
|
|
f(c.dewpoint_f) <>
|
|
opt(c[:wind_speed_kts]) <>
|
|
opt(c[:sky_cover_pct]) <>
|
|
<<c.utc_hour::unsigned-8, c.utc_minute::unsigned-8, c.month::unsigned-8>> <>
|
|
f(c.longitude) <>
|
|
opt(c[:latitude]) <>
|
|
opt(c[:pressure_mb]) <>
|
|
opt(c[:prev_pressure_mb]) <>
|
|
opt(c[:rain_rate_mmhr]) <>
|
|
opt(c[:min_refractivity_gradient]) <>
|
|
opt(c[:bl_depth_m]) <>
|
|
opt(c[:pwat_mm]) <>
|
|
opt(c[:best_duct_band_ghz]) <>
|
|
opt(c[:bulk_richardson]) <>
|
|
<<band_mhz::unsigned-little-32, score::signed-little-32>>
|
|
end
|
|
|
|
# IEEE 754 quiet-NaN bit pattern, little-endian. Used as the sentinel
|
|
# for `None` on the Rust side — `f64::is_nan` → `Option::None`.
|
|
@nan_bytes <<0x7FF8_0000_0000_0000::unsigned-little-64>>
|
|
|
|
defp f(n) when is_float(n), do: <<n::float-little-64>>
|
|
defp f(n) when is_integer(n), do: <<n * 1.0::float-little-64>>
|
|
defp opt(nil), do: @nan_bytes
|
|
defp opt(v), do: f(v)
|
|
end
|