Extracts the memory-hostile HRRR fetch → decode → score pipeline for
forecast hours 1–18 into a separate Rust service (`prop-grid-rs`).
Elixir retains f00 with its native-duct + NEXRAD + commercial-link
enrichment; Rust handles the other 18 steps per hourly chain.
Hand-off via a new `grid_tasks` table (`FOR UPDATE SKIP LOCKED` claim
from Rust, Postgrex `NOTIFY propagation_ready` back to Elixir). Rust
writes the same on-disk score-grid format Elixir already uses, to the
same `/data/scores` NFS tree. Phase 1 ships in shadow mode with
PROP_SCORES_DIR=/data/scores_shadow.
Rust crate layout at `rust/prop_grid_rs/` (1:1 module parity with the
Elixir source it ports):
- grid, region, band_config, scores_file, sounding_params
- scorer: all 10 factors + composite. Matches the Elixir scorer
byte-for-byte across 115 golden-fixture samples (5 scenarios
× 23 bands).
- decoder: wgrib2 subprocess + Fortran-record lola-binary parser
- fetcher: HRRR URL derivation + idx cache (1h TTL) + 8-way
parallel byte-range downloads with 429/5xx retry
- pipeline: end-to-end chain step, f00 rejected at the boundary
- db: sqlx grid_tasks claim/complete + propagation_ready NOTIFY
- bin/worker: tokio main loop, JSON logs, SIGTERM-safe
91 Rust tests + clippy -D warnings clean. 2,158 Elixir tests green.
Elixir additions:
- `GridTaskEnqueuer` seeds fh=1..18 rows from
`PropagationGridWorker.seed_chain/0`
- `NotifyListener` LISTEN → warm `ScoreCache` → PubSub fan-out
- `ShadowComparator` diffs prod vs shadow `.ntms` bodies
- `mix rust.golden` task writes the Rust-side golden fixture
k8s: new `deployment-grid-rs.yaml` pinned to talos5 (32 GB NUC) via
`prop-grid-rs=primary` nodeSelector + `workload=grid-rs:NoSchedule`
toleration, 512 Mi limit, sharing the existing NFS `/data` mount.
The plan document is at `plans/vivid-hatching-quail.md` (local to my
workstation); phases 2 (cutover) and 3 (talos5 concurrency tuning)
follow after 72h of shadow-mode parity.
119 lines
3.9 KiB
Elixir
119 lines
3.9 KiB
Elixir
defmodule Microwaveprop.Propagation.ShadowComparator do
|
|
@moduledoc """
|
|
Shadow-mode validator for the Rust `prop-grid-rs` port.
|
|
|
|
During Phase 1 shadow mode the Rust worker writes score-grid files to
|
|
`/data/scores_shadow/<band>/<iso>.ntms` while Elixir keeps writing to
|
|
`/data/scores/`. This module compares the two files and logs the
|
|
per-band delta distribution so drift is visible before cutover.
|
|
|
|
Callable from IEx on demand or from a one-shot Mix task; not started
|
|
in the supervision tree (this is temporary instrumentation).
|
|
"""
|
|
|
|
alias Microwaveprop.Propagation.ScoresFile
|
|
|
|
require Logger
|
|
|
|
@default_prod_dir "/data/scores"
|
|
@default_shadow_dir "/data/scores_shadow"
|
|
|
|
@type diff :: %{
|
|
band_mhz: non_neg_integer(),
|
|
valid_time: DateTime.t(),
|
|
total_cells: non_neg_integer(),
|
|
matching_cells: non_neg_integer(),
|
|
missing_shadow_cells: non_neg_integer(),
|
|
max_abs_delta: non_neg_integer(),
|
|
mean_abs_delta: float()
|
|
}
|
|
|
|
@spec compare(non_neg_integer(), DateTime.t(), keyword()) ::
|
|
{:ok, diff()} | {:error, term()}
|
|
def compare(band_mhz, %DateTime{} = valid_time, opts \\ []) do
|
|
prod_dir = Keyword.get(opts, :prod_dir, @default_prod_dir)
|
|
shadow_dir = Keyword.get(opts, :shadow_dir, @default_shadow_dir)
|
|
|
|
with {:ok, prod} <- read_file(prod_dir, band_mhz, valid_time),
|
|
{:ok, shadow} <- read_file(shadow_dir, band_mhz, valid_time) do
|
|
{:ok, diff_payloads(band_mhz, valid_time, prod, shadow)}
|
|
end
|
|
end
|
|
|
|
defp read_file(dir, band_mhz, valid_time) do
|
|
path = band_mhz |> ScoresFile.path_for(valid_time) |> swap_base(dir)
|
|
|
|
case File.read(path) do
|
|
{:ok, binary} -> decode_scores_file(binary)
|
|
{:error, reason} -> {:error, {:read, path, reason}}
|
|
end
|
|
end
|
|
|
|
defp swap_base(path, new_base) do
|
|
suffix = Path.relative_to(path, ScoresFile.base_dir())
|
|
Path.join(new_base, suffix)
|
|
end
|
|
|
|
# Inline parse of the v1 scores-file on-disk format so this module
|
|
# doesn't reach into `ScoresFile`'s private `decode/1`. The format is
|
|
# documented in `ScoresFile`'s moduledoc; only the body size and
|
|
# dimensions matter for a diff, so we intentionally keep this minimal.
|
|
# The literal "NTMS" magic is the file-header marker already on disk.
|
|
defp decode_scores_file(
|
|
<<"NTMS", 1::unsigned-8, _band_mhz::unsigned-little-32, _valid_time_unix::signed-little-64,
|
|
_lat_min::float-little-32, _lon_min::float-little-32, _step::float-little-32, n_rows::unsigned-little-16,
|
|
n_cols::unsigned-little-16, body::binary>>
|
|
) do
|
|
expected = n_rows * n_cols
|
|
|
|
if byte_size(body) == expected do
|
|
{:ok, %{n_rows: n_rows, n_cols: n_cols, scores: body}}
|
|
else
|
|
{:error, :invalid_format}
|
|
end
|
|
end
|
|
|
|
defp decode_scores_file(_), do: {:error, :invalid_format}
|
|
|
|
defp diff_payloads(band_mhz, valid_time, prod, shadow) do
|
|
%{scores: p_body, n_cols: n_cols, n_rows: n_rows} = prod
|
|
%{scores: s_body} = shadow
|
|
|
|
{total, matching, missing, max_abs, sum_abs} =
|
|
for row <- 0..(n_rows - 1),
|
|
col <- 0..(n_cols - 1),
|
|
reduce: {0, 0, 0, 0, 0} do
|
|
{t, m, miss, max_d, sum_d} ->
|
|
idx = row * n_cols + col
|
|
p = :binary.at(p_body, idx)
|
|
s = :binary.at(s_body, idx)
|
|
|
|
cond do
|
|
p == 255 and s == 255 ->
|
|
{t, m, miss, max_d, sum_d}
|
|
|
|
p != 255 and s == 255 ->
|
|
{t + 1, m, miss + 1, max_d, sum_d}
|
|
|
|
p == s ->
|
|
{t + 1, m + 1, miss, max_d, sum_d}
|
|
|
|
true ->
|
|
d = abs(p - s)
|
|
{t + 1, m, miss, max(max_d, d), sum_d + d}
|
|
end
|
|
end
|
|
|
|
mean = if total > 0, do: sum_abs / total, else: 0.0
|
|
|
|
%{
|
|
band_mhz: band_mhz,
|
|
valid_time: valid_time,
|
|
total_cells: total,
|
|
matching_cells: matching,
|
|
missing_shadow_cells: missing,
|
|
max_abs_delta: max_abs,
|
|
mean_abs_delta: mean
|
|
}
|
|
end
|
|
end
|