defmodule Microwaveprop.Pskr.CalibrationSamplerTest do use Microwaveprop.DataCase, async: true import Ecto.Query alias Microwaveprop.Pskr.CalibrationSample alias Microwaveprop.Pskr.CalibrationSampler alias Microwaveprop.Pskr.SpotHourly alias Microwaveprop.Repo alias Microwaveprop.SpaceWeather.GeomagneticObservation alias Microwaveprop.Weather.HrrrProfile @hour ~U[2026-05-04 18:00:00Z] defp insert_spot!(attrs) do {:ok, _} = %SpotHourly{} |> SpotHourly.changeset( Map.merge( %{ hour_utc: @hour, band: "10000", sender_grid: "EM12KL", receiver_grid: "DM43ST", spot_count: 1, midpoint_lat: 33.0, midpoint_lon: -97.0, distance_km: 200.0, modes: ["FT8"] }, attrs ) ) |> Repo.insert() end defp insert_hrrr!(attrs) do {:ok, _} = %HrrrProfile{} |> HrrrProfile.changeset( Map.merge( %{ valid_time: @hour, lat: 33.0, lon: -97.0, run_time: @hour, surface_temp_c: 25.0, surface_dewpoint_c: 18.0, pwat_mm: 30.0, surface_pressure_mb: 1013.0, min_refractivity_gradient: -100.0, hpbl_m: 500.0, ducting_detected: false, profile: [] }, attrs ) ) |> Repo.insert() end defp insert_kp!(value) do {:ok, _} = %GeomagneticObservation{} |> GeomagneticObservation.changeset(%{ valid_time: DateTime.add(@hour, 60, :second), kp_index: value }) |> Repo.insert() end describe "build_for_hour/1" do test "skips and returns zeros when no spots exist for the hour" do assert %{upserted: 0, missing_hrrr_cells: 0} = CalibrationSampler.build_for_hour(@hour) assert Repo.aggregate(CalibrationSample, :count) == 0 end test "builds one sample per (band, snapped midpoint) cell" do insert_spot!(%{midpoint_lat: 33.01, midpoint_lon: -97.02}) insert_spot!(%{midpoint_lat: 33.04, midpoint_lon: -97.03, sender_grid: "EM13", spot_count: 3}) # Distinct grid cell (>0.125° away) insert_spot!(%{midpoint_lat: 35.0, midpoint_lon: -97.0, sender_grid: "EM15"}) insert_hrrr!(%{lat: 33.0, lon: -97.0}) assert %{upserted: 2} = CalibrationSampler.build_for_hour(@hour) [a, b] = CalibrationSample |> Repo.all() |> Enum.sort_by(& &1.midpoint_lat) # Two near-midpoints collapsed; one distant cell stayed separate. assert a.spot_count == 4 assert a.distinct_paths == 2 assert b.spot_count == 1 end test "snaps midpoints to the 0.125° propagation grid" do insert_spot!(%{midpoint_lat: 33.06, midpoint_lon: -97.07}) insert_hrrr!(%{lat: 33.0, lon: -97.0}) CalibrationSampler.build_for_hour(@hour) [sample] = Repo.all(CalibrationSample) # 33.06 → nearest 0.125° step → 33.0; -97.07 → -97.125 assert sample.midpoint_lat == 33.0 assert sample.midpoint_lon == -97.125 end test "joins HRRR atmospheric features at the midpoint" do insert_spot!(%{}) insert_hrrr!(%{ lat: 33.0, lon: -97.0, surface_temp_c: 22.5, surface_dewpoint_c: 16.0, pwat_mm: 28.0, min_refractivity_gradient: -150.0, hpbl_m: 420.0, ducting_detected: true }) CalibrationSampler.build_for_hour(@hour) [sample] = Repo.all(CalibrationSample) assert sample.surface_temp_c == 22.5 assert sample.surface_dewpoint_c == 16.0 assert sample.pwat_mm == 28.0 assert sample.min_refractivity_gradient == -150.0 assert sample.hpbl_m == 420.0 assert sample.hrrr_ducting_detected == true end test "leaves HRRR fields nil when no nearby profile exists" do insert_spot!(%{}) # No HRRR profile inserted at all. CalibrationSampler.build_for_hour(@hour) [sample] = Repo.all(CalibrationSample) assert sample.surface_temp_c == nil assert sample.hpbl_m == nil assert sample.hrrr_ducting_detected == nil end test "enqueues an hrrr_fetch_tasks row when a cell has no nearby HRRR profile" do insert_spot!(%{midpoint_lat: 33.0, midpoint_lon: -97.0}) assert %{upserted: 1, missing_hrrr_cells: 1} = CalibrationSampler.build_for_hour(@hour) [task] = Repo.all(from(t in "hrrr_fetch_tasks", select: %{points: t.points, status: t.status, valid_time: t.valid_time})) assert task.status == "queued" assert task.valid_time == DateTime.to_naive(@hour) # The cell midpoint, snapped to HRRR grid (0.01° rounding). assert task.points == [%{"lat" => 33.0, "lon" => -97.0}] end test "does not enqueue an hrrr_fetch_tasks row when an HRRR profile already covers the cell" do insert_spot!(%{}) insert_hrrr!(%{lat: 33.0, lon: -97.0}) CalibrationSampler.build_for_hour(@hour) assert Repo.aggregate(from(t in "hrrr_fetch_tasks"), :count) == 0 end test "deduplicates HRRR enqueue points across bands sharing a midpoint" do insert_spot!(%{band: "10000", midpoint_lat: 33.0, midpoint_lon: -97.0}) insert_spot!(%{band: "24000", midpoint_lat: 33.0, midpoint_lon: -97.0, sender_grid: "EM13"}) insert_spot!(%{band: "10000", midpoint_lat: 35.0, midpoint_lon: -97.0, sender_grid: "EM15"}) CalibrationSampler.build_for_hour(@hour) [task] = Repo.all(from(t in "hrrr_fetch_tasks", select: %{points: t.points})) # Two unique midpoints (33.0,-97.0) and (35.0,-97.0); the second band at the # first midpoint must collapse, otherwise the Rust worker double-fetches the # same lat/lon for one valid_time. assert length(task.points) == 2 sorted = Enum.sort_by(task.points, & &1["lat"]) assert sorted == [%{"lat" => 33.0, "lon" => -97.0}, %{"lat" => 35.0, "lon" => -97.0}] end test "stamps the latest Kp from SpaceWeather on every sample in the hour" do insert_spot!(%{}) insert_hrrr!(%{lat: 33.0, lon: -97.0}) insert_kp!(5) CalibrationSampler.build_for_hour(@hour) [sample] = Repo.all(CalibrationSample) assert sample.kp_index == 5 end test "computes median distance across the cell's path mix" do insert_spot!(%{distance_km: 100.0, sender_grid: "EM10"}) insert_spot!(%{distance_km: 200.0, sender_grid: "EM11"}) insert_spot!(%{distance_km: 300.0, sender_grid: "EM12"}) insert_hrrr!(%{lat: 33.0, lon: -97.0}) CalibrationSampler.build_for_hour(@hour) [sample] = Repo.all(CalibrationSample) assert sample.median_distance_km == 200.0 end test "deduplicates modes across the cell's spots" do insert_spot!(%{modes: ["FT8"], sender_grid: "EM10"}) insert_spot!(%{modes: ["FT4", "FT8"], sender_grid: "EM11"}) insert_hrrr!(%{lat: 33.0, lon: -97.0}) CalibrationSampler.build_for_hour(@hour) [sample] = Repo.all(CalibrationSample) assert Enum.sort(sample.modes) == ["FT4", "FT8"] end test "two-pass pipeline: first run NULL+enqueue, second run UPSERTs with HRRR" do # End-to-end behaviour the producer was added for: a cell with no # nearby HRRR profile gets a sample row written with NULL weather # fields *and* a fetch task enqueued; once the Rust hrrr-point-worker # delivers the profile (simulated here by directly inserting an # hrrr_profiles row), the next sampler pass UPSERTs the same # calibration_samples row, this time with non-NULL HRRR fields. insert_spot!(%{midpoint_lat: 33.0, midpoint_lon: -97.0}) # ── Pass 1 ───────────────────────────────────────────────── assert %{upserted: 1, missing_hrrr_cells: 1} = CalibrationSampler.build_for_hour(@hour) [first_sample] = Repo.all(CalibrationSample) assert first_sample.surface_temp_c == nil assert first_sample.hpbl_m == nil assert Repo.aggregate(from(t in "hrrr_fetch_tasks"), :count) == 1 # ── Rust worker drains the task (simulated) ──────────────── insert_hrrr!(%{ lat: 33.0, lon: -97.0, surface_temp_c: 24.0, surface_dewpoint_c: 18.0, pwat_mm: 31.0, min_refractivity_gradient: -120.0, hpbl_m: 480.0, ducting_detected: true }) # ── Pass 2 ───────────────────────────────────────────────── assert %{upserted: 1, missing_hrrr_cells: 0} = CalibrationSampler.build_for_hour(@hour) [second_sample] = Repo.all(CalibrationSample) # Same row, UPSERTed (id preserved, weather populated). assert second_sample.id == first_sample.id assert second_sample.surface_temp_c == 24.0 assert second_sample.surface_dewpoint_c == 18.0 assert second_sample.pwat_mm == 31.0 assert second_sample.min_refractivity_gradient == -120.0 assert second_sample.hpbl_m == 480.0 assert second_sample.hrrr_ducting_detected == true end test "is idempotent — re-running the same hour upserts the row" do insert_spot!(%{}) insert_hrrr!(%{lat: 33.0, lon: -97.0}) CalibrationSampler.build_for_hour(@hour) first_count = Repo.aggregate(CalibrationSample, :count) # Mutate the spot count and re-run; the sample must update, not duplicate. Repo.update_all(SpotHourly, set: [spot_count: 99]) CalibrationSampler.build_for_hour(@hour) second_count = Repo.aggregate(CalibrationSample, :count) assert first_count == second_count [sample] = Repo.all(CalibrationSample) assert sample.spot_count == 99 end end end