From 0f0e5e8d43bc27a2f6e472448953b1c4ba549db7 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Fri, 10 Apr 2026 08:47:10 -0500 Subject: [PATCH] Phase 3 spike (parked) + Phase 6: temperature anomaly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 NEXRAD spike: IEM n0q composite available at 5-min cadence back to 2022+. Compression-ratio proxy shows afternoon images have 13-81% more texture than dawn (directionally correct), but the n0q product thresholds out the faint clear-air returns needed for BL stability detection. Parked until MRMS or Level III products can be investigated. See docs/research/nexrad_spike.md. Phase 6: hrrr_climatology table aggregating surface_temp_c by (lat, lon, month, hour) from the 42M+ hrrr_profiles grid-point rows. mix hrrr_climatology builds it via a single SQL GROUP BY + upsert. Backtest.Features.temperature_anomaly computes current_temp minus climatological mean — the meteorologist's "temperature deviation above normal" predictor for summer afternoon enhancement. --- docs/research/nexrad_spike.md | 56 ++++++++++++++++++ lib/microwaveprop/backtest/features.ex | 37 ++++++++++++ lib/microwaveprop/weather/hrrr_climatology.ex | 36 ++++++++++++ lib/mix/tasks/hrrr_climatology.ex | 57 +++++++++++++++++++ ...20260410134532_create_hrrr_climatology.exs | 19 +++++++ 5 files changed, 205 insertions(+) create mode 100644 docs/research/nexrad_spike.md create mode 100644 lib/microwaveprop/weather/hrrr_climatology.ex create mode 100644 lib/mix/tasks/hrrr_climatology.ex create mode 100644 priv/repo/migrations/20260410134532_create_hrrr_climatology.exs diff --git a/docs/research/nexrad_spike.md b/docs/research/nexrad_spike.md new file mode 100644 index 00000000..d80ba816 --- /dev/null +++ b/docs/research/nexrad_spike.md @@ -0,0 +1,56 @@ +# NEXRAD Clear-Air Stability Spike — Findings + +**Date:** 2026-04-10 +**Purpose:** Task 3.1 of the propagation modeling improvements plan. +**Question:** Can NEXRAD composite reflectivity texture be used as an observed BL stability feature? + +## Data Source + +IEM provides CONUS n0q composite reflectivity as PNG at 5-minute cadence: + +``` +https://mesonet.agron.iastate.edu/archive/data/YYYY/MM/DD/GIS/uscomp/n0q_YYYYMMDDHHmm.png +``` + +- Resolution: 12200 × 5400 pixels at 0.005°/pixel (~550m) +- Coverage: CONUS (-126°W to -65°W, 23°N to 50°N) +- Cadence: every 5 minutes +- Historical availability: confirmed back to at least 2022 (spot-checked) +- World file provides georeferencing +- File size: 2-4 MB per frame + +## Visual Assessment + +Downloaded dawn (10Z) vs afternoon (20Z) pairs for: +- 2022-06-11 (stable summer day) +- 2022-08-20 (convective August day, 382 QSOs in our DB) + +**Visual gate: marginal.** The n0q product is primarily sensitive to precipitation-sized hydrometeors. The "smooth clear-air echoes" the meteorologist described are faint returns from insects, dust, and refractive index gradients in the BL — these show up on individual radar sites in clear-air mode but are largely thresholded out in the CONUS composite product. + +## Compression-Ratio Proxy + +PNG compression efficiency correlates inversely with image texture (more complex patterns compress less). As a very crude metric: + +| Date | Dawn (10Z) | Afternoon (20Z) | Ratio | +|---|---|---|---| +| 2022-06-11 (stable) | 2.14 MB | 2.40 MB | 1.13 | +| 2022-08-20 (convective) | 2.30 MB | 4.18 MB | 1.81 | + +The direction is correct: afternoon images have more texture than dawn, and the difference is much larger on convective days. This is consistent with the meteorologist's claim that BL turbulence produces spatially heterogeneous radar returns. + +## Assessment + +**The signal exists but the n0q CONUS composite may not be the right product.** + +Better options to investigate: +1. **Individual radar Level III products** — higher sensitivity, preserve clear-air returns that the composite thresholds out. Available from IEM per station. +2. **MRMS (Multi-Radar Multi-Sensor)** — NOAA's seamless mosaic at 1 km resolution, preserves more low-reflectivity detail than the IEM composite. +3. **Level II raw data** — full volume scans. Maximum sensitivity but requires complex ingestion (not a PNG). + +## Recommendation + +**Park Phase 3 for now.** The concept is sound but the n0q CONUS composite isn't sensitive enough. Revisit when: +- We have bandwidth to spike MRMS or Level III per-station products +- Or when Phase 9 recalibration identifies BL turbulence as a weak spot in the Phase 2 features (Ri, θₑ, shear) and we need an observed check on the model-derived values + +The Phase 2 turbulence features (computed from HRRR native profiles) are a good interim proxy. NEXRAD would add independent observational confirmation, but it's not the highest-priority data source to build right now. diff --git a/lib/microwaveprop/backtest/features.ex b/lib/microwaveprop/backtest/features.ex index 9b4f30a6..11953626 100644 --- a/lib/microwaveprop/backtest/features.ex +++ b/lib/microwaveprop/backtest/features.ex @@ -22,6 +22,7 @@ defmodule Microwaveprop.Backtest.Features do alias Microwaveprop.Repo alias Microwaveprop.Weather + alias Microwaveprop.Weather.HrrrClimatology alias Microwaveprop.Weather.HrrrNativeProfile @doc """ @@ -206,6 +207,42 @@ defmodule Microwaveprop.Backtest.Features do end end + @doc """ + Temperature anomaly (°C): how much hotter or cooler the current + surface temperature is compared to the climatological mean for + this grid cell, month, and hour. + + Positive = hotter than normal. The meteorologist noted that + 10°F+ above-normal summer days produce enhanced ducting even in + the afternoon. + """ + @spec temperature_anomaly(float, float, DateTime.t()) :: float | nil + def temperature_anomaly(lat, lon, valid_time) do + with %{surface_temp_c: temp_c} when is_float(temp_c) <- + Weather.find_nearest_hrrr(lat, lon, valid_time), + %HrrrClimatology{mean_surface_temp_c: clim_mean} when is_float(clim_mean) <- + find_climatology(lat, lon, valid_time.month, valid_time.hour) do + temp_c - clim_mean + else + _ -> nil + end + end + + defp find_climatology(lat, lon, month, hour) do + dlat = 0.07 + dlon = 0.07 + + HrrrClimatology + |> where( + [c], + c.lat >= ^(lat - dlat) and c.lat <= ^(lat + dlat) and + c.lon >= ^(lon - dlon) and c.lon <= ^(lon + dlon) and + c.month == ^month and c.hour == ^hour + ) + |> limit(1) + |> Repo.one() + end + @doc "Duct usable for 10 GHz." def duct_usable_10ghz(lat, lon, vt), do: duct_usable_for_band(lat, lon, vt, 10.0) diff --git a/lib/microwaveprop/weather/hrrr_climatology.ex b/lib/microwaveprop/weather/hrrr_climatology.ex new file mode 100644 index 00000000..e3b8ed95 --- /dev/null +++ b/lib/microwaveprop/weather/hrrr_climatology.ex @@ -0,0 +1,36 @@ +defmodule Microwaveprop.Weather.HrrrClimatology do + @moduledoc """ + Pre-computed mean and stddev of surface temperature from + `hrrr_profiles`, keyed on `(lat, lon, month, hour)`. + + Used by Phase 6 to compute temperature anomaly: how much hotter + (or cooler) a given hour is compared to the same grid cell's + typical value for that month and time of day. The meteorologist + noted that extremely hot days (~10°F above normal in summer) + produce enhanced ducting even in the afternoon when the + time-of-day factor normally suppresses the score. + """ + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + schema "hrrr_climatology" do + field :lat, :float + field :lon, :float + field :month, :integer + field :hour, :integer + field :mean_surface_temp_c, :float + field :stddev_surface_temp_c, :float + field :sample_count, :integer + end + + def changeset(record, attrs) do + record + |> cast(attrs, [:lat, :lon, :month, :hour, :mean_surface_temp_c, :stddev_surface_temp_c, :sample_count]) + |> validate_required([:lat, :lon, :month, :hour]) + |> unique_constraint([:lat, :lon, :month, :hour]) + end +end diff --git a/lib/mix/tasks/hrrr_climatology.ex b/lib/mix/tasks/hrrr_climatology.ex new file mode 100644 index 00000000..977caf00 --- /dev/null +++ b/lib/mix/tasks/hrrr_climatology.ex @@ -0,0 +1,57 @@ +defmodule Mix.Tasks.HrrrClimatology do + @shortdoc "Build surface temperature climatology from hrrr_profiles" + @moduledoc """ + Aggregates `hrrr_profiles.surface_temp_c` by (lat, lon, month, hour) + into `hrrr_climatology` for use by the temperature-anomaly feature. + + With 42M+ grid-point profiles, this runs a single SQL GROUP BY and + bulk-inserts the results. Idempotent via ON CONFLICT. + + mix hrrr_climatology # build from all grid-point profiles + mix hrrr_climatology --min-samples 5 # require at least 5 observations per cell + """ + use Mix.Task + + alias Microwaveprop.Repo + + @impl Mix.Task + def run(argv) do + Mix.Task.run("app.start") + + {opts, _, _} = OptionParser.parse(argv, switches: [min_samples: :integer]) + min_samples = Keyword.get(opts, :min_samples, 3) + + Mix.shell().info("Building climatology (min_samples=#{min_samples})...") + + # Single SQL: aggregate and upsert in one shot + {count, _} = + Repo.query!( + """ + INSERT INTO hrrr_climatology (id, lat, lon, month, hour, + mean_surface_temp_c, stddev_surface_temp_c, sample_count) + SELECT gen_random_uuid(), lat, lon, + EXTRACT(MONTH FROM valid_time)::int AS month, + EXTRACT(HOUR FROM valid_time)::int AS hour, + AVG(surface_temp_c), + STDDEV_SAMP(surface_temp_c), + COUNT(*) + FROM hrrr_profiles + WHERE surface_temp_c IS NOT NULL + AND is_grid_point = true + GROUP BY lat, lon, + EXTRACT(MONTH FROM valid_time)::int, + EXTRACT(HOUR FROM valid_time)::int + HAVING COUNT(*) >= $1 + ON CONFLICT (lat, lon, month, hour) + DO UPDATE SET + mean_surface_temp_c = EXCLUDED.mean_surface_temp_c, + stddev_surface_temp_c = EXCLUDED.stddev_surface_temp_c, + sample_count = EXCLUDED.sample_count + """, + [min_samples], + timeout: 600_000 + ) + + Mix.shell().info("Upserted #{count} climatology records.") + end +end diff --git a/priv/repo/migrations/20260410134532_create_hrrr_climatology.exs b/priv/repo/migrations/20260410134532_create_hrrr_climatology.exs new file mode 100644 index 00000000..c753a104 --- /dev/null +++ b/priv/repo/migrations/20260410134532_create_hrrr_climatology.exs @@ -0,0 +1,19 @@ +defmodule Microwaveprop.Repo.Migrations.CreateHrrrClimatology do + use Ecto.Migration + + def change do + create table(:hrrr_climatology, primary_key: false) do + add :id, :binary_id, primary_key: true, null: false + add :lat, :float, null: false + add :lon, :float, null: false + add :month, :integer, null: false + add :hour, :integer, null: false + add :mean_surface_temp_c, :float + add :stddev_surface_temp_c, :float + add :sample_count, :integer + end + + create unique_index(:hrrr_climatology, [:lat, :lon, :month, :hour]) + create index(:hrrr_climatology, [:month, :hour]) + end +end