From a283ad9c666f9e7d76029a695b3fce7d433c1b39 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 30 Mar 2026 17:21:47 -0500 Subject: [PATCH] Refactor HRRR fetch to batch points per hour QSO enrichment now groups all path points by HRRR hour and creates one batch job per hour instead of one job per point. The batch job downloads the GRIB2 data once and extracts all needed points from the same binary. Legacy single-point jobs are still supported for backward compatibility. --- algo3.md => docs/algo_commercial.md | 0 algo1.md => docs/algo_original.md | 0 algo2.md => docs/algo_updated_with_data.md | 0 .../plans/2026-03-30-conus-propagation-map.md | 2042 +++++++++++++++++ .../workers/hrrr_fetch_worker.ex | 99 +- .../workers/qso_weather_enqueue_worker.ex | 29 +- .../qso_weather_enqueue_worker_test.exs | 29 +- 7 files changed, 2144 insertions(+), 55 deletions(-) rename algo3.md => docs/algo_commercial.md (100%) rename algo1.md => docs/algo_original.md (100%) rename algo2.md => docs/algo_updated_with_data.md (100%) create mode 100644 docs/plans/2026-03-30-conus-propagation-map.md diff --git a/algo3.md b/docs/algo_commercial.md similarity index 100% rename from algo3.md rename to docs/algo_commercial.md diff --git a/algo1.md b/docs/algo_original.md similarity index 100% rename from algo1.md rename to docs/algo_original.md diff --git a/algo2.md b/docs/algo_updated_with_data.md similarity index 100% rename from algo2.md rename to docs/algo_updated_with_data.md diff --git a/docs/plans/2026-03-30-conus-propagation-map.md b/docs/plans/2026-03-30-conus-propagation-map.md new file mode 100644 index 00000000..eed709ac --- /dev/null +++ b/docs/plans/2026-03-30-conus-propagation-map.md @@ -0,0 +1,2042 @@ +# CONUS Propagation Map Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Build a Leaflet-based CONUS propagation map at `/map` showing per-band microwave propagation conditions, updated hourly from HRRR data, with a band selector to compare conditions across frequencies. + +**Architecture:** HRRR GRIB2 data is downloaded once per hour for the entire CONUS grid, then scored at ~10,000 points (0.125° resolution) across all bands. Scores are stored in a `propagation_scores` table and served to a Leaflet map via a LiveView at `/map`. The scoring algorithm is fully data-driven — weights, thresholds, and band configs are module attributes that can be updated without touching scoring logic. + +**Tech Stack:** Phoenix 1.8 + LiveView, Leaflet.js (vendored), Oban workers, existing GRIB2 decoder, PostgreSQL + +**Key Design Constraint:** The scoring algorithm will evolve over time. All scoring parameters (weights, thresholds, band configs, seasonal tables) are defined as data in a single config module (`Microwaveprop.Propagation.BandConfig`). Scoring functions are generic — they read thresholds from the config, not hardcoded values. This means tuning the algorithm is a config change, not a logic change. + +--- + +## Task 1: HRRR Multi-Point GRIB2 Extraction + +Refactor the GRIB2 extractor to extract many points from a single download instead of one point per download. This is the foundation — currently every HRRR fetch job downloads the same ~50MB of GRIB2 data to extract a single lat/lon. + +**Files:** +- Modify: `lib/microwaveprop/weather/grib2/extractor.ex` +- Modify: `lib/microwaveprop/weather/grib2/simple_packing.ex` +- Modify: `lib/microwaveprop/weather/grib2/complex_packing.ex` +- Test: `test/microwaveprop/weather/grib2/extractor_test.exs` + +### Step 1: Write failing test for multi-point extraction + +```elixir +# test/microwaveprop/weather/grib2/extractor_test.exs +defmodule Microwaveprop.Weather.Grib2.ExtractorTest do + use ExUnit.Case, async: true + + alias Microwaveprop.Weather.Grib2.Extractor + + # Use the existing single-point test as baseline, then add multi-point + describe "extract_grid/2" do + test "extracts multiple points from a single GRIB2 binary" do + # We'll need a small fixture GRIB2 binary for this test. + # For now, test that the function exists and returns the right shape. + # A real integration test will use a downloaded HRRR fixture. + points = [{35.0, -97.0}, {36.0, -96.0}, {37.0, -95.0}] + + # This will be an integration test using a real GRIB2 fixture. + # For unit testing, verify the batch extraction calls through to + # single-point extraction correctly. + assert is_function(&Extractor.extract_grid/2, 2) + end + end +end +``` + +### Step 2: Run test to verify it fails + +Run: `mix test test/microwaveprop/weather/grib2/extractor_test.exs --trace` +Expected: Compilation error — `extract_grid/2` doesn't exist + +### Step 3: Add `extract_values/3` to SimplePacking for batch index extraction + +The key optimization: for simple packing, extracting N values is N independent bit reads from the same binary — no sequential dependency. For complex packing, `decode_all/2` already decodes the entire grid, so we just index into the result array at multiple positions. + +```elixir +# In simple_packing.ex, add: +def extract_values(params, data, indices) do + %{ + reference_value: r, + binary_scale: e, + decimal_scale: d, + bits_per_value: n, + num_data_points: num + } = params + + factor_2e = :math.pow(2, e) + factor_10d = :math.pow(10, -d) + + results = + Enum.reduce(indices, %{}, fn index, acc -> + if index >= 0 and index < num do + x = extract_bits(data, index, n) + value = (r + x * factor_2e) * factor_10d + Map.put(acc, index, value) + else + acc + end + end) + + {:ok, results} +end +``` + +```elixir +# In complex_packing.ex, add: +def extract_values(params, data, indices) do + if Enum.any?(indices, &(&1 < 0 or &1 >= params.num_data_points)) do + {:error, :index_out_of_range} + else + case decode_all(params, data) do + {:ok, array} -> + results = + Enum.reduce(indices, %{}, fn index, acc -> + Map.put(acc, index, :array.get(index, array)) + end) + {:ok, results} + + {:error, _} = err -> + err + end + end +end +``` + +### Step 4: Implement `extract_grid/2` in Extractor + +```elixir +# In extractor.ex, add: + +@doc """ +Extract weather values from a GRIB2 binary at multiple lat/lon points. + +Returns `{:ok, %{{lat, lon} => %{"VAR:LEVEL" => float}}}` or `{:error, term}`. +""" +def extract_grid(binary, points) when is_list(points) do + messages = split_messages(binary) + + Enum.reduce_while(messages, {:ok, %{}}, fn msg, {:ok, acc} -> + case extract_grid_single(msg, points) do + {:ok, key, point_values} -> + merged = + Enum.reduce(point_values, acc, fn {point, value}, a -> + existing = Map.get(a, point, %{}) + Map.put(a, point, Map.put(existing, key, value)) + end) + {:cont, {:ok, merged}} + + {:error, :outside_grid} -> + {:halt, {:error, :outside_grid}} + + {:error, reason} -> + {:halt, {:error, reason}} + end + end) +end + +defp extract_grid_single(msg, points) do + with {:ok, parsed} <- Section.parse_message(msg), + %{grid_params: grid, product: prod, packing_params: packing, data: data} = parsed, + key = "#{prod.var}:#{prod.level}" do + # Convert all lat/lons to grid indices + indexed_points = + Enum.flat_map(points, fn {lat, lon} = point -> + case LambertConformal.to_grid_index(grid, lat, lon) do + {:ok, {i, j}} -> + index = linear_index({i, j}, grid.nx, grid.scan_mode) + [{point, index}] + {:error, :outside_grid} -> + [] # Skip points outside HRRR grid + end + end) + + indices = Enum.map(indexed_points, &elem(&1, 1)) + + case unpack_values(packing, data, indices) do + {:ok, index_values} -> + point_values = + Enum.flat_map(indexed_points, fn {point, index} -> + case Map.get(index_values, index) do + nil -> [] + value -> [{point, value}] + end + end) + {:ok, key, point_values} + {:error, _} = err -> + err + end + end +rescue + e -> {:error, "GRIB2 grid extraction failed: #{inspect(e)}"} +end + +defp unpack_values(%{template: 3} = params, data, indices) do + ComplexPacking.extract_values(params, data, indices) +end + +defp unpack_values(params, data, indices) do + SimplePacking.extract_values(params, data, indices) +end +``` + +### Step 5: Run tests and verify they pass + +Run: `mix test test/microwaveprop/weather/grib2/extractor_test.exs --trace` +Expected: PASS + +### Step 6: Commit + +```bash +git add lib/microwaveprop/weather/grib2/extractor.ex \ + lib/microwaveprop/weather/grib2/simple_packing.ex \ + lib/microwaveprop/weather/grib2/complex_packing.ex \ + test/microwaveprop/weather/grib2/extractor_test.exs +git commit -m "Add multi-point GRIB2 extraction for batch grid queries" +``` + +--- + +## Task 2: HRRR Batch Fetch Client + +Add a function to HrrrClient that downloads one HRRR hour and extracts data at a list of points. Also add the additional GRIB2 fields needed for full scoring (wind, cloud cover, precip). + +**Files:** +- Modify: `lib/microwaveprop/weather/hrrr_client.ex` +- Test: `test/microwaveprop/weather/hrrr_client_test.exs` + +### Step 1: Write failing test + +```elixir +defmodule Microwaveprop.Weather.HrrrClientTest do + use ExUnit.Case, async: true + + alias Microwaveprop.Weather.HrrrClient + + describe "additional surface messages" do + test "surface_messages/0 includes wind, cloud cover, and precip fields" do + messages = HrrrClient.surface_messages() + vars = Enum.map(messages, & &1.var) + + assert "UGRD" in vars + assert "VGRD" in vars + assert "TCDC" in vars + assert "APCP" in vars + end + end + + describe "fetch_grid/3" do + test "function exists with correct arity" do + assert is_function(&HrrrClient.fetch_grid/3, 3) + end + end +end +``` + +### Step 2: Run test to verify it fails + +Run: `mix test test/microwaveprop/weather/hrrr_client_test.exs --trace` +Expected: FAIL + +### Step 3: Add new GRIB2 fields and `fetch_grid/3` + +```elixir +# In hrrr_client.ex, update @surface_messages to add wind/cloud/precip: +@surface_messages [ + %{var: "TMP", level: "2 m above ground"}, + %{var: "DPT", level: "2 m above ground"}, + %{var: "PRES", level: "surface"}, + %{var: "HPBL", level: "surface"}, + %{var: "PWAT", level: "entire atmosphere (considered as a single layer)"}, + %{var: "UGRD", level: "10 m above ground"}, + %{var: "VGRD", level: "10 m above ground"}, + %{var: "TCDC", level: "entire atmosphere"}, + %{var: "APCP", level: "surface"} +] + +# Add public accessor for tests: +def surface_messages, do: @surface_messages + +# Add fetch_grid/3: +@doc """ +Download one HRRR hour and extract data at multiple lat/lon points. +Returns `{:ok, %{{lat, lon} => profile_map}}` or `{:error, term}`. +""" +def fetch_grid(points, valid_time, opts \\ []) do + hour_dt = nearest_hrrr_hour(valid_time) + date = DateTime.to_date(hour_dt) + hour = hour_dt.hour + + include_pressure = Keyword.get(opts, :include_pressure, true) + + with {:ok, sfc_data} <- fetch_product_grid(date, hour, :surface, points), + {:ok, prs_data} <- maybe_fetch_pressure_grid(include_pressure, date, hour, points) do + merged = merge_grid_data(sfc_data, prs_data) + profiles = Map.new(merged, fn {point, data} -> + {point, Map.put(build_profile(data), :run_time, hour_dt)} + end) + {:ok, profiles} + end +end + +defp maybe_fetch_pressure_grid(false, _date, _hour, _points), do: {:ok, %{}} +defp maybe_fetch_pressure_grid(true, date, hour, points) do + fetch_product_grid(date, hour, :pressure, points) +end + +defp fetch_product_grid(date, hour, product, points) do + url = hrrr_url(date, hour, product) + idx_url = url <> ".idx" + + wanted = + case product do + :surface -> @surface_messages + :pressure -> pressure_messages() + end + + Logger.info("HRRR grid fetching #{product} idx from #{idx_url}") + + with {:ok, idx_text} <- fetch_idx(idx_url), + idx_entries = parse_idx(idx_text), + ranges = byte_ranges_for_messages(idx_entries, wanted), + {:ok, grib_binary} <- download_grib_ranges(url, ranges) do + Extractor.extract_grid(grib_binary, points) + end +end + +defp merge_grid_data(sfc, prs) do + all_points = MapSet.union(MapSet.new(Map.keys(sfc)), MapSet.new(Map.keys(prs))) + Map.new(all_points, fn point -> + sfc_data = Map.get(sfc, point, %{}) + prs_data = Map.get(prs, point, %{}) + {point, Map.merge(sfc_data, prs_data)} + end) +end +``` + +Also update `build_profile/1` to include the new fields: + +```elixir +# Add to the result map in build_profile/1: +wind_u: parsed["UGRD:10 m above ground"], +wind_v: parsed["VGRD:10 m above ground"], +cloud_cover_pct: parsed["TCDC:entire atmosphere"], +precip_mm: parsed["APCP:surface"], +``` + +### Step 4: Run tests + +Run: `mix test test/microwaveprop/weather/hrrr_client_test.exs --trace` +Expected: PASS + +### Step 5: Commit + +```bash +git add lib/microwaveprop/weather/hrrr_client.ex \ + test/microwaveprop/weather/hrrr_client_test.exs +git commit -m "Add HRRR batch grid fetch with wind, cloud, and precip fields" +``` + +--- + +## Task 3: Band Configuration Module + +Create the data-driven band configuration module. All scoring parameters live here — weights, thresholds, seasonal tables, band-specific coefficients. When the algorithm evolves, this is the only file that changes. + +**Files:** +- Create: `lib/microwaveprop/propagation/band_config.ex` +- Test: `test/microwaveprop/propagation/band_config_test.exs` + +### Step 1: Write failing test + +```elixir +defmodule Microwaveprop.Propagation.BandConfigTest do + use ExUnit.Case, async: true + + alias Microwaveprop.Propagation.BandConfig + + describe "get/1" do + test "returns config for 10 GHz" do + config = BandConfig.get(10_000) + assert config.label == "10 GHz" + assert config.humidity_effect == :beneficial + assert config.humidity_penalty == 0.0 + assert map_size(config.seasonal_base) == 12 + end + + test "returns config for 24 GHz" do + config = BandConfig.get(24_000) + assert config.label == "24 GHz" + assert config.humidity_effect == :harmful + assert config.humidity_penalty == 1.6 + end + + test "returns nil for unknown band" do + assert BandConfig.get(99_999) == nil + end + end + + describe "all_bands/0" do + test "returns all 8 band configs" do + bands = BandConfig.all_bands() + assert length(bands) == 8 + freqs = Enum.map(bands, & &1.freq_mhz) + assert 10_000 in freqs + assert 241_000 in freqs + end + end + + describe "weights/0" do + test "weights sum to 1.0" do + weights = BandConfig.weights() + total = Enum.reduce(weights, 0.0, fn {_k, v}, acc -> acc + v end) + assert_in_delta total, 1.0, 0.001 + end + + test "includes all 9 scoring factors" do + weights = BandConfig.weights() + expected = ~w(humidity time_of_day td_depression refractivity sky season wind rain pressure)a + for key <- expected, do: assert(Map.has_key?(weights, key)) + end + end +end +``` + +### Step 2: Run test to verify it fails + +Run: `mix test test/microwaveprop/propagation/band_config_test.exs --trace` +Expected: FAIL — module doesn't exist + +### Step 3: Implement BandConfig + +```elixir +defmodule Microwaveprop.Propagation.BandConfig do + @moduledoc """ + Band-specific configuration for microwave propagation scoring. + + All scoring parameters — weights, thresholds, seasonal tables, and + band-specific coefficients — are defined here as data. The scoring + functions in `Scorer` are generic and read from these configs. + + To tune the algorithm, update the values in this module. No scoring + logic needs to change. + """ + + @weights %{ + humidity: 0.20, + time_of_day: 0.20, + td_depression: 0.12, + refractivity: 0.10, + sky: 0.10, + season: 0.10, + wind: 0.06, + rain: 0.08, + pressure: 0.04 + } + + @sunrise_table [7.4, 7.3, 7.0, 6.7, 6.35, 6.25, + 6.35, 6.65, 6.9, 7.1, 7.35, 7.45] + + # Score tiers and their colors + @tiers [ + %{min: 80, label: "EXCELLENT", color: "#00ffa3"}, + %{min: 65, label: "GOOD", color: "#7dffd4"}, + %{min: 50, label: "MARGINAL", color: "#ffe566"}, + %{min: 33, label: "POOR", color: "#ff9044"}, + %{min: 0, label: "NEGLIGIBLE", color: "#ff4f4f"} + ] + + # Humidity scoring thresholds for :beneficial bands (10 GHz) + # {max_humidity, score} — evaluated in order, first match wins + @humidity_beneficial_thresholds [ + {4, 55}, {7, 70}, {10, 82}, {14, 90}, {18, 95}, {22, 88} + ] + @humidity_beneficial_default 75 + + # Refractivity gradient thresholds + # {max_gradient, score_beneficial, score_harmful} + @refractivity_thresholds [ + {-500, 98, 85}, + {-300, 92, 78}, + {-200, 80, 80}, + {-100, 65, 65}, + {-60, 55, 55} + ] + @refractivity_default {42, 42} + + # BL depth threshold for shallow BL bonus + @shallow_bl_threshold_m 300 + @shallow_bl_score 82 + + @band_configs %{ + 10_000 => %{ + freq_mhz: 10_000, + label: "10 GHz", + o2_db_km: 0.008, + h2o_coeff: 0.0005, + humidity_effect: :beneficial, + humidity_penalty: 0.0, + rain_k: 0.010, rain_alpha: 1.28, + seasonal_base: %{1 => 38, 2 => 32, 3 => 22, 4 => 55, 5 => 68, + 6 => 90, 7 => 95, 8 => 75, 9 => 78, 10 => 82, + 11 => 78, 12 => 25}, + seasonal_adj: %{}, + typical_range_km: 200, + extended_range_km: 500, + exceptional_range_km: 1000 + }, + 24_000 => %{ + freq_mhz: 24_000, + label: "24 GHz", + o2_db_km: 0.015, + h2o_coeff: 0.012, + humidity_effect: :harmful, + humidity_penalty: 1.6, + rain_k: 0.070, rain_alpha: 1.07, + seasonal_base: %{1 => 88, 2 => 84, 3 => 68, 4 => 62, 5 => 51, + 6 => 34, 7 => 18, 8 => 18, 9 => 48, 10 => 68, + 11 => 96, 12 => 88}, + seasonal_adj: %{5 => -4, 6 => -8, 7 => -10, 8 => -10, 9 => -4}, + typical_range_km: 100, + extended_range_km: 250, + exceptional_range_km: 500 + }, + 47_000 => %{ + freq_mhz: 47_000, + label: "47 GHz", + o2_db_km: 0.045, + h2o_coeff: 0.003, + humidity_effect: :harmful, + humidity_penalty: 1.0, + rain_k: 0.187, rain_alpha: 0.93, + seasonal_base: %{1 => 90, 2 => 88, 3 => 78, 4 => 68, 5 => 55, + 6 => 38, 7 => 22, 8 => 22, 9 => 48, 10 => 74, + 11 => 96, 12 => 90}, + seasonal_adj: %{}, + typical_range_km: 70, + extended_range_km: 150, + exceptional_range_km: 300 + }, + 68_000 => %{ + freq_mhz: 68_000, + label: "68 GHz", + o2_db_km: 0.90, + h2o_coeff: 0.007, + humidity_effect: :harmful, + humidity_penalty: 1.4, + rain_k: 0.310, rain_alpha: 0.86, + seasonal_base: %{1 => 90, 2 => 88, 3 => 78, 4 => 65, 5 => 50, + 6 => 32, 7 => 18, 8 => 18, 9 => 44, 10 => 70, + 11 => 92, 12 => 90}, + seasonal_adj: %{}, + typical_range_km: 40, + extended_range_km: 80, + exceptional_range_km: 150 + }, + 75_000 => %{ + freq_mhz: 75_000, + label: "75 GHz", + o2_db_km: 0.012, + h2o_coeff: 0.006, + humidity_effect: :harmful, + humidity_penalty: 1.2, + rain_k: 0.345, rain_alpha: 0.84, + seasonal_base: %{1 => 90, 2 => 90, 3 => 80, 4 => 68, 5 => 55, + 6 => 38, 7 => 22, 8 => 22, 9 => 48, 10 => 74, + 11 => 96, 12 => 90}, + seasonal_adj: %{}, + typical_range_km: 50, + extended_range_km: 120, + exceptional_range_km: 250 + }, + 122_000 => %{ + freq_mhz: 122_000, + label: "122 GHz", + o2_db_km: 0.80, + h2o_coeff: 0.010, + humidity_effect: :harmful, + humidity_penalty: 1.0, + rain_k: 0.498, rain_alpha: 0.77, + seasonal_base: %{1 => 92, 2 => 90, 3 => 78, 4 => 62, 5 => 45, + 6 => 28, 7 => 15, 8 => 15, 9 => 38, 10 => 68, + 11 => 92, 12 => 92}, + seasonal_adj: %{}, + typical_range_km: 30, + extended_range_km: 80, + exceptional_range_km: 140 + }, + 134_000 => %{ + freq_mhz: 134_000, + label: "134 GHz", + o2_db_km: 0.08, + h2o_coeff: 0.015, + humidity_effect: :harmful, + humidity_penalty: 1.3, + rain_k: 0.520, rain_alpha: 0.75, + seasonal_base: %{1 => 92, 2 => 90, 3 => 78, 4 => 65, 5 => 48, + 6 => 30, 7 => 18, 8 => 18, 9 => 42, 10 => 70, + 11 => 92, 12 => 92}, + seasonal_adj: %{}, + typical_range_km: 40, + extended_range_km: 100, + exceptional_range_km: 160 + }, + 241_000 => %{ + freq_mhz: 241_000, + label: "241 GHz", + o2_db_km: 0.08, + h2o_coeff: 0.30, + humidity_effect: :harmful, + humidity_penalty: 3.0, + rain_k: 0.550, rain_alpha: 0.70, + seasonal_base: %{1 => 95, 2 => 92, 3 => 75, 4 => 55, 5 => 35, + 6 => 15, 7 => 8, 8 => 8, 9 => 30, 10 => 65, + 11 => 95, 12 => 95}, + seasonal_adj: %{}, + typical_range_km: 10, + extended_range_km: 50, + exceptional_range_km: 115 + } + } + + def get(freq_mhz), do: Map.get(@band_configs, freq_mhz) + + def all_bands do + @band_configs + |> Map.values() + |> Enum.sort_by(& &1.freq_mhz) + end + + def all_freqs, do: @band_configs |> Map.keys() |> Enum.sort() + + def weights, do: @weights + + def sunrise_table, do: @sunrise_table + + def tiers, do: @tiers + + def humidity_beneficial_thresholds, do: @humidity_beneficial_thresholds + def humidity_beneficial_default, do: @humidity_beneficial_default + + def refractivity_thresholds, do: @refractivity_thresholds + def refractivity_default, do: @refractivity_default + + def shallow_bl_threshold_m, do: @shallow_bl_threshold_m + def shallow_bl_score, do: @shallow_bl_score +end +``` + +### Step 4: Run tests + +Run: `mix test test/microwaveprop/propagation/band_config_test.exs --trace` +Expected: PASS + +### Step 5: Commit + +```bash +git add lib/microwaveprop/propagation/band_config.ex \ + test/microwaveprop/propagation/band_config_test.exs +git commit -m "Add data-driven band configuration module for propagation scoring" +``` + +--- + +## Task 4: Scoring Algorithm Module + +Implement the 9 scoring functions + composite score. All functions read thresholds from BandConfig — no hardcoded magic numbers in the scoring logic itself. This makes the algorithm tunable by editing BandConfig only. + +**Files:** +- Create: `lib/microwaveprop/propagation/scorer.ex` +- Test: `test/microwaveprop/propagation/scorer_test.exs` + +### Step 1: Write failing tests + +```elixir +defmodule Microwaveprop.Propagation.ScorerTest do + use ExUnit.Case, async: true + + alias Microwaveprop.Propagation.BandConfig + alias Microwaveprop.Propagation.Scorer + + describe "score_humidity/2" do + test "10 GHz — high humidity is beneficial" do + config = BandConfig.get(10_000) + score = Scorer.score_humidity(15.0, config) + assert score >= 90 + end + + test "10 GHz — low humidity scores lower" do + config = BandConfig.get(10_000) + score = Scorer.score_humidity(3.0, config) + assert score <= 60 + end + + test "24 GHz — high humidity is harmful" do + config = BandConfig.get(24_000) + high = Scorer.score_humidity(15.0, config) + low = Scorer.score_humidity(3.0, config) + assert low > high + end + end + + describe "score_time_of_day/3" do + test "dawn scores highest" do + # January, 7 AM UTC = ~1 AM CST... let's use 13 UTC = 7 AM CST (near sunrise) + {score, _label} = Scorer.score_time_of_day(13, 0, 1) + assert score == 100 + end + + test "afternoon scores lowest" do + # January, 20 UTC = 2 PM CST + {score, _label} = Scorer.score_time_of_day(20, 0, 1) + assert score <= 40 + end + end + + describe "score_td_depression/3" do + test "10 GHz — moderate depression scores well" do + config = BandConfig.get(10_000) + score = Scorer.score_td_depression(70.0, 60.0, config) + assert score >= 75 + end + + test "24 GHz — large depression (dry) scores well" do + config = BandConfig.get(24_000) + score = Scorer.score_td_depression(80.0, 55.0, config) + assert score >= 80 + end + end + + describe "score_refractivity/3" do + test "strong ducting gradient scores high for 10 GHz" do + config = BandConfig.get(10_000) + score = Scorer.score_refractivity(-550.0, nil, config) + assert score >= 95 + end + + test "nil gradient returns neutral score" do + config = BandConfig.get(10_000) + score = Scorer.score_refractivity(nil, nil, config) + assert score == 50 + end + end + + describe "score_sky/1" do + test "clear sky scores 100" do + assert Scorer.score_sky(0.0) == 100 + end + + test "overcast scores low" do + assert Scorer.score_sky(95.0) <= 10 + end + end + + describe "score_season/2" do + test "10 GHz — July scores highest" do + config = BandConfig.get(10_000) + assert Scorer.score_season(7, config) == 95 + end + + test "24 GHz — November scores highest" do + config = BandConfig.get(24_000) + assert Scorer.score_season(11, config) == 96 + end + end + + describe "score_wind/1" do + test "calm wind scores 100" do + assert Scorer.score_wind(2.0) == 100 + end + + test "strong wind scores low" do + assert Scorer.score_wind(30.0) <= 20 + end + end + + describe "score_rain/2" do + test "no rain scores 100" do + config = BandConfig.get(24_000) + assert Scorer.score_rain(0.0, config) == 100 + end + + test "heavy rain at 24 GHz scores very low" do + config = BandConfig.get(24_000) + score = Scorer.score_rain(25.0, config) + assert score <= 25 + end + end + + describe "score_pressure/2" do + test "rising pressure scores well" do + score = Scorer.score_pressure(1018.0, 1015.0) + assert score >= 70 + end + + test "nil previous gives absolute-only scoring" do + score = Scorer.score_pressure(1018.0, nil) + assert score >= 50 + end + end + + describe "composite_score/2" do + test "returns score 0-100 and factor breakdown" do + config = BandConfig.get(10_000) + + conditions = %{ + abs_humidity: 12.0, + temp_f: 75.0, + dewpoint_f: 65.0, + wind_speed_kts: 5.0, + sky_cover_pct: 10.0, + utc_hour: 13, + utc_minute: 0, + month: 7, + pressure_mb: 1015.0, + prev_pressure_mb: nil, + rain_rate_mmhr: 0.0, + min_refractivity_gradient: -350.0, + bl_depth_m: nil + } + + result = Scorer.composite_score(conditions, config) + assert result.score >= 0 and result.score <= 100 + assert map_size(result.factors) == 9 + end + end +end +``` + +### Step 2: Run test to verify it fails + +Run: `mix test test/microwaveprop/propagation/scorer_test.exs --trace` +Expected: FAIL — module doesn't exist + +### Step 3: Implement Scorer + +```elixir +defmodule Microwaveprop.Propagation.Scorer do + @moduledoc """ + Propagation scoring functions for microwave bands. + + All scoring thresholds and parameters are read from `BandConfig`. + To tune the algorithm, update BandConfig — no changes needed here. + """ + + alias Microwaveprop.Propagation.BandConfig + + # --- Individual scoring functions --- + + def score_humidity(abs_humidity_gm3, band_config) do + case band_config.humidity_effect do + :beneficial -> + BandConfig.humidity_beneficial_thresholds() + |> Enum.find_value(fn {threshold, score} -> + if abs_humidity_gm3 < threshold, do: score + end) || BandConfig.humidity_beneficial_default() + + :harmful -> + r = abs_humidity_gm3 * band_config.humidity_penalty + cond do + r <= 6 -> 100 + r <= 9 -> round(95 - (r - 6) / 3 * 20) + r <= 13 -> round(75 - (r - 9) / 4 * 30) + r <= 18 -> round(45 - (r - 13) / 5 * 35) + true -> max(0, round(10 - (r - 18) * 2)) + end + end + end + + def score_time_of_day(utc_hour, utc_minute, month) do + sunrise_table = BandConfig.sunrise_table() + offset = if month >= 3 and month <= 10, do: -5, else: -6 + local = :math.fmod(utc_hour + utc_minute / 60 + offset + 24, 24) + sunrise = Enum.at(sunrise_table, month - 1) + d = local - sunrise + + cond do + d >= -1.5 and d <= 1.5 -> {100, "Peak — inversion maximum"} + d > 1.5 and d <= 3.0 -> {78, "Good — inversion eroding"} + d > -3.0 and d < -1.5 -> {82, "Pre-dawn — inversion building"} + d > 3.0 and d <= 6.0 -> {38, "Marginal — boundary layer mixing"} + local >= 20.0 or local <= 1.0 -> {72, "Evening — cooling, inversion reforming"} + d > 6.0 -> {18, "Afternoon — full convective mixing"} + true -> {55, "Night — gradual cooling"} + end + end + + def score_td_depression(temp_f, dewpoint_f, band_config) do + dep = temp_f - dewpoint_f + + case band_config.humidity_effect do + :beneficial -> + cond do + dep < 3 -> 40 + dep < 8 -> 75 + dep < 14 -> 85 + dep < 22 -> 70 + true -> 55 + end + + :harmful -> + cond do + dep > 22 -> 96 + dep > 14 -> 80 + dep > 8 -> 60 + dep > 4 -> 38 + true -> 18 + end + end + end + + def score_refractivity(nil, _bl_depth_m, _band_config), do: 50 + + def score_refractivity(min_gradient, bl_depth_m, band_config) do + thresholds = BandConfig.refractivity_thresholds() + {default_beneficial, default_harmful} = BandConfig.refractivity_default() + + found = + Enum.find_value(thresholds, fn {max_grad, score_b, score_h} -> + if min_gradient < max_grad do + case band_config.humidity_effect do + :beneficial -> score_b + :harmful -> score_h + end + end + end) + + cond do + found != nil -> + found + + bl_depth_m != nil and bl_depth_m < BandConfig.shallow_bl_threshold_m() -> + BandConfig.shallow_bl_score() + + true -> + case band_config.humidity_effect do + :beneficial -> default_beneficial + :harmful -> default_harmful + end + end + end + + def score_sky(pct) when is_number(pct) do + cond do + pct <= 6 -> 100 + pct <= 25 -> 88 + pct <= 50 -> 60 + pct <= 87 -> 25 + true -> 5 + end + end + + def score_sky(nil), do: 50 + + def score_season(month, band_config) do + base = Map.get(band_config.seasonal_base, month, 50) + adj = Map.get(band_config.seasonal_adj, month, 0) + max(0, min(100, base + adj)) + end + + def score_wind(speed_kts) when is_number(speed_kts) do + cond do + speed_kts < 5 -> 100 + speed_kts < 10 -> 90 + speed_kts < 15 -> 75 + speed_kts < 20 -> 55 + speed_kts < 25 -> 35 + true -> 15 + end + end + + def score_wind(nil), do: 50 + + def score_rain(nil, _band_config), do: 100 + def score_rain(rate, _band_config) when rate == 0, do: 100 + + def score_rain(rate, band_config) do + gamma = band_config.rain_k * :math.pow(rate, band_config.rain_alpha) + cond do + gamma < 0.1 -> 95 + gamma < 0.5 -> 75 + gamma < 1.0 -> 50 + gamma < 2.0 -> 25 + gamma < 5.0 -> 10 + true -> 0 + end + end + + def score_pressure(current_mb, nil) do + cond do + current_mb > 1025 -> 55 + current_mb > 1018 -> 65 + current_mb > 1010 -> 60 + current_mb > 1005 -> 55 + true -> 40 + end + end + + def score_pressure(current_mb, previous_mb) do + delta = current_mb - previous_mb + cond do + delta > 2.5 -> 80 + delta > 0.8 -> 70 + delta > -0.5 -> 60 + delta > -2.0 -> 65 + true -> 45 + end + end + + # --- Composite --- + + def composite_score(conditions, band_config) do + weights = BandConfig.weights() + {tod_score, _label} = score_time_of_day( + conditions.utc_hour, conditions.utc_minute, conditions.month + ) + + factors = %{ + humidity: score_humidity(conditions.abs_humidity, band_config), + time_of_day: tod_score, + td_depression: score_td_depression( + conditions.temp_f, conditions.dewpoint_f, band_config + ), + refractivity: score_refractivity( + conditions.min_refractivity_gradient, + conditions.bl_depth_m, + band_config + ), + sky: score_sky(conditions.sky_cover_pct), + season: score_season(conditions.month, band_config), + wind: score_wind(conditions.wind_speed_kts), + rain: score_rain(conditions.rain_rate_mmhr, band_config), + pressure: score_pressure( + conditions.pressure_mb, conditions.prev_pressure_mb + ) + } + + score = + factors + |> Enum.reduce(0.0, fn {key, value}, acc -> + acc + value * Map.fetch!(weights, key) + end) + |> round() + |> max(0) + |> min(100) + + %{score: score, factors: factors} + end + + # --- Helpers --- + + @doc "Compute absolute humidity (g/m³) from temp (°C) and dewpoint (°C)." + def absolute_humidity(temp_c, dewpoint_c) do + e_s = 6.112 * :math.exp(17.67 * dewpoint_c / (dewpoint_c + 243.5)) + t_k = temp_c + 273.15 + 217.0 * e_s / t_k + end + + @doc "Wind speed (kts) from U and V components (m/s)." + def wind_speed_kts(u_ms, v_ms) when is_number(u_ms) and is_number(v_ms) do + :math.sqrt(u_ms * u_ms + v_ms * v_ms) * 1.94384 + end + + def wind_speed_kts(_, _), do: nil + + @doc "Convert precip accumulation (mm) to approximate rate (mm/hr)." + def precip_to_rate_mmhr(nil), do: 0.0 + def precip_to_rate_mmhr(mm) when mm <= 0, do: 0.0 + def precip_to_rate_mmhr(mm), do: mm + + @doc "Convert temp F to C." + def f_to_c(nil), do: nil + def f_to_c(f), do: (f - 32) * 5 / 9 + + @doc "Convert temp C to F." + def c_to_f(nil), do: nil + def c_to_f(c), do: c * 9 / 5 + 32 +end +``` + +### Step 4: Run tests + +Run: `mix test test/microwaveprop/propagation/scorer_test.exs --trace` +Expected: PASS + +### Step 5: Commit + +```bash +git add lib/microwaveprop/propagation/scorer.ex \ + test/microwaveprop/propagation/scorer_test.exs +git commit -m "Implement propagation scoring algorithm with data-driven thresholds" +``` + +--- + +## Task 5: CONUS Grid Definition and Score Schema + +Define the CONUS grid (0.125° resolution) and the database schema to store computed scores. + +**Files:** +- Create: `lib/microwaveprop/propagation/grid.ex` +- Create: `lib/microwaveprop/propagation/grid_score.ex` +- Create migration for `propagation_scores` table +- Test: `test/microwaveprop/propagation/grid_test.exs` + +### Step 1: Write failing test + +```elixir +defmodule Microwaveprop.Propagation.GridTest do + use ExUnit.Case, async: true + + alias Microwaveprop.Propagation.Grid + + describe "conus_points/0" do + test "generates grid at 0.125° resolution" do + points = Grid.conus_points() + assert length(points) > 5000 + assert length(points) < 15000 + + # All points within CONUS bounds + Enum.each(points, fn {lat, lon} -> + assert lat >= 25.0 and lat <= 50.0 + assert lon >= -125.0 and lon <= -66.0 + end) + end + + test "points are on 0.125° grid" do + [{lat, lon} | _] = Grid.conus_points() + assert Float.round(lat * 8, 0) == lat * 8 + assert Float.round(lon * 8, 0) == lon * 8 + end + end +end +``` + +### Step 2: Run test to verify it fails + +Run: `mix test test/microwaveprop/propagation/grid_test.exs --trace` +Expected: FAIL + +### Step 3: Implement Grid module + +```elixir +defmodule Microwaveprop.Propagation.Grid do + @moduledoc """ + CONUS grid definition for propagation scoring. + 0.125° resolution (~14 km), covering 25-50°N, 125-66°W. + """ + + @lat_min 25.0 + @lat_max 50.0 + @lon_min -125.0 + @lon_max -66.0 + @step 0.125 + + def conus_points do + for lat <- float_range(@lat_min, @lat_max, @step), + lon <- float_range(@lon_min, @lon_max, @step) do + {Float.round(lat, 3), Float.round(lon, 3)} + end + end + + def step, do: @step + def bounds, do: %{lat_min: @lat_min, lat_max: @lat_max, lon_min: @lon_min, lon_max: @lon_max} + + defp float_range(start, stop, step) do + count = round((stop - start) / step) + 1 + Enum.map(0..(count - 1), fn i -> start + i * step end) + end +end +``` + +### Step 4: Generate migration and create GridScore schema + +Run: `mix ecto.gen.migration create_propagation_scores` + +Then populate the migration: + +```elixir +defmodule Microwaveprop.Repo.Migrations.CreatePropagationScores do + use Ecto.Migration + + def change do + create table(:propagation_scores, primary_key: false) do + add :id, :binary_id, primary_key: true + add :lat, :float, null: false + add :lon, :float, null: false + add :valid_time, :utc_datetime, null: false + add :band_mhz, :integer, null: false + add :score, :integer, null: false + add :factors, :map, null: false + + timestamps(type: :utc_datetime) + end + + create unique_index(:propagation_scores, [:lat, :lon, :valid_time, :band_mhz]) + create index(:propagation_scores, [:valid_time]) + create index(:propagation_scores, [:band_mhz, :valid_time]) + end +end +``` + +Schema: + +```elixir +defmodule Microwaveprop.Propagation.GridScore do + @moduledoc false + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + + schema "propagation_scores" do + field :lat, :float + field :lon, :float + field :valid_time, :utc_datetime + field :band_mhz, :integer + field :score, :integer + field :factors, :map + + timestamps(type: :utc_datetime) + end + + def changeset(grid_score, attrs) do + grid_score + |> cast(attrs, [:lat, :lon, :valid_time, :band_mhz, :score, :factors]) + |> validate_required([:lat, :lon, :valid_time, :band_mhz, :score, :factors]) + end +end +``` + +### Step 5: Run migration and tests + +Run: `mix ecto.migrate && mix test test/microwaveprop/propagation/grid_test.exs --trace` +Expected: PASS + +### Step 6: Commit + +```bash +git add lib/microwaveprop/propagation/grid.ex \ + lib/microwaveprop/propagation/grid_score.ex \ + priv/repo/migrations/*_create_propagation_scores.exs \ + test/microwaveprop/propagation/grid_test.exs +git commit -m "Add CONUS grid definition and propagation_scores schema" +``` + +--- + +## Task 6: Grid Score Computation Context + +Create the context module that orchestrates: take HRRR data for a grid of points → derive atmospheric params → score all bands → upsert results. + +**Files:** +- Create: `lib/microwaveprop/propagation.ex` +- Test: `test/microwaveprop/propagation_test.exs` + +### Step 1: Write failing test + +```elixir +defmodule Microwaveprop.PropagationTest do + use Microwaveprop.DataCase + + alias Microwaveprop.Propagation + alias Microwaveprop.Propagation.GridScore + + describe "score_grid_point/3" do + test "scores a single point for all bands" do + hrrr_profile = %{ + surface_temp_c: 25.0, + surface_dewpoint_c: 18.0, + surface_pressure_mb: 1013.0, + hpbl_m: 500.0, + wind_u: 3.0, + wind_v: 2.0, + cloud_cover_pct: 15.0, + precip_mm: 0.0, + profile: [ + %{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0}, + %{"pres" => 975.0, "tmpc" => 22.0, "dwpc" => 15.0, "hght" => 350.0}, + %{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0} + ] + } + + valid_time = ~U[2026-07-15 13:00:00Z] + results = Propagation.score_grid_point(hrrr_profile, valid_time) + + assert length(results) == 8 + Enum.each(results, fn result -> + assert result.score >= 0 and result.score <= 100 + assert map_size(result.factors) == 9 + end) + end + end + + describe "upsert_scores/1" do + test "inserts and upserts grid scores" do + valid_time = ~U[2026-07-15 13:00:00Z] + + scores = [ + %{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 10_000, + score: 75, factors: %{humidity: 90, time_of_day: 100}}, + %{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 24_000, + score: 60, factors: %{humidity: 40, time_of_day: 100}} + ] + + assert {:ok, 2} = Propagation.upsert_scores(scores) + + # Upsert same point — should update, not duplicate + updated = [ + %{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 10_000, + score: 80, factors: %{humidity: 95, time_of_day: 100}} + ] + + assert {:ok, 1} = Propagation.upsert_scores(updated) + + assert Repo.aggregate(GridScore, :count) == 2 + end + end + + describe "latest_scores/1" do + test "returns latest scores for a band" do + valid_time = ~U[2026-07-15 13:00:00Z] + + scores = [ + %{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 10_000, + score: 75, factors: %{}}, + %{lat: 36.0, lon: -96.0, valid_time: valid_time, band_mhz: 10_000, + score: 80, factors: %{}} + ] + + Propagation.upsert_scores(scores) + results = Propagation.latest_scores(10_000) + assert length(results) == 2 + end + end +end +``` + +### Step 2: Run test to verify it fails + +Run: `mix test test/microwaveprop/propagation_test.exs --trace` +Expected: FAIL + +### Step 3: Implement Propagation context + +```elixir +defmodule Microwaveprop.Propagation do + @moduledoc false + + import Ecto.Query + + alias Microwaveprop.Propagation.BandConfig + alias Microwaveprop.Propagation.GridScore + alias Microwaveprop.Propagation.Scorer + alias Microwaveprop.Repo + alias Microwaveprop.Weather.SoundingParams + + @doc """ + Score a single grid point across all bands using HRRR profile data. + Returns a list of %{band_mhz, score, factors} maps. + """ + def score_grid_point(hrrr_profile, valid_time) do + # Derive refractivity gradient from profile + derived = derive_from_hrrr(hrrr_profile) + + # Build conditions map from HRRR data + temp_c = hrrr_profile.surface_temp_c + dewpoint_c = hrrr_profile.surface_dewpoint_c + temp_f = Scorer.c_to_f(temp_c) + dewpoint_f = Scorer.c_to_f(dewpoint_c) + + conditions = %{ + abs_humidity: Scorer.absolute_humidity(temp_c, dewpoint_c), + temp_f: temp_f, + dewpoint_f: dewpoint_f, + wind_speed_kts: Scorer.wind_speed_kts( + hrrr_profile[:wind_u], hrrr_profile[:wind_v] + ), + sky_cover_pct: hrrr_profile[:cloud_cover_pct], + utc_hour: valid_time.hour, + utc_minute: valid_time.minute, + month: valid_time.month, + pressure_mb: hrrr_profile.surface_pressure_mb, + prev_pressure_mb: nil, + rain_rate_mmhr: Scorer.precip_to_rate_mmhr(hrrr_profile[:precip_mm]), + min_refractivity_gradient: derived[:min_refractivity_gradient], + bl_depth_m: hrrr_profile[:hpbl_m] + } + + Enum.map(BandConfig.all_bands(), fn band_config -> + result = Scorer.composite_score(conditions, band_config) + Map.put(result, :band_mhz, band_config.freq_mhz) + end) + end + + @doc "Upsert propagation scores in batches." + def upsert_scores(scores) do + now = DateTime.utc_now() |> DateTime.truncate(:second) + + entries = + Enum.map(scores, fn s -> + %{ + id: Ecto.UUID.generate(), + lat: s.lat, + lon: s.lon, + valid_time: s.valid_time, + band_mhz: s.band_mhz, + score: s.score, + factors: s.factors, + inserted_at: now, + updated_at: now + } + end) + + total = + entries + |> Enum.chunk_every(500) + |> Enum.reduce(0, fn chunk, acc -> + {count, _} = + Repo.insert_all(GridScore, chunk, + on_conflict: {:replace, [:score, :factors, :updated_at]}, + conflict_target: [:lat, :lon, :valid_time, :band_mhz] + ) + acc + count + end) + + {:ok, total} + end + + @doc "Get the latest scores for a band." + def latest_scores(band_mhz) do + latest_time_query = + from(gs in GridScore, + where: gs.band_mhz == ^band_mhz, + select: max(gs.valid_time) + ) + + case Repo.one(latest_time_query) do + nil -> + [] + + latest_time -> + from(gs in GridScore, + where: gs.band_mhz == ^band_mhz and gs.valid_time == ^latest_time, + select: %{lat: gs.lat, lon: gs.lon, score: gs.score} + ) + |> Repo.all() + end + end + + @doc "Get the latest valid_time across all scores." + def latest_valid_time do + Repo.one(from gs in GridScore, select: max(gs.valid_time)) + end + + # Derive refractivity gradient from HRRR profile using SoundingParams + defp derive_from_hrrr(%{profile: profile}) when is_list(profile) and length(profile) >= 3 do + case SoundingParams.derive(profile) do + nil -> %{} + derived -> %{min_refractivity_gradient: derived.min_refractivity_gradient} + end + end + + defp derive_from_hrrr(_), do: %{} +end +``` + +### Step 4: Run tests + +Run: `mix test test/microwaveprop/propagation_test.exs --trace` +Expected: PASS + +### Step 5: Commit + +```bash +git add lib/microwaveprop/propagation.ex \ + test/microwaveprop/propagation_test.exs +git commit -m "Add propagation context for grid scoring and score persistence" +``` + +--- + +## Task 7: Hourly Grid Worker + +Oban worker that runs hourly: downloads HRRR, extracts CONUS grid, scores all bands, upserts results. + +**Files:** +- Create: `lib/microwaveprop/workers/propagation_grid_worker.ex` +- Modify: `config/config.exs` (add cron entry + queue) +- Test: `test/microwaveprop/workers/propagation_grid_worker_test.exs` + +### Step 1: Write failing test + +```elixir +defmodule Microwaveprop.Workers.PropagationGridWorkerTest do + use Microwaveprop.DataCase + + alias Microwaveprop.Workers.PropagationGridWorker + + describe "new/1" do + test "creates a valid Oban job" do + job = PropagationGridWorker.new(%{}) + assert job.args == %{} + end + end + + # Integration tests would require mocking HRRR S3 — skip for unit tests. + # The worker delegates to tested modules (HrrrClient.fetch_grid, Propagation.score_grid_point). +end +``` + +### Step 2: Implement worker + +```elixir +defmodule Microwaveprop.Workers.PropagationGridWorker do + @moduledoc """ + Hourly Oban worker that downloads the latest HRRR data and computes + propagation scores across the CONUS grid for all bands. + + Downloads GRIB2 data once per hour, extracts at ~10,000 grid points, + scores all 8 bands at each point, and upserts results. + """ + + use Oban.Worker, + queue: :propagation, + max_attempts: 3, + unique: [period: 3600, states: [:available, :scheduled, :executing]] + + require Logger + + alias Microwaveprop.Propagation + alias Microwaveprop.Propagation.Grid + alias Microwaveprop.Weather.HrrrClient + + @impl Oban.Worker + def perform(%Oban.Job{}) do + valid_time = HrrrClient.nearest_hrrr_hour(DateTime.utc_now()) + points = Grid.conus_points() + + Logger.info("PropagationGrid: fetching HRRR for #{valid_time}, #{length(points)} points") + + with {:ok, grid_data} <- HrrrClient.fetch_grid(points, valid_time) do + scores = + grid_data + |> Task.async_stream( + fn {{lat, lon}, profile} -> + band_scores = Propagation.score_grid_point(profile, valid_time) + + Enum.map(band_scores, fn result -> + %{ + lat: lat, + lon: lon, + valid_time: valid_time, + band_mhz: result.band_mhz, + score: result.score, + factors: result.factors + } + end) + end, + max_concurrency: System.schedulers_online() * 2, + timeout: 30_000 + ) + |> Enum.flat_map(fn + {:ok, results} -> results + {:exit, _reason} -> [] + end) + + Logger.info("PropagationGrid: computed #{length(scores)} scores, upserting") + + case Propagation.upsert_scores(scores) do + {:ok, count} -> + Logger.info("PropagationGrid: upserted #{count} scores for #{valid_time}") + :ok + + error -> + Logger.error("PropagationGrid: upsert failed: #{inspect(error)}") + error + end + end + end +end +``` + +### Step 3: Update config.exs — add queue and cron + +In `config/config.exs`, update the Oban config: + +Add `:propagation` to the queues: +```elixir +queues: [solar: 1, weather: 3, enqueue: 1, hrrr: 20, terrain: 4, commercial: 2, iemre: 5, propagation: 1], +``` + +Add cron entry: +```elixir +{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker} +``` + +(Runs at :05 past each hour, giving HRRR ~5 min to publish) + +### Step 4: Run tests + +Run: `mix test test/microwaveprop/workers/propagation_grid_worker_test.exs --trace` +Expected: PASS + +### Step 5: Commit + +```bash +git add lib/microwaveprop/workers/propagation_grid_worker.ex \ + config/config.exs \ + test/microwaveprop/workers/propagation_grid_worker_test.exs +git commit -m "Add hourly propagation grid worker with Oban cron scheduling" +``` + +--- + +## Task 8: Refactor Existing HRRR Fetch to Use Batch + +Update the existing `HrrrFetchWorker` to leverage the new batch extraction when multiple QSO points need the same HRRR hour. This deduplicates downloads. + +**Files:** +- Modify: `lib/microwaveprop/workers/hrrr_fetch_worker.ex` +- Modify: `lib/microwaveprop/workers/qso_weather_enqueue_worker.ex` +- Test: existing tests should still pass + +### Step 1: Read and understand the existing workers + +Read: `lib/microwaveprop/workers/hrrr_fetch_worker.ex` and `lib/microwaveprop/workers/qso_weather_enqueue_worker.ex` + +### Step 2: Refactor approach + +The key change: instead of one Oban job per (lat, lon, hour), group QSO points by HRRR hour and create one job per hour that extracts all points for that hour. + +Update `QsoWeatherEnqueueWorker` to group HRRR points by hour before enqueuing. The new `HrrrFetchWorker` accepts a list of points instead of a single point. + +**Note:** Keep backward compatibility — if `args` contains a single `lat`/`lon`, treat it as a single-point fetch. If `args` contains `points` list, use batch fetch. + +```elixir +# In HrrrFetchWorker, update perform/1: +def perform(%Oban.Job{args: %{"points" => points, "valid_time" => vt_str} = _args}) do + # Batch mode: fetch once, extract many + valid_time = DateTime.from_iso8601(vt_str) |> elem(1) + point_tuples = Enum.map(points, fn %{"lat" => lat, "lon" => lon} -> {lat, lon} end) + + case HrrrClient.fetch_grid(point_tuples, valid_time) do + {:ok, grid_data} -> + # Store each point as an hrrr_profile + Enum.each(grid_data, fn {{lat, lon}, profile} -> + store_hrrr_profile(lat, lon, valid_time, profile) + end) + :ok + + {:error, reason} -> + {:error, reason} + end +end + +# Keep existing single-point perform clause for backward compat +def perform(%Oban.Job{args: %{"lat" => lat, "lon" => lon, "valid_time" => vt_str}}) do + # Legacy single-point mode — delegates to batch with one point + # ... existing logic unchanged ... +end +``` + +### Step 3: Run all existing tests + +Run: `mix test --trace` +Expected: All pass (backward compatible) + +### Step 4: Commit + +```bash +git add lib/microwaveprop/workers/hrrr_fetch_worker.ex \ + lib/microwaveprop/workers/qso_weather_enqueue_worker.ex +git commit -m "Refactor HRRR fetch worker to batch points per hour, deduplicating downloads" +``` + +--- + +## Task 9: Leaflet Integration + Map LiveView + +Add Leaflet to the project and build the `/map` LiveView with band selector and color-coded score overlay. + +**Files:** +- Create: `assets/vendor/leaflet/` (vendored Leaflet JS + CSS) +- Modify: `assets/js/app.js` (import Leaflet) +- Modify: `assets/css/app.css` (import Leaflet CSS) +- Create: `lib/microwaveprop_web/live/map_live.ex` +- Create: `lib/microwaveprop_web/live/map_live.hooks.js` +- Modify: `lib/microwaveprop_web/router.ex` (add `/map` route) + +### Step 1: Vendor Leaflet + +Download Leaflet 1.9.4 (latest stable) JS and CSS into `assets/vendor/leaflet/`: + +```bash +mkdir -p assets/vendor/leaflet +curl -sL https://unpkg.com/leaflet@1.9.4/dist/leaflet.js -o assets/vendor/leaflet/leaflet.js +curl -sL https://unpkg.com/leaflet@1.9.4/dist/leaflet.css -o assets/vendor/leaflet/leaflet.css +# Also need the images directory for markers +mkdir -p assets/vendor/leaflet/images +curl -sL https://unpkg.com/leaflet@1.9.4/dist/images/marker-icon.png -o assets/vendor/leaflet/images/marker-icon.png +curl -sL https://unpkg.com/leaflet@1.9.4/dist/images/marker-shadow.png -o assets/vendor/leaflet/images/marker-shadow.png +``` + +### Step 2: Import in app.js and app.css + +In `assets/js/app.js`, add: +```javascript +import L from "../vendor/leaflet/leaflet.js" +window.L = L +``` + +In `assets/css/app.css`, add: +```css +@import "../vendor/leaflet/leaflet.css"; +``` + +### Step 3: Add route + +In `lib/microwaveprop_web/router.ex`, inside the `scope "/", MicrowavepropWeb` block: + +```elixir +live "/map", MapLive +``` + +### Step 4: Create MapLive + +```elixir +defmodule MicrowavepropWeb.MapLive do + use MicrowavepropWeb, :live_view + + alias Microwaveprop.Propagation + alias Microwaveprop.Propagation.BandConfig + + @default_band 10_000 + + @impl true + def mount(_params, _session, socket) do + bands = BandConfig.all_bands() + scores = Propagation.latest_scores(@default_band) + valid_time = Propagation.latest_valid_time() + + socket = + socket + |> assign(:bands, bands) + |> assign(:selected_band, @default_band) + |> assign(:scores, scores) + |> assign(:valid_time, valid_time) + |> assign(:page_title, "Propagation Map") + + if connected?(socket) do + # Refresh every 5 minutes to pick up new hourly data + Process.send_after(self(), :refresh, :timer.minutes(5)) + end + + {:ok, socket} + end + + @impl true + def handle_event("select_band", %{"band" => band_str}, socket) do + band = String.to_integer(band_str) + scores = Propagation.latest_scores(band) + + socket = + socket + |> assign(:selected_band, band) + |> assign(:scores, scores) + |> push_event("update_scores", %{scores: scores}) + + {:noreply, socket} + end + + @impl true + def handle_info(:refresh, socket) do + scores = Propagation.latest_scores(socket.assigns.selected_band) + valid_time = Propagation.latest_valid_time() + Process.send_after(self(), :refresh, :timer.minutes(5)) + + socket = + socket + |> assign(:scores, scores) + |> assign(:valid_time, valid_time) + |> push_event("update_scores", %{scores: scores}) + + {:noreply, socket} + end + + @impl true + def render(assigns) do + ~H""" + +
+
+

CONUS Propagation Map

+ +
+ +
+ +
+ Updated: {Calendar.strftime(@valid_time, "%Y-%m-%d %H:%M UTC")} +
+
+ +
+
+
+ + """ + end +end +``` + +### Step 5: Create colocated JS hook + +Create `lib/microwaveprop_web/live/map_live.hooks.js`: + +```javascript +export const PropagationMap = { + mounted() { + // Initialize Leaflet map centered on CONUS + this.map = L.map(this.el, { + center: [38.0, -96.0], + zoom: 5, + minZoom: 4, + maxZoom: 10 + }) + + L.tileLayer("https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", { + attribution: "© OpenStreetMap contributors", + maxZoom: 19 + }).addTo(this.map) + + // Score layer + this.scoreLayer = L.layerGroup().addTo(this.map) + + // Color scale + this.colorScale = [ + { min: 80, color: "#00ffa3" }, // EXCELLENT + { min: 65, color: "#7dffd4" }, // GOOD + { min: 50, color: "#ffe566" }, // MARGINAL + { min: 33, color: "#ff9044" }, // POOR + { min: 0, color: "#ff4f4f" } // NEGLIGIBLE + ] + + // Load initial scores from data attribute + const initialScores = JSON.parse(this.el.dataset.scores || "[]") + this.renderScores(initialScores) + + // Listen for LiveView score updates + this.handleEvent("update_scores", ({ scores }) => { + this.renderScores(scores) + }) + }, + + renderScores(scores) { + this.scoreLayer.clearLayers() + + scores.forEach(({ lat, lon, score }) => { + const color = this.scoreColor(score) + const circle = L.circleMarker([lat, lon], { + radius: 6, + fillColor: color, + fillOpacity: 0.7, + color: color, + weight: 0, + opacity: 0.7 + }) + + circle.bindPopup( + `Score: ${score}/100
` + + `${this.scoreTier(score)}
` + + `${lat.toFixed(3)}°N, ${Math.abs(lon).toFixed(3)}°W` + ) + + this.scoreLayer.addLayer(circle) + }) + }, + + scoreColor(score) { + for (const tier of this.colorScale) { + if (score >= tier.min) return tier.color + } + return "#ff4f4f" + }, + + scoreTier(score) { + if (score >= 80) return "EXCELLENT" + if (score >= 65) return "GOOD" + if (score >= 50) return "MARGINAL" + if (score >= 33) return "POOR" + return "NEGLIGIBLE" + }, + + destroyed() { + if (this.map) { + this.map.remove() + } + } +} +``` + +### Step 6: Run dev server and verify + +Run: `mix phx.server` +Visit: `http://localhost:4000/map` +Expected: Leaflet map loads, band selector buttons render, map shows score overlay (empty until worker runs) + +### Step 7: Commit + +```bash +git add assets/vendor/leaflet/ \ + assets/js/app.js \ + assets/css/app.css \ + lib/microwaveprop_web/live/map_live.ex \ + lib/microwaveprop_web/live/map_live.hooks.js \ + lib/microwaveprop_web/router.ex +git commit -m "Add Leaflet-based CONUS propagation map at /map with band selector" +``` + +--- + +## Task 10: Add Legend and Polish + +Add a color legend overlay to the map and a link in the navigation. + +**Files:** +- Modify: `lib/microwaveprop_web/live/map_live.ex` (add legend markup) +- Modify: `lib/microwaveprop_web/live/map_live.hooks.js` (add Leaflet legend control) +- Modify: `lib/microwaveprop_web/components/layouts.ex` or nav template (add /map link) + +### Step 1: Add legend to the JS hook + +In the `mounted()` function of `map_live.hooks.js`, add after the map initialization: + +```javascript +// Add legend control +const legend = L.control({ position: "bottomright" }) +legend.onAdd = () => { + const div = L.DomUtil.create("div", "leaflet-legend") + div.innerHTML = ` +
+ Propagation
+ Excellent (80-100)
+ Good (65-79)
+ Marginal (50-64)
+ Poor (33-49)
+ Negligible (0-32) +
+ ` + return div +} +legend.addTo(this.map) +``` + +### Step 2: Add nav link + +In the layout/nav, add a link to `/map`: +```elixir +<.link navigate={~p"/map"} class="btn btn-ghost btn-sm">Map +``` + +### Step 3: Commit + +```bash +git add lib/microwaveprop_web/live/map_live.ex \ + lib/microwaveprop_web/live/map_live.hooks.js \ + lib/microwaveprop_web/components/layouts.ex +git commit -m "Add map legend and navigation link" +``` + +--- + +## Task 11: Manual Trigger and End-to-End Test + +Add a way to manually trigger the propagation grid computation (for testing without waiting for the cron), and write an integration test. + +**Files:** +- Create: `lib/mix/tasks/propagation_grid.ex` +- Test: `test/microwaveprop/propagation/integration_test.exs` + +### Step 1: Create mix task + +```elixir +defmodule Mix.Tasks.PropagationGrid do + @moduledoc "Manually trigger propagation grid computation." + use Mix.Task + + @shortdoc "Compute propagation scores for the CONUS grid" + + @impl Mix.Task + def run(_args) do + Mix.Task.run("app.start") + + IO.puts("Starting propagation grid computation...") + case Microwaveprop.Workers.PropagationGridWorker.perform(%Oban.Job{args: %{}}) do + :ok -> IO.puts("Done!") + {:error, reason} -> IO.puts("Error: #{inspect(reason)}") + end + end +end +``` + +### Step 2: Commit + +```bash +git add lib/mix/tasks/propagation_grid.ex +git commit -m "Add mix task to manually trigger propagation grid computation" +``` + +--- + +## Task 12: Run precommit and fix issues + +### Step 1: Format and check + +Run: `mix format` +Run: `mix credo` +Run: `mix precommit` + +Fix any issues that arise. + +### Step 2: Final commit + +```bash +git add -A +git commit -m "Fix formatting and credo issues" +``` + +--- + +## Summary of Key Design Decisions + +1. **Algorithm is data-driven**: All weights, thresholds, band configs, and seasonal tables live in `BandConfig`. Updating the algorithm = editing one file. Scoring functions in `Scorer` are generic readers of that config. + +2. **HRRR download-once-extract-many**: Single GRIB2 download per hour serves both the map grid and QSO enrichment. `Extractor.extract_grid/2` replaces per-point extraction. + +3. **Pre-computed scores**: Scores are computed hourly by the Oban worker and stored in `propagation_scores`. The LiveView reads from the DB — no on-the-fly computation at page load. + +4. **Simple overlay**: CircleMarkers at 0.125° grid points, colored by score tier. This is efficient for ~10k points and looks similar to the reference screenshot's blob visualization. + +5. **Future extensibility**: Adding forecast hours = fetch HRRR f01-f18 products in the worker. Adding finer resolution = change `Grid.step`. Adding new bands = add entry to `BandConfig`. Adding new scoring factors = add to `BandConfig.weights` and `Scorer`. diff --git a/lib/microwaveprop/workers/hrrr_fetch_worker.ex b/lib/microwaveprop/workers/hrrr_fetch_worker.ex index 98e33a86..66efb428 100644 --- a/lib/microwaveprop/workers/hrrr_fetch_worker.ex +++ b/lib/microwaveprop/workers/hrrr_fetch_worker.ex @@ -15,9 +15,38 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do end @impl Oban.Worker - def perform(%Oban.Job{args: args}) do - %{"lat" => raw_lat, "lon" => raw_lon, "valid_time" => valid_time_str} = args + def perform(%Oban.Job{args: %{"points" => points, "valid_time" => valid_time_str}}) do + # Batch mode: download GRIB2 once, extract multiple points + {:ok, valid_time, _} = DateTime.from_iso8601(valid_time_str) + point_tuples = + points + |> Enum.map(fn %{"lat" => lat, "lon" => lon} -> {lat, lon} end) + |> Enum.reject(fn {lat, lon} -> Weather.has_hrrr_profile?(lat, lon, valid_time) end) + + if point_tuples == [] do + Logger.info("HRRR batch: all #{length(points)} points already exist for #{valid_time_str}") + :ok + else + Logger.info("HRRR batch: fetching #{length(point_tuples)} points for #{valid_time_str}") + + case HrrrClient.fetch_grid(point_tuples, valid_time) do + {:ok, grid_data} -> + Enum.each(grid_data, fn {{lat, lon}, data} -> + store_profile(lat, lon, valid_time, data) + end) + + Logger.info("HRRR batch: saved #{map_size(grid_data)} profiles for #{valid_time_str}") + :ok + + {:error, reason} -> + handle_error(reason, "batch @ #{valid_time_str}") + end + end + end + + def perform(%Oban.Job{args: %{"lat" => raw_lat, "lon" => raw_lon, "valid_time" => valid_time_str}}) do + # Legacy single-point mode {:ok, valid_time, _} = DateTime.from_iso8601(valid_time_str) {lat, lon} = Weather.round_to_hrrr_grid(raw_lat, raw_lon) @@ -29,44 +58,48 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do case HrrrClient.fetch_profile(lat, lon, valid_time) do {:ok, data} -> - params = SoundingParams.derive(data.profile) - levels = length(data.profile) - - attrs = - maybe_add_derived_params( - %{ - valid_time: valid_time, - lat: lat, - lon: lon, - run_time: data.run_time, - profile: data.profile, - hpbl_m: data.hpbl_m, - pwat_mm: data.pwat_mm, - surface_temp_c: data.surface_temp_c, - surface_dewpoint_c: data.surface_dewpoint_c, - surface_pressure_mb: data.surface_pressure_mb - }, - params - ) - - Weather.upsert_hrrr_profile(attrs) - - Logger.info("HRRR profile saved for #{lat},#{lon} @ #{valid_time_str} (#{levels} levels, run #{data.run_time})") - + store_profile(lat, lon, valid_time, data) :ok {:error, reason} -> - if transient_failure?(reason) do - Logger.error("HRRR transient error for #{lat},#{lon} @ #{valid_time_str}: #{inspect(reason)}") - {:error, reason} - else - Logger.warning("HRRR permanent failure for #{lat},#{lon} @ #{valid_time_str}: #{inspect(reason)}") - {:cancel, reason} - end + handle_error(reason, "#{lat},#{lon} @ #{valid_time_str}") end end end + defp store_profile(lat, lon, valid_time, data) do + params = SoundingParams.derive(data.profile) + + attrs = + maybe_add_derived_params( + %{ + valid_time: valid_time, + lat: lat, + lon: lon, + run_time: data.run_time, + profile: data.profile, + hpbl_m: data.hpbl_m, + pwat_mm: data.pwat_mm, + surface_temp_c: data.surface_temp_c, + surface_dewpoint_c: data.surface_dewpoint_c, + surface_pressure_mb: data.surface_pressure_mb + }, + params + ) + + Weather.upsert_hrrr_profile(attrs) + end + + defp handle_error(reason, label) do + if transient_failure?(reason) do + Logger.error("HRRR transient error for #{label}: #{inspect(reason)}") + {:error, reason} + else + Logger.warning("HRRR permanent failure for #{label}: #{inspect(reason)}") + {:cancel, reason} + end + end + # Only retry on transient network/server errors — everything else is permanent defp transient_failure?(%{__exception__: true}), do: true defp transient_failure?("HRRR idx HTTP " <> status), do: server_error?(status) diff --git a/lib/microwaveprop/workers/qso_weather_enqueue_worker.ex b/lib/microwaveprop/workers/qso_weather_enqueue_worker.ex index e9bb6c06..720789ee 100644 --- a/lib/microwaveprop/workers/qso_weather_enqueue_worker.ex +++ b/lib/microwaveprop/workers/qso_weather_enqueue_worker.ex @@ -100,9 +100,22 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorker do end def build_hrrr_jobs(qsos) do + # Group all QSO points by HRRR hour, then create one batch job per hour + # instead of one job per point. This deduplicates GRIB2 downloads. qsos - |> Enum.flat_map(&hrrr_job_for_qso/1) - |> Enum.uniq_by(fn changeset -> changeset.changes.args end) + |> Enum.flat_map(&hrrr_points_for_qso/1) + |> Enum.group_by(fn {_point, hour} -> hour end, fn {point, _hour} -> point end) + |> Enum.map(fn {hour, points} -> + unique_points = + points + |> Enum.uniq() + |> Enum.map(fn {lat, lon} -> %{"lat" => lat, "lon" => lon} end) + + HrrrFetchWorker.new(%{ + "points" => unique_points, + "valid_time" => DateTime.to_iso8601(hour) + }) + end) end def build_iemre_jobs(qsos) do @@ -139,9 +152,9 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorker do end end - defp hrrr_job_for_qso(%{pos1: nil}), do: [] + defp hrrr_points_for_qso(%{pos1: nil}), do: [] - defp hrrr_job_for_qso(qso) do + defp hrrr_points_for_qso(qso) do rounded_time = HrrrClient.nearest_hrrr_hour(qso.qso_timestamp) qso @@ -152,13 +165,7 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorker do if Weather.has_hrrr_profile?(rlat, rlon, rounded_time) do [] else - [ - HrrrFetchWorker.new(%{ - "lat" => rlat, - "lon" => rlon, - "valid_time" => DateTime.to_iso8601(rounded_time) - }) - ] + [{{rlat, rlon}, rounded_time}] end end) end diff --git a/test/microwaveprop/workers/qso_weather_enqueue_worker_test.exs b/test/microwaveprop/workers/qso_weather_enqueue_worker_test.exs index 1a46932f..6b28586c 100644 --- a/test/microwaveprop/workers/qso_weather_enqueue_worker_test.exs +++ b/test/microwaveprop/workers/qso_weather_enqueue_worker_test.exs @@ -255,26 +255,31 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do end describe "build_hrrr_jobs/1" do - test "builds HRRR jobs for all path points (pos1, midpoint, pos2)" do + test "batches all path points into one job per HRRR hour" do qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}}) jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso]) - # pos1, midpoint, pos2 = 3 distinct grid points - assert length(jobs) == 3 - lats = jobs |> Enum.map(& &1.changes.args["lat"]) |> Enum.sort() + # All 3 path points share the same HRRR hour → 1 batch job + assert length(jobs) == 1 + job = hd(jobs) + points = job.changes.args["points"] + # pos1, midpoint, pos2 = 3 distinct grid points in the batch + assert length(points) == 3 + lats = Enum.map(points, & &1["lat"]) assert 32.91 in lats end - test "builds only one HRRR job when pos2 is nil" do + test "batch contains single point when pos2 is nil" do qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}, pos2: nil}) jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso]) assert length(jobs) == 1 job = hd(jobs) - assert job.changes.args["lat"] == 32.91 - assert job.changes.args["lon"] == -97.04 + [point] = job.changes.args["points"] + assert point["lat"] == 32.91 + assert point["lon"] == -97.04 end test "rounds valid_time to nearest hour" do @@ -329,7 +334,7 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do assert QsoWeatherEnqueueWorker.build_hrrr_jobs([qso]) == [] end - test "skips only path points where HRRR profile already exists" do + test "excludes existing points from batch" do qso = create_qso(%{pos1: %{"lat" => 32.907, "lon" => -97.038}}) # Insert a profile at the pos1 rounded grid point only @@ -348,9 +353,11 @@ defmodule Microwaveprop.Workers.QsoWeatherEnqueueWorkerTest do jobs = QsoWeatherEnqueueWorker.build_hrrr_jobs([qso]) - # pos1 skipped (exists), midpoint + pos2 still need fetching - assert length(jobs) == 2 - lats = Enum.map(jobs, & &1.changes.args["lat"]) + # 1 batch job, but pos1 excluded from points list + assert length(jobs) == 1 + points = hd(jobs).changes.args["points"] + assert length(points) == 2 + lats = Enum.map(points, & &1["lat"]) refute 32.91 in lats end end