Graham McIntire
fc3eb58910
Direct queries against prod (75M HRRR profiles, 16,864 soundings, 392k surface obs, new hrrr_native_profiles table) drive these changes: * Reinstate shallow-BL bonus as an HPBL multiplier on the refractivity score (1.10× <200m → 0.78× ≥2000m). The Apr 11 "shallow BL bonus removed" conclusion was an artifact of the prior matching strategy; with the cleaner contact↔HRRR join (n=680) the binned data goes 230 km avg at HPBL <200m vs 100 km at ≥2000m, monotonic across all bins. * Add 6 missing bands (142, 145, 288, 322, 403, 411 GHz) so contacts in those bands stop being silently dropped. Coefficients extrapolate ITU-R P.676/P.838 trends from the existing 134/241 GHz entries. * Bump ERA5 poll timeout 10 min → 1 hour and Era5MonthBatchWorker max_attempts 3 → 5 so CDS slowness stops discarding tiles. Also dump the full failure body when CDS omits the message field — the prior "ERA5 job failed: nil" was masking real error reasons in oban_jobs. * Add scripts/recalibrate_algo.py so this analysis can be re-run any time new data lands without manual SQL. Reads PROP_PROD_DB_URL from .envrc, drops a Markdown report into docs/algo-reports/. * Append a dated Part 2c to algo.md documenting the corpus expansion, the honest accounting of historical HRRR coverage (only ~1,020 of 58k contacts are precision-matchable), and the empty era5/rtma/climatology tables. Update the gaseous-absorption and rain-attenuation tables to include the new bands. Test suite: 1,335 tests, 0 failures.
649 lines
15 KiB
Elixir
649 lines
15 KiB
Elixir
defmodule Microwaveprop.Propagation.BandConfig do
|
||
@moduledoc """
|
||
Data-driven band configuration for the propagation scoring algorithm.
|
||
|
||
This is the single source of truth for all scoring parameters: weights,
|
||
thresholds, seasonal tables, and band-specific coefficients. When the
|
||
algorithm is tuned or new bands are added, only this module changes.
|
||
"""
|
||
|
||
# Recalibrated 2026-04-11 via gradient descent on 5000 QSOs (loss 0.42 → 0.12).
|
||
# Key shifts: rain +70%, season +39%, wind +60%; time_of_day -50%, pressure -31%.
|
||
@weights %{
|
||
humidity: 0.1243,
|
||
time_of_day: 0.0496,
|
||
td_depression: 0.0978,
|
||
refractivity: 0.1049,
|
||
sky: 0.08,
|
||
season: 0.1112,
|
||
wind: 0.08,
|
||
rain: 0.1362,
|
||
pressure: 0.1032,
|
||
pwat: 0.1128
|
||
}
|
||
|
||
@sunrise_table [7.4, 7.3, 7.0, 6.7, 6.35, 6.25, 6.35, 6.65, 6.9, 7.1, 7.35, 7.45]
|
||
|
||
@tiers [
|
||
%{min_score: 80, label: "EXCELLENT", color: "#059669"},
|
||
%{min_score: 65, label: "GOOD", color: "#0d9488"},
|
||
%{min_score: 50, label: "MARGINAL", color: "#ca8a04"},
|
||
%{min_score: 33, label: "POOR", color: "#ea580c"},
|
||
%{min_score: 0, label: "NEGLIGIBLE", color: "#dc2626"}
|
||
]
|
||
|
||
@humidity_beneficial_thresholds [{4, 55}, {7, 70}, {10, 82}, {14, 90}, {18, 95}, {22, 88}]
|
||
@humidity_beneficial_default 75
|
||
|
||
# Thresholds calibrated for HRRR-derived gradients (coarser than sounding data).
|
||
# HRRR percentiles: p1=-230, p5=-162, p10=-130, p25=-94, p50=-70, p75=-53, p95=-40
|
||
@refractivity_thresholds [
|
||
{-200, 98, 85},
|
||
{-150, 92, 80},
|
||
{-100, 82, 72},
|
||
{-75, 68, 62},
|
||
{-55, 55, 55},
|
||
{-40, 48, 48}
|
||
]
|
||
@refractivity_default {42, 42}
|
||
|
||
@band_configs %{
|
||
902 => %{
|
||
freq_mhz: 902,
|
||
label: "902 MHz",
|
||
o2_db_km: 0.006,
|
||
h2o_coeff: 0.0,
|
||
humidity_effect: :beneficial,
|
||
humidity_penalty: 0.0,
|
||
rain_k: 0.000,
|
||
rain_alpha: 1.0,
|
||
seasonal_base: %{
|
||
1 => 38,
|
||
2 => 40,
|
||
3 => 22,
|
||
4 => 55,
|
||
5 => 68,
|
||
6 => 90,
|
||
7 => 95,
|
||
8 => 75,
|
||
9 => 78,
|
||
10 => 88,
|
||
11 => 78,
|
||
12 => 25
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 400,
|
||
extended_range_km: 800,
|
||
exceptional_range_km: 1500
|
||
},
|
||
1_296 => %{
|
||
freq_mhz: 1_296,
|
||
label: "1296 MHz",
|
||
o2_db_km: 0.006,
|
||
h2o_coeff: 0.0,
|
||
humidity_effect: :beneficial,
|
||
humidity_penalty: 0.0,
|
||
rain_k: 0.000,
|
||
rain_alpha: 1.0,
|
||
seasonal_base: %{
|
||
1 => 38,
|
||
2 => 40,
|
||
3 => 22,
|
||
4 => 55,
|
||
5 => 68,
|
||
6 => 90,
|
||
7 => 95,
|
||
8 => 75,
|
||
9 => 78,
|
||
10 => 88,
|
||
11 => 78,
|
||
12 => 25
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 350,
|
||
extended_range_km: 700,
|
||
exceptional_range_km: 1200
|
||
},
|
||
2_304 => %{
|
||
freq_mhz: 2_304,
|
||
label: "2304 MHz",
|
||
o2_db_km: 0.006,
|
||
h2o_coeff: 0.0,
|
||
humidity_effect: :beneficial,
|
||
humidity_penalty: 0.0,
|
||
rain_k: 0.001,
|
||
rain_alpha: 1.15,
|
||
seasonal_base: %{
|
||
1 => 38,
|
||
2 => 40,
|
||
3 => 22,
|
||
4 => 55,
|
||
5 => 68,
|
||
6 => 90,
|
||
7 => 95,
|
||
8 => 75,
|
||
9 => 78,
|
||
10 => 88,
|
||
11 => 78,
|
||
12 => 25
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 300,
|
||
extended_range_km: 600,
|
||
exceptional_range_km: 1000
|
||
},
|
||
3_456 => %{
|
||
freq_mhz: 3_456,
|
||
label: "3456 MHz",
|
||
o2_db_km: 0.006,
|
||
h2o_coeff: 0.0,
|
||
humidity_effect: :beneficial,
|
||
humidity_penalty: 0.0,
|
||
rain_k: 0.002,
|
||
rain_alpha: 1.20,
|
||
seasonal_base: %{
|
||
1 => 38,
|
||
2 => 40,
|
||
3 => 22,
|
||
4 => 55,
|
||
5 => 68,
|
||
6 => 90,
|
||
7 => 95,
|
||
8 => 75,
|
||
9 => 78,
|
||
10 => 88,
|
||
11 => 78,
|
||
12 => 25
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 250,
|
||
extended_range_km: 550,
|
||
exceptional_range_km: 900
|
||
},
|
||
5_760 => %{
|
||
freq_mhz: 5_760,
|
||
label: "5760 MHz",
|
||
o2_db_km: 0.007,
|
||
h2o_coeff: 0.0,
|
||
humidity_effect: :beneficial,
|
||
humidity_penalty: 0.0,
|
||
rain_k: 0.005,
|
||
rain_alpha: 1.25,
|
||
seasonal_base: %{
|
||
1 => 38,
|
||
2 => 40,
|
||
3 => 22,
|
||
4 => 55,
|
||
5 => 68,
|
||
6 => 90,
|
||
7 => 95,
|
||
8 => 75,
|
||
9 => 78,
|
||
10 => 88,
|
||
11 => 78,
|
||
12 => 25
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 220,
|
||
extended_range_km: 500,
|
||
exceptional_range_km: 1000
|
||
},
|
||
10_000 => %{
|
||
freq_mhz: 10_000,
|
||
label: "10 GHz",
|
||
o2_db_km: 0.007,
|
||
h2o_coeff: 0.0,
|
||
humidity_effect: :beneficial,
|
||
humidity_penalty: 0.0,
|
||
rain_k: 0.010,
|
||
rain_alpha: 1.28,
|
||
seasonal_base: %{
|
||
1 => 38,
|
||
2 => 40,
|
||
3 => 22,
|
||
4 => 55,
|
||
5 => 68,
|
||
6 => 90,
|
||
7 => 95,
|
||
8 => 75,
|
||
9 => 78,
|
||
10 => 88,
|
||
11 => 78,
|
||
12 => 25
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 200,
|
||
extended_range_km: 500,
|
||
exceptional_range_km: 1000
|
||
},
|
||
24_000 => %{
|
||
freq_mhz: 24_000,
|
||
label: "24 GHz",
|
||
o2_db_km: 0.02,
|
||
h2o_coeff: 0.002,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 1.6,
|
||
rain_k: 0.070,
|
||
rain_alpha: 1.07,
|
||
seasonal_base: %{
|
||
1 => 88,
|
||
2 => 84,
|
||
3 => 68,
|
||
4 => 62,
|
||
5 => 51,
|
||
6 => 34,
|
||
7 => 18,
|
||
8 => 18,
|
||
9 => 48,
|
||
10 => 68,
|
||
11 => 96,
|
||
12 => 88
|
||
},
|
||
seasonal_adj: %{5 => -4, 6 => -8, 7 => -10, 8 => -10, 9 => -4},
|
||
typical_range_km: 100,
|
||
extended_range_km: 250,
|
||
exceptional_range_km: 500
|
||
},
|
||
47_000 => %{
|
||
freq_mhz: 47_000,
|
||
label: "47 GHz",
|
||
o2_db_km: 0.04,
|
||
h2o_coeff: 0.003,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 1.0,
|
||
rain_k: 0.187,
|
||
rain_alpha: 0.93,
|
||
seasonal_base: %{
|
||
1 => 90,
|
||
2 => 88,
|
||
3 => 78,
|
||
4 => 68,
|
||
5 => 55,
|
||
6 => 38,
|
||
7 => 22,
|
||
8 => 22,
|
||
9 => 48,
|
||
10 => 74,
|
||
11 => 96,
|
||
12 => 90
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 70,
|
||
extended_range_km: 150,
|
||
exceptional_range_km: 300
|
||
},
|
||
68_000 => %{
|
||
freq_mhz: 68_000,
|
||
label: "68 GHz",
|
||
o2_db_km: 0.90,
|
||
h2o_coeff: 0.007,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 1.4,
|
||
rain_k: 0.310,
|
||
rain_alpha: 0.86,
|
||
seasonal_base: %{
|
||
1 => 90,
|
||
2 => 88,
|
||
3 => 78,
|
||
4 => 65,
|
||
5 => 50,
|
||
6 => 32,
|
||
7 => 18,
|
||
8 => 18,
|
||
9 => 44,
|
||
10 => 70,
|
||
11 => 92,
|
||
12 => 90
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 40,
|
||
extended_range_km: 80,
|
||
exceptional_range_km: 150
|
||
},
|
||
75_000 => %{
|
||
freq_mhz: 75_000,
|
||
label: "75 GHz",
|
||
o2_db_km: 0.012,
|
||
h2o_coeff: 0.006,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 1.2,
|
||
rain_k: 0.345,
|
||
rain_alpha: 0.84,
|
||
seasonal_base: %{
|
||
1 => 90,
|
||
2 => 90,
|
||
3 => 80,
|
||
4 => 68,
|
||
5 => 55,
|
||
6 => 38,
|
||
7 => 22,
|
||
8 => 22,
|
||
9 => 48,
|
||
10 => 74,
|
||
11 => 96,
|
||
12 => 90
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 50,
|
||
extended_range_km: 120,
|
||
exceptional_range_km: 250
|
||
},
|
||
122_000 => %{
|
||
freq_mhz: 122_000,
|
||
label: "122 GHz",
|
||
o2_db_km: 0.80,
|
||
h2o_coeff: 0.010,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 1.0,
|
||
rain_k: 0.498,
|
||
rain_alpha: 0.77,
|
||
seasonal_base: %{
|
||
1 => 92,
|
||
2 => 90,
|
||
3 => 78,
|
||
4 => 62,
|
||
5 => 45,
|
||
6 => 28,
|
||
7 => 15,
|
||
8 => 15,
|
||
9 => 38,
|
||
10 => 68,
|
||
11 => 92,
|
||
12 => 92
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 30,
|
||
extended_range_km: 80,
|
||
exceptional_range_km: 140
|
||
},
|
||
134_000 => %{
|
||
freq_mhz: 134_000,
|
||
label: "134 GHz",
|
||
o2_db_km: 0.08,
|
||
h2o_coeff: 0.015,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 1.3,
|
||
rain_k: 0.520,
|
||
rain_alpha: 0.75,
|
||
seasonal_base: %{
|
||
1 => 92,
|
||
2 => 90,
|
||
3 => 78,
|
||
4 => 65,
|
||
5 => 48,
|
||
6 => 30,
|
||
7 => 18,
|
||
8 => 18,
|
||
9 => 42,
|
||
10 => 70,
|
||
11 => 92,
|
||
12 => 92
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 40,
|
||
extended_range_km: 100,
|
||
exceptional_range_km: 160
|
||
},
|
||
# 142–145 GHz — between the 118 GHz O2 line and the 183 GHz H2O line.
|
||
# Coefficients linearly interpolate between 134 GHz and 241 GHz; rain
|
||
# tracks ITU-R P.838 in the K-band-extrapolated regime. Ranges scaled
|
||
# from the handful of contacts in the corpus (n=4 at 142 GHz, n=2 at
|
||
# 145 GHz) — these are scaffolding values and will need calibration as
|
||
# the band sees more activity.
|
||
142_000 => %{
|
||
freq_mhz: 142_000,
|
||
label: "142 GHz",
|
||
o2_db_km: 0.05,
|
||
h2o_coeff: 0.025,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 1.4,
|
||
rain_k: 0.530,
|
||
rain_alpha: 0.74,
|
||
seasonal_base: %{
|
||
1 => 92,
|
||
2 => 90,
|
||
3 => 78,
|
||
4 => 65,
|
||
5 => 47,
|
||
6 => 28,
|
||
7 => 16,
|
||
8 => 16,
|
||
9 => 40,
|
||
10 => 68,
|
||
11 => 92,
|
||
12 => 92
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 35,
|
||
extended_range_km: 80,
|
||
exceptional_range_km: 160
|
||
},
|
||
145_000 => %{
|
||
freq_mhz: 145_000,
|
||
label: "145 GHz",
|
||
o2_db_km: 0.06,
|
||
h2o_coeff: 0.040,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 1.5,
|
||
rain_k: 0.535,
|
||
rain_alpha: 0.74,
|
||
seasonal_base: %{
|
||
1 => 92,
|
||
2 => 90,
|
||
3 => 78,
|
||
4 => 64,
|
||
5 => 46,
|
||
6 => 27,
|
||
7 => 15,
|
||
8 => 15,
|
||
9 => 38,
|
||
10 => 66,
|
||
11 => 92,
|
||
12 => 92
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 35,
|
||
extended_range_km: 80,
|
||
exceptional_range_km: 150
|
||
},
|
||
241_000 => %{
|
||
freq_mhz: 241_000,
|
||
label: "241 GHz",
|
||
o2_db_km: 0.08,
|
||
h2o_coeff: 0.30,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 3.0,
|
||
rain_k: 0.550,
|
||
rain_alpha: 0.70,
|
||
seasonal_base: %{
|
||
1 => 95,
|
||
2 => 92,
|
||
3 => 75,
|
||
4 => 55,
|
||
5 => 35,
|
||
6 => 15,
|
||
7 => 8,
|
||
8 => 8,
|
||
9 => 30,
|
||
10 => 65,
|
||
11 => 95,
|
||
12 => 95
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 10,
|
||
extended_range_km: 50,
|
||
exceptional_range_km: 115
|
||
},
|
||
# Sub-mm bands (288–411 GHz) — extrapolated scaffolding so the scoring
|
||
# pipeline doesn't silently drop these contacts. The prod corpus has 1
|
||
# contact per band, all under 1.5 km (effectively LOS), so the absorption
|
||
# values dominate any duct-related score. Coefficients track ITU-R P.676/
|
||
# 838 trends past the 325 GHz H2O line; ranges are nominal.
|
||
288_000 => %{
|
||
freq_mhz: 288_000,
|
||
label: "288 GHz",
|
||
o2_db_km: 0.10,
|
||
h2o_coeff: 0.45,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 3.5,
|
||
rain_k: 0.560,
|
||
rain_alpha: 0.68,
|
||
seasonal_base: %{
|
||
1 => 95,
|
||
2 => 92,
|
||
3 => 75,
|
||
4 => 55,
|
||
5 => 35,
|
||
6 => 14,
|
||
7 => 7,
|
||
8 => 7,
|
||
9 => 28,
|
||
10 => 64,
|
||
11 => 95,
|
||
12 => 95
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 5,
|
||
extended_range_km: 15,
|
||
exceptional_range_km: 50
|
||
},
|
||
322_000 => %{
|
||
freq_mhz: 322_000,
|
||
label: "322 GHz",
|
||
o2_db_km: 0.12,
|
||
h2o_coeff: 0.55,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 4.0,
|
||
rain_k: 0.570,
|
||
rain_alpha: 0.66,
|
||
seasonal_base: %{
|
||
1 => 96,
|
||
2 => 92,
|
||
3 => 74,
|
||
4 => 52,
|
||
5 => 32,
|
||
6 => 12,
|
||
7 => 6,
|
||
8 => 6,
|
||
9 => 26,
|
||
10 => 62,
|
||
11 => 96,
|
||
12 => 96
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 5,
|
||
extended_range_km: 15,
|
||
exceptional_range_km: 40
|
||
},
|
||
403_000 => %{
|
||
freq_mhz: 403_000,
|
||
label: "403 GHz",
|
||
o2_db_km: 0.15,
|
||
h2o_coeff: 0.40,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 3.0,
|
||
rain_k: 0.580,
|
||
rain_alpha: 0.64,
|
||
seasonal_base: %{
|
||
1 => 96,
|
||
2 => 92,
|
||
3 => 74,
|
||
4 => 52,
|
||
5 => 32,
|
||
6 => 12,
|
||
7 => 6,
|
||
8 => 6,
|
||
9 => 26,
|
||
10 => 62,
|
||
11 => 96,
|
||
12 => 96
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 3,
|
||
extended_range_km: 10,
|
||
exceptional_range_km: 30
|
||
},
|
||
411_000 => %{
|
||
freq_mhz: 411_000,
|
||
label: "411 GHz",
|
||
o2_db_km: 0.15,
|
||
h2o_coeff: 0.42,
|
||
humidity_effect: :harmful,
|
||
humidity_penalty: 3.2,
|
||
rain_k: 0.580,
|
||
rain_alpha: 0.64,
|
||
seasonal_base: %{
|
||
1 => 96,
|
||
2 => 92,
|
||
3 => 74,
|
||
4 => 52,
|
||
5 => 32,
|
||
6 => 12,
|
||
7 => 6,
|
||
8 => 6,
|
||
9 => 26,
|
||
10 => 62,
|
||
11 => 96,
|
||
12 => 96
|
||
},
|
||
seasonal_adj: %{},
|
||
typical_range_km: 3,
|
||
extended_range_km: 10,
|
||
exceptional_range_km: 30
|
||
}
|
||
}
|
||
|
||
@doc "Returns the band config for the given frequency in MHz, or nil if not found."
|
||
@spec get(integer()) :: map() | nil
|
||
def get(freq_mhz), do: Map.get(@band_configs, freq_mhz)
|
||
|
||
@doc "Returns all band configs sorted by frequency (ascending)."
|
||
@spec all_bands() :: [map()]
|
||
def all_bands do
|
||
@band_configs
|
||
|> Enum.sort_by(fn {freq, _config} -> freq end)
|
||
|> Enum.map(fn {_freq, config} -> config end)
|
||
end
|
||
|
||
@doc "Returns all supported frequencies sorted ascending."
|
||
@spec all_freqs() :: [integer()]
|
||
def all_freqs do
|
||
@band_configs
|
||
|> Map.keys()
|
||
|> Enum.sort()
|
||
end
|
||
|
||
@doc "Returns band options as {label, value_string} tuples for form dropdowns."
|
||
@spec band_options() :: [{String.t(), String.t()}]
|
||
def band_options do
|
||
Enum.map(all_bands(), fn band -> {band.label, to_string(band.freq_mhz)} end)
|
||
end
|
||
|
||
@doc "Returns the scoring weights map. All 10 values sum to 1.0."
|
||
@spec weights() :: map()
|
||
def weights, do: @weights
|
||
|
||
@doc "Returns the 12-element sunrise hour table (Jan-Dec, local time)."
|
||
@spec sunrise_table() :: [float()]
|
||
def sunrise_table, do: @sunrise_table
|
||
|
||
@doc "Returns score tier definitions ordered by threshold descending."
|
||
@spec tiers() :: [map()]
|
||
def tiers, do: @tiers
|
||
|
||
@doc "Returns humidity beneficial thresholds as {hour, score} tuples."
|
||
@spec humidity_beneficial_thresholds() :: [{integer(), integer()}]
|
||
def humidity_beneficial_thresholds, do: @humidity_beneficial_thresholds
|
||
|
||
@doc "Returns the default humidity beneficial score."
|
||
@spec humidity_beneficial_default() :: integer()
|
||
def humidity_beneficial_default, do: @humidity_beneficial_default
|
||
|
||
@doc "Returns refractivity thresholds as {gradient, score_beneficial, score_harmful} tuples."
|
||
@spec refractivity_thresholds() :: [{number(), number(), number()}]
|
||
def refractivity_thresholds, do: @refractivity_thresholds
|
||
|
||
@doc "Returns the default refractivity scores as {beneficial, harmful}."
|
||
@spec refractivity_default() :: {number(), number()}
|
||
def refractivity_default, do: @refractivity_default
|
||
end
|