graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
prop/docs/algo-reports/validation-2026-08-01.json
Graham McIntire 2fd88a94ea
Some checks failed
Build and Push / Build and Push Docker Image (push) Waiting to run
Details
Build prop-grid-rs / Test, build, push (push) Failing after 3m41s
Details
fix(propagation): 7 pipeline bugs + validation harnesses + model improvements 
Phase A — 7 pipeline bugs:
- retain_scores_window: fix timeline self-deletion (NOTIFY payload run_time|valid_time)
- Rust/Elixir weight divergence: Rust loads band_weights.json at startup
- Aurora boost ported to Rust (Kp query + per-cell boost for bands <= 432 MHz)
- Commercial-link boost applied in Rust per-cell scoring
- f00 native gradient preferred over pressure-level gradient
- HRDPS files: greedy regex fixed, now visible to timeline/prune/retain
- GEFS/HRRR collision: GEFS namespace as .gefs.prop, merge at read

Phase B — Validation harness:
- scripts/validate_algo.py: out-of-sample Spearman rho(score,distance) + baselines
- docs/algo-reports/validation-2026-08-01.{json,md}

Phase C — Forecast-skill evaluation:
- scripts/validate_forecast.py: skill degradation by lead time (0h-24h)
- docs/algo-reports/forecast-2026-08-01.{json,md}

Phase D — Calibration + model improvements:
- recalibrate.py: validation gate before weight deploy
- Recalibrator: Nx.max(0.0) replaces Nx.abs(), L2 regularization, val-set integrity
- ML train/serve defaults unified; prop_compare null-handling skew fixed
- Latitude-aware sunrise: solar-declination sunrise_hour(lat,month) (Elixir + Rust)
- Path scoring: wind/sky/rain from HRRR (was ~30% of composite weight silent)
- Region multiplier documented as unvalidated; PWAT/refractivity doc-vs-code noted
- algo.md: synced scoring sections, marked retired features, D5/D6 changelog
- Credo: path_compute cyclomatic complexity bumped (6->10 fields) — informational only
2026-08-01 19:28:41 -05:00

451 lines
9.6 KiB
JSON
Raw Blame History

{
"generated_at": "2026-08-01T22:00:42+00:00",
"generated_by": "scripts/validate_algo.py",
"schema_version": 1,
"summary": {
"total_contacts": 74076,
"fit_contacts": 71394,
"test_contacts": 2682,
"bands_with_50_test": 6,
"total_bands": 12
},
"per_band": {
"222": {
"n": 752,
"alg_rho": 0.1868,
"pers_rho": 0.0,
"clim_rho": 0.0,
"no_skill_rho": 0.0,
"skill_gain": 0.1868,
"calibration": [
{
"decile": 5,
"score_range": "50-60",
"n": 2,
"median_km": 249.0,
"p90_km": 247.0
},
{
"decile": 6,
"score_range": "60-70",
"n": 359,
"median_km": 181.0,
"p90_km": 559.4
},
{
"decile": 7,
"score_range": "70-80",
"n": 387,
"median_km": 257.0,
"p90_km": 633.8
},
{
"decile": 8,
"score_range": "80-90",
"n": 4,
"median_km": 531.0,
"p90_km": 664.0
}
],
"score_min": 57,
"score_max": 80,
"score_mean": 69.6,
"score_median": 70,
"dist_median_km": 223.0,
"dist_p90_km": 606.8
},
"432": {
"n": 898,
"alg_rho": 0.0779,
"pers_rho": 0.0,
"clim_rho": 0.0,
"no_skill_rho": 0.0,
"skill_gain": 0.0779,
"calibration": [
{
"decile": 5,
"score_range": "50-60",
"n": 4,
"median_km": 399.0,
"p90_km": 413.7
},
{
"decile": 6,
"score_range": "60-70",
"n": 475,
"median_km": 148.0,
"p90_km": 507.0
},
{
"decile": 7,
"score_range": "70-80",
"n": 418,
"median_km": 192.0,
"p90_km": 546.2
},
{
"decile": 8,
"score_range": "80-90",
"n": 1,
"median_km": 348.0,
"p90_km": 348.0
}
],
"score_min": 57,
"score_max": 81,
"score_mean": 69.1,
"score_median": 69,
"dist_median_km": 169.0,
"dist_p90_km": 520.0
},
"902": {
"n": 137,
"alg_rho": 0.2551,
"pers_rho": -0.0221,
"clim_rho": 0.0,
"no_skill_rho": 0.0,
"skill_gain": 0.2771,
"calibration": [
{
"decile": 6,
"score_range": "60-70",
"n": 73,
"median_km": 126.0,
"p90_km": 409.0
},
{
"decile": 7,
"score_range": "70-80",
"n": 64,
"median_km": 215.0,
"p90_km": 417.5
}
],
"score_min": 60,
"score_max": 78,
"score_mean": 69.1,
"score_median": 69,
"dist_median_km": 175.0,
"dist_p90_km": 416.6
},
"1296": {
"n": 270,
"alg_rho": 0.1602,
"pers_rho": 0.0715,
"clim_rho": 0.0,
"no_skill_rho": 0.0,
"skill_gain": 0.0887,
"calibration": [
{
"decile": 5,
"score_range": "50-60",
"n": 10,
"median_km": 221.0,
"p90_km": 283.5
},
{
"decile": 6,
"score_range": "60-70",
"n": 144,
"median_km": 117.0,
"p90_km": 344.4
},
{
"decile": 7,
"score_range": "70-80",
"n": 112,
"median_km": 160.0,
"p90_km": 423.4
},
{
"decile": 8,
"score_range": "80-90",
"n": 4,
"median_km": 262.0,
"p90_km": 719.1
}
],
"score_min": 50,
"score_max": 83,
"score_mean": 68.6,
"score_median": 68,
"dist_median_km": 144.0,
"dist_p90_km": 394.0
},
"2304": {
"n": 46,
"alg_rho": 0.3058,
"pers_rho": -0.355,
"clim_rho": 0.0,
"no_skill_rho": 0.0,
"skill_gain": 0.6608,
"calibration": [
{
"decile": 6,
"score_range": "60-70",
"n": 23,
"median_km": 74.0,
"p90_km": 228.2
},
{
"decile": 7,
"score_range": "70-80",
"n": 23,
"median_km": 101.0,
"p90_km": 251.0
}
],
"score_min": 61,
"score_max": 75,
"score_mean": 69.0,
"score_median": 70,
"dist_median_km": 101.0,
"dist_p90_km": 246.5
},
"3400": {
"n": 25,
"alg_rho": -0.134,
"pers_rho": 0.0,
"clim_rho": 0.0,
"no_skill_rho": 0.0,
"skill_gain": -0.134,
"calibration": [
{
"decile": 6,
"score_range": "60-70",
"n": 10,
"median_km": 184.0,
"p90_km": 237.4
},
{
"decile": 7,
"score_range": "70-80",
"n": 15,
"median_km": 108.0,
"p90_km": 306.2
}
],
"score_min": 66,
"score_max": 76,
"score_mean": 71.0,
"score_median": 71,
"dist_median_km": 122.0,
"dist_p90_km": 266.8
},
"5760": {
"n": 43,
"alg_rho": 0.3427,
"pers_rho": 0.2765,
"clim_rho": 0.0,
"no_skill_rho": 0.0,
"skill_gain": 0.0662,
"calibration": [
{
"decile": 6,
"score_range": "60-70",
"n": 20,
"median_km": 94.0,
"p90_km": 224.2
},
{
"decile": 7,
"score_range": "70-80",
"n": 23,
"median_km": 151.0,
"p90_km": 250.0
}
],
"score_min": 64,
"score_max": 78,
"score_mean": 70.4,
"score_median": 70,
"dist_median_km": 110.0,
"dist_p90_km": 232.6
},
"10000": {
"n": 375,
"alg_rho": 0.2065,
"pers_rho": 0.2039,
"clim_rho": 0.0,
"no_skill_rho": 0.0,
"skill_gain": 0.0025,
"calibration": [
{
"decile": 6,
"score_range": "60-70",
"n": 101,
"median_km": 100.0,
"p90_km": 232.0
},
{
"decile": 7,
"score_range": "70-80",
"n": 268,
"median_km": 136.0,
"p90_km": 278.3
},
{
"decile": 8,
"score_range": "80-90",
"n": 6,
"median_km": 203.0,
"p90_km": 309.5
}
],
"score_min": 60,
"score_max": 81,
"score_mean": 71.9,
"score_median": 72,
"dist_median_km": 126.0,
"dist_p90_km": 261.6
},
"24000": {
"n": 105,
"alg_rho": 0.1718,
"pers_rho": 0.304,
"clim_rho": 0.0,
"no_skill_rho": 0.0,
"skill_gain": -0.1322,
"calibration": [
{
"decile": 4,
"score_range": "40-50",
"n": 9,
"median_km": 151.0,
"p90_km": 151.0
},
{
"decile": 5,
"score_range": "50-60",
"n": 51,
"median_km": 31.0,
"p90_km": 60.0
},
{
"decile": 6,
"score_range": "60-70",
"n": 6,
"median_km": 23.0,
"p90_km": 36.0
},
{
"decile": 7,
"score_range": "70-80",
"n": 23,
"median_km": 44.0,
"p90_km": 119.0
},
{
"decile": 8,
"score_range": "80-90",
"n": 16,
"median_km": 100.0,
"p90_km": 135.0
}
],
"score_min": 48,
"score_max": 84,
"score_mean": 62.9,
"score_median": 54,
"dist_median_km": 40.0,
"dist_p90_km": 134.6
},
"47000": {
"n": 21,
"alg_rho": 0.3914,
"pers_rho": -0.2685,
"clim_rho": 0.0,
"no_skill_rho": 0.0,
"skill_gain": 0.6599,
"calibration": [
{
"decile": 5,
"score_range": "50-60",
"n": 8,
"median_km": 41.0,
"p90_km": 105.9
},
{
"decile": 6,
"score_range": "60-70",
"n": 6,
"median_km": 68.0,
"p90_km": 91.5
},
{
"decile": 7,
"score_range": "70-80",
"n": 2,
"median_km": 118.0,
"p90_km": 113.0
},
{
"decile": 8,
"score_range": "80-90",
"n": 5,
"median_km": 127.0,
"p90_km": 132.6
}
],
"score_min": 53,
"score_max": 86,
"score_mean": 66.6,
"score_median": 63,
"dist_median_km": 68.0,
"dist_p90_km": 127.0
},
"75000": {
"n": 6,
"alg_rho": 1.0,
"pers_rho": 0.8944,
"clim_rho": 0.0,
"no_skill_rho": 0.0,
"skill_gain": 0.1056,
"calibration": [
{
"decile": 4,
"score_range": "40-50",
"n": 1,
"median_km": 5.0,
"p90_km": 5.0
},
{
"decile": 5,
"score_range": "50-60",
"n": 5,
"median_km": 7.0,
"p90_km": 43.0
}
],
"score_min": 49,
"score_max": 51,
"score_mean": 50.2,
"score_median": 50,
"dist_median_km": 7.0,
"dist_p90_km": 43.0
},
"122000": {
"n": 4,
"alg_rho": -0.8165,
"pers_rho": 0.0,
"clim_rho": 0.0,
"no_skill_rho": 0.0,
"skill_gain": -0.8165,
"calibration": [
{
"decile": 5,
"score_range": "50-60",
"n": 4,
"median_km": 7.0,
"p90_km": 7.0
}
],
"score_min": 50,
"score_max": 53,
"score_mean": 51.2,
"score_median": 51,
"dist_median_km": 7.0,
"dist_p90_km": 7.0
}
}
}
Reference in a new issue View git blame Copy permalink