From 1f7e9e18a471094ef1876db6e50dc6678aa1d2a8 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 6 Apr 2026 16:10:28 -0500 Subject: [PATCH 01/88] Store stub records on empty weather fetches to prevent infinite backfill retries When ASOS/RAOB/IEMRE APIs return empty data, store a stub record so dedup checks (has_surface_observations?, has_sounding?, has_iemre_observation?) see coverage and don't recreate the same jobs on future backfill runs. --- .../workers/iemre_fetch_worker.ex | 4 ++- .../workers/weather_fetch_worker.ex | 19 +++++++++- .../workers/iemre_fetch_worker_test.exs | 9 +++-- .../workers/weather_fetch_worker_test.exs | 36 +++++++++++++++++-- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/lib/microwaveprop/workers/iemre_fetch_worker.ex b/lib/microwaveprop/workers/iemre_fetch_worker.ex index 98b34274..75630e65 100644 --- a/lib/microwaveprop/workers/iemre_fetch_worker.ex +++ b/lib/microwaveprop/workers/iemre_fetch_worker.ex @@ -25,7 +25,9 @@ defmodule Microwaveprop.Workers.IemreFetchWorker do case IemClient.fetch_iemre(lat, lon, date) do {:ok, []} -> - Logger.info("IEMRE returned empty data for #{lat},#{lon} @ #{date_str}") + # Store stub so this lat/lon/date isn't retried on future backfills + Weather.upsert_iemre_observation(%{lat: lat, lon: lon, date: date, hourly: []}) + Logger.info("IEMRE: no data available for #{lat},#{lon} @ #{date_str}, stored stub") :ok {:ok, data} -> diff --git a/lib/microwaveprop/workers/weather_fetch_worker.ex b/lib/microwaveprop/workers/weather_fetch_worker.ex index 433ffba7..c773fbb7 100644 --- a/lib/microwaveprop/workers/weather_fetch_worker.ex +++ b/lib/microwaveprop/workers/weather_fetch_worker.ex @@ -10,6 +10,7 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do alias Microwaveprop.Weather.IemClient alias Microwaveprop.Weather.SoundingParams alias Microwaveprop.Weather.Station + alias Microwaveprop.Weather.SurfaceObservation require Logger @@ -50,6 +51,15 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do if row.observed_at, do: Weather.upsert_surface_observation(station, row) end) + # Store stub so this station/window isn't retried on future backfills + if count == 0 do + midpoint = DateTime.add(start_dt, div(DateTime.diff(end_dt, start_dt), 2)) + + %SurfaceObservation{} + |> SurfaceObservation.changeset(%{station_id: station.id, observed_at: midpoint}) + |> Repo.insert(on_conflict: :nothing, conflict_target: [:station_id, :observed_at]) + end + Logger.info("WeatherFetch ASOS: #{station_code} ingested #{count} observations") Phoenix.PubSub.broadcast( @@ -132,7 +142,14 @@ defmodule Microwaveprop.Workers.WeatherFetchWorker do :ok {:ok, []} -> - Logger.info("WeatherFetch RAOB: #{station_code} returned no data for #{sounding_time_str}") + # Store stub sounding so this station/time isn't retried on future backfills + Weather.upsert_sounding(station, %{ + observed_at: sounding_time, + profile: [], + level_count: 0 + }) + + Logger.info("WeatherFetch RAOB: #{station_code} no data for #{sounding_time_str}, stored stub") :ok {:error, reason} -> diff --git a/test/microwaveprop/workers/iemre_fetch_worker_test.exs b/test/microwaveprop/workers/iemre_fetch_worker_test.exs index 2f65d35c..0de0d747 100644 --- a/test/microwaveprop/workers/iemre_fetch_worker_test.exs +++ b/test/microwaveprop/workers/iemre_fetch_worker_test.exs @@ -60,7 +60,7 @@ defmodule Microwaveprop.Workers.IemreFetchWorkerTest do assert length(obs.hourly) == 2 end - test "returns :ok on empty data" do + test "stores stub on empty data so backfill doesn't retry" do Req.Test.stub(IemClient, fn conn -> Req.Test.json(conn, %{"data" => []}) end) @@ -74,7 +74,12 @@ defmodule Microwaveprop.Workers.IemreFetchWorkerTest do } assert :ok = IemreFetchWorker.perform(job) - assert Repo.aggregate(IemreObservation, :count) == 0 + assert Repo.aggregate(IemreObservation, :count) == 1 + + obs = Repo.one!(IemreObservation) + assert obs.hourly == [] + assert obs.lat == 32.875 + assert obs.lon == -97.0 end test "retries on transient server error" do diff --git a/test/microwaveprop/workers/weather_fetch_worker_test.exs b/test/microwaveprop/workers/weather_fetch_worker_test.exs index f6d77b9b..87adb021 100644 --- a/test/microwaveprop/workers/weather_fetch_worker_test.exs +++ b/test/microwaveprop/workers/weather_fetch_worker_test.exs @@ -122,6 +122,33 @@ defmodule Microwaveprop.Workers.WeatherFetchWorkerTest do assert {:error, "IEM ASOS HTTP 503"} = WeatherFetchWorker.perform(%Oban.Job{args: args}) end + test "stores stub observation on empty ASOS data so backfill doesn't retry", %{station: station} do + Req.Test.stub(IemClient, fn conn -> + body = """ + #DEBUG, + station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1,p01i,wxcodes + """ + + Plug.Conn.send_resp(conn, 200, body) + end) + + args = %{ + "fetch_type" => "asos", + "station_id" => station.id, + "station_code" => "KORD", + "start_dt" => "2026-03-28T16:53:00Z", + "end_dt" => "2026-03-28T20:53:00Z" + } + + assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args}) + assert Repo.aggregate(SurfaceObservation, :count) == 1 + + obs = Repo.one!(SurfaceObservation) + assert obs.station_id == station.id + # All weather fields nil — it's a stub + assert is_nil(obs.temp_f) + end + test "returns :ok when station no longer exists" do Req.Test.stub(IemClient, fn conn -> Plug.Conn.send_resp(conn, 500, "Should not be called") @@ -185,7 +212,7 @@ defmodule Microwaveprop.Workers.WeatherFetchWorkerTest do assert Repo.aggregate(Sounding, :count) == 1 end - test "handles empty profiles", %{station: station} do + test "stores stub sounding on empty profiles so backfill doesn't retry", %{station: station} do Req.Test.stub(IemClient, fn conn -> Req.Test.json(conn, %{"profiles" => []}) end) @@ -198,7 +225,12 @@ defmodule Microwaveprop.Workers.WeatherFetchWorkerTest do } assert :ok = WeatherFetchWorker.perform(%Oban.Job{args: args}) - assert Repo.aggregate(Sounding, :count) == 0 + assert Repo.aggregate(Sounding, :count) == 1 + + sounding = Repo.one!(Sounding) + assert sounding.profile == [] + assert sounding.level_count == 0 + assert sounding.station_id == station.id end test "returns error on HTTP failure", %{station: station} do From 00bacae2c295472332980f3945beba664a452117 Mon Sep 17 00:00:00 2001 From: FluxCD Date: Mon, 6 Apr 2026 18:17:26 +0000 Subject: [PATCH 02/88] chore: update prop image to git.mcintire.me/graham/prop:main-1775499376-15ec8af [skip ci] --- k8s/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index e8cb8f0b..5bc7590f 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -41,7 +41,7 @@ spec: type: RuntimeDefault containers: - name: prop - image: git.mcintire.me/graham/prop:main-1775494893-2fd0175 # {"$imagepolicy": "flux-system:prop"} + image: git.mcintire.me/graham/prop:main-1775499376-15ec8af # {"$imagepolicy": "flux-system:prop"} imagePullPolicy: IfNotPresent env: - name: POD_IP From 25d7ec317af0b0ff6dfce08b7f431c0a0eb51c3e Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 6 Apr 2026 16:54:12 -0500 Subject: [PATCH 03/88] Mark terrain status complete when profile already exists TerrainProfileWorker early-exited with :ok when a profile existed but never updated the contact status, leaving contacts stuck at queued. --- lib/microwaveprop/workers/terrain_profile_worker.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/microwaveprop/workers/terrain_profile_worker.ex b/lib/microwaveprop/workers/terrain_profile_worker.ex index 1249155b..353d4d94 100644 --- a/lib/microwaveprop/workers/terrain_profile_worker.ex +++ b/lib/microwaveprop/workers/terrain_profile_worker.ex @@ -19,6 +19,7 @@ defmodule Microwaveprop.Workers.TerrainProfileWorker do @impl Oban.Worker def perform(%Oban.Job{args: %{"contact_id" => contact_id}}) do if Terrain.has_terrain_profile?(contact_id) do + Radio.set_enrichment_status!([contact_id], :terrain_status, :complete) :ok else contact = Radio.get_contact!(contact_id) From 25df2e6766ede4f4ffc82defd57ce3a7ffeba9cf Mon Sep 17 00:00:00 2001 From: FluxCD Date: Mon, 6 Apr 2026 21:11:31 +0000 Subject: [PATCH 04/88] chore: update prop image to git.mcintire.me/graham/prop:main-1775509847-dcf5190 [skip ci] --- k8s/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index 5bc7590f..ad1ab639 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -41,7 +41,7 @@ spec: type: RuntimeDefault containers: - name: prop - image: git.mcintire.me/graham/prop:main-1775499376-15ec8af # {"$imagepolicy": "flux-system:prop"} + image: git.mcintire.me/graham/prop:main-1775509847-dcf5190 # {"$imagepolicy": "flux-system:prop"} imagePullPolicy: IfNotPresent env: - name: POD_IP From da4979751d1592fefcacb5b69f3bfcb6aebb2332 Mon Sep 17 00:00:00 2001 From: FluxCD Date: Mon, 6 Apr 2026 21:56:05 +0000 Subject: [PATCH 05/88] chore: update prop image to git.mcintire.me/graham/prop:main-1775512477-be9b890 [skip ci] --- k8s/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index ad1ab639..6c920b8e 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -41,7 +41,7 @@ spec: type: RuntimeDefault containers: - name: prop - image: git.mcintire.me/graham/prop:main-1775509847-dcf5190 # {"$imagepolicy": "flux-system:prop"} + image: git.mcintire.me/graham/prop:main-1775512477-be9b890 # {"$imagepolicy": "flux-system:prop"} imagePullPolicy: IfNotPresent env: - name: POD_IP From c68e1ba108e424e8c09cefec3542089b0c888565 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 6 Apr 2026 17:01:32 -0500 Subject: [PATCH 06/88] Re-enable hourly propagation grid scoring in production --- config/runtime.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index ebf04c15..0dc1d9b2 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -139,8 +139,8 @@ if config_env() == :prod do {Oban.Plugins.Cron, crontab: [ {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, - {"*/5 * * * *", Microwaveprop.Commercial.PollWorker} - # {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker} + {"*/5 * * * *", Microwaveprop.Commercial.PollWorker}, + {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker} ]} ] From 5583fb616763761e5cb9719a06aded0703e87b7f Mon Sep 17 00:00:00 2001 From: FluxCD Date: Mon, 6 Apr 2026 22:03:25 +0000 Subject: [PATCH 07/88] chore: update prop image to git.mcintire.me/graham/prop:main-1775512945-6ab4ba7 [skip ci] --- k8s/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index 6c920b8e..ac0aa6d3 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -41,7 +41,7 @@ spec: type: RuntimeDefault containers: - name: prop - image: git.mcintire.me/graham/prop:main-1775512477-be9b890 # {"$imagepolicy": "flux-system:prop"} + image: git.mcintire.me/graham/prop:main-1775512945-6ab4ba7 # {"$imagepolicy": "flux-system:prop"} imagePullPolicy: IfNotPresent env: - name: POD_IP From 5bd2516c1550a2d004f9d095a18c535d40d3e4f6 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 6 Apr 2026 17:04:11 -0500 Subject: [PATCH 08/88] Bump terrain queue concurrency to 3 per pod --- config/runtime.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/runtime.exs b/config/runtime.exs index 0dc1d9b2..384d6ff8 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -132,7 +132,7 @@ if config_env() == :prod do # Production Oban: live scoring, polling, and on-demand QSO enrichment (no cron backfill) config :microwaveprop, Oban, # Per-pod concurrency (×3 pods = effective cluster total) - queues: [propagation: 1, commercial: 1, solar: 1, weather: 3, hrrr: 5, terrain: 2, iemre: 3, backfill_enqueue: 1], + queues: [propagation: 1, commercial: 1, solar: 1, weather: 3, hrrr: 5, terrain: 3, iemre: 3, backfill_enqueue: 1], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, {Oban.Plugins.Lifeline, rescue_after: 10 * 60 * 1000}, From ec82f4ae71dfb75d5d79be90604d8d27be8b0f1e Mon Sep 17 00:00:00 2001 From: FluxCD Date: Mon, 6 Apr 2026 22:05:15 +0000 Subject: [PATCH 09/88] chore: update prop image to git.mcintire.me/graham/prop:main-1775513079-b5a87e0 [skip ci] --- k8s/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index ac0aa6d3..e4fe6a97 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -41,7 +41,7 @@ spec: type: RuntimeDefault containers: - name: prop - image: git.mcintire.me/graham/prop:main-1775512945-6ab4ba7 # {"$imagepolicy": "flux-system:prop"} + image: git.mcintire.me/graham/prop:main-1775513079-b5a87e0 # {"$imagepolicy": "flux-system:prop"} imagePullPolicy: IfNotPresent env: - name: POD_IP From 51c73a3cbdfc48d54b55f725ad85a59c6878e70e Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 07:36:45 -0500 Subject: [PATCH 10/88] Add periodic enrichment backfill cron and terrain reconciliation Runs BackfillEnqueueWorker every 30 minutes to pick up contacts with pending/queued/failed enrichment status and enqueue missing jobs. Before enqueueing, reconciles terrain contacts stuck in "queued" by checking the terrain_profiles FK directly. --- config/runtime.exs | 6 ++-- .../workers/backfill_enqueue_worker.ex | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 384d6ff8..828dbc48 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -129,7 +129,7 @@ if config_env() == :prod do ], secret_key_base: secret_key_base - # Production Oban: live scoring, polling, and on-demand QSO enrichment (no cron backfill) + # Production Oban: live scoring, polling, and on-demand QSO enrichment config :microwaveprop, Oban, # Per-pod concurrency (×3 pods = effective cluster total) queues: [propagation: 1, commercial: 1, solar: 1, weather: 3, hrrr: 5, terrain: 3, iemre: 3, backfill_enqueue: 1], @@ -140,7 +140,9 @@ if config_env() == :prod do crontab: [ {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, {"*/5 * * * *", Microwaveprop.Commercial.PollWorker}, - {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker} + {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker}, + {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker, + args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}} ]} ] diff --git a/lib/microwaveprop/workers/backfill_enqueue_worker.ex b/lib/microwaveprop/workers/backfill_enqueue_worker.ex index eeb58110..7dfdaba5 100644 --- a/lib/microwaveprop/workers/backfill_enqueue_worker.ex +++ b/lib/microwaveprop/workers/backfill_enqueue_worker.ex @@ -20,6 +20,12 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do def perform(%Oban.Job{args: %{"limit" => limit} = args}) do types = parse_types(args) + reconciled = reconcile_stale_queued(types) + + if reconciled > 0 do + Logger.info("BackfillEnqueue: reconciled #{reconciled} stale queued contacts") + end + contacts = Contact |> where([c], not is_nil(c.pos1) or not is_nil(c.grid1)) @@ -58,6 +64,28 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do ] end + # Fast SQL reconciliation for contacts stuck in "queued" where data already exists. + # Terrain profiles have a direct contact_id FK so we can reconcile in bulk. + defp reconcile_stale_queued(types) do + terrain_count = + if :terrain in types do + {count, _} = + Repo.update_all( + from(c in Contact, + where: c.terrain_status == :queued, + where: c.id in subquery(from(tp in "terrain_profiles", select: tp.contact_id)) + ), + set: [terrain_status: :complete] + ) + + count + else + 0 + end + + terrain_count + end + defp type_filter(types) do Enum.reduce(types, dynamic(false), fn type, acc -> field = :"#{type}_status" From 970bc29afed76728924270f3a7e2093627c54643 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 07:37:55 -0500 Subject: [PATCH 11/88] Remove stale propagation scores maintenance banner --- lib/microwaveprop_web/live/map_live.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/microwaveprop_web/live/map_live.ex b/lib/microwaveprop_web/live/map_live.ex index de38acb7..944d390c 100644 --- a/lib/microwaveprop_web/live/map_live.ex +++ b/lib/microwaveprop_web/live/map_live.ex @@ -276,15 +276,10 @@ defmodule MicrowavepropWeb.MapLive do > - <%!-- Maintenance banner --%> -
- Propagation scores are not currently being updated while we backfill historical data. Displayed data may be stale. -
- <%!-- Top-left control panel --%>
From f17dfc7270134a0ef4453e6ccf54ba46083f6a0c Mon Sep 17 00:00:00 2001 From: FluxCD Date: Tue, 7 Apr 2026 12:38:36 +0000 Subject: [PATCH 12/88] chore: update prop image to git.mcintire.me/graham/prop:main-1775565429-f0340a2 [skip ci] --- k8s/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index e4fe6a97..f0a0eb2b 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -41,7 +41,7 @@ spec: type: RuntimeDefault containers: - name: prop - image: git.mcintire.me/graham/prop:main-1775513079-b5a87e0 # {"$imagepolicy": "flux-system:prop"} + image: git.mcintire.me/graham/prop:main-1775565429-f0340a2 # {"$imagepolicy": "flux-system:prop"} imagePullPolicy: IfNotPresent env: - name: POD_IP From 114f25a461c7bc8b93af093026892995ac38d03b Mon Sep 17 00:00:00 2001 From: FluxCD Date: Tue, 7 Apr 2026 12:39:36 +0000 Subject: [PATCH 13/88] chore: update prop image to git.mcintire.me/graham/prop:main-1775565490-61e4196 [skip ci] --- k8s/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index f0a0eb2b..b0c2a93c 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -41,7 +41,7 @@ spec: type: RuntimeDefault containers: - name: prop - image: git.mcintire.me/graham/prop:main-1775565429-f0340a2 # {"$imagepolicy": "flux-system:prop"} + image: git.mcintire.me/graham/prop:main-1775565490-61e4196 # {"$imagepolicy": "flux-system:prop"} imagePullPolicy: IfNotPresent env: - name: POD_IP From d1e42dd8a096b565346217b3236ec6b36b8eee3e Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 07:42:23 -0500 Subject: [PATCH 14/88] Smooth viewshed boundary to remove SRTM elevation spikes Apply 5-point circular moving average to reach_km values before rendering the boundary polygon. Eliminates sharp spikes caused by isolated terrain anomalies in SRTM data. --- lib/microwaveprop/terrain/viewshed.ex | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/lib/microwaveprop/terrain/viewshed.ex b/lib/microwaveprop/terrain/viewshed.ex index 08362868..e1a8dfd2 100644 --- a/lib/microwaveprop/terrain/viewshed.ex +++ b/lib/microwaveprop/terrain/viewshed.ex @@ -129,6 +129,7 @@ defmodule Microwaveprop.Terrain.Viewshed do {:exit, _} -> nil end) |> Enum.reject(&is_nil/1) + |> smooth_boundary(lat, lon) %{origin: %{lat: lat, lon: lon}, boundary: boundary} end @@ -180,6 +181,34 @@ defmodule Microwaveprop.Terrain.Viewshed do Application.get_env(:microwaveprop, :srtm_tiles_dir, Path.expand("~/srtm/tiles")) end + # Smooth reach_km values with a circular moving average to remove spikes + # from SRTM elevation artifacts, then recompute boundary lat/lon. + defp smooth_boundary(points, _origin_lat, _origin_lon) when length(points) < 5, do: points + + defp smooth_boundary(points, origin_lat, origin_lon) do + reaches = Enum.map(points, & &1.reach_km) + n = length(reaches) + arr = :array.from_list(reaches) + half = 2 + + smoothed_reaches = + Enum.map(0..(n - 1), fn i -> + window = + for offset <- -half..half do + :array.get(rem(i + offset + n, n), arr) + end + + Enum.sum(window) / length(window) + end) + + points + |> Enum.zip(smoothed_reaches) + |> Enum.map(fn {pt, reach} -> + {lat, lon} = destination_point(origin_lat, origin_lon, pt.bearing, reach) + %{pt | reach_km: reach, lat: lat, lon: lon} + end) + end + defp find_first_obstructed_index(interior) do Enum.find_index(interior, & &1.obstructed) end From 214b706ba81ff56ae458d123382dd1bd3d2f84fd Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 07:50:31 -0500 Subject: [PATCH 15/88] Remove icons from navigation links in map control panels --- lib/microwaveprop_web/live/map_live.ex | 20 +++++++++---------- .../live/weather_map_live.ex | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/microwaveprop_web/live/map_live.ex b/lib/microwaveprop_web/live/map_live.ex index 944d390c..43dde84d 100644 --- a/lib/microwaveprop_web/live/map_live.ex +++ b/lib/microwaveprop_web/live/map_live.ex @@ -360,20 +360,20 @@ defmodule MicrowavepropWeb.MapLive do <%!-- Links --%>
- <.link navigate="/weather" class="btn btn-xs btn-ghost justify-start gap-1.5"> - <.icon name="hero-cloud" class="size-3.5" /> Weather Map + <.link navigate="/weather" class="btn btn-xs btn-ghost justify-start"> + Weather Map - <.link navigate="/algo" class="btn btn-xs btn-ghost justify-start gap-1.5"> - <.icon name="hero-calculator" class="size-3.5" /> Scoring Algorithm + <.link navigate="/algo" class="btn btn-xs btn-ghost justify-start"> + Scoring Algorithm - <.link navigate="/submit" class="btn btn-xs btn-ghost justify-start gap-1.5"> - <.icon name="hero-arrow-up-tray" class="size-3.5" /> Submit a Contact + <.link navigate="/submit" class="btn btn-xs btn-ghost justify-start"> + Submit a Contact - <.link navigate="/contacts" class="btn btn-xs btn-ghost justify-start gap-1.5"> - <.icon name="hero-signal" class="size-3.5" /> Contact Training Data + <.link navigate="/contacts" class="btn btn-xs btn-ghost justify-start"> + Contact Training Data - <.link navigate="/contacts/map" class="btn btn-xs btn-ghost justify-start gap-1.5"> - <.icon name="hero-map" class="size-3.5" /> Contact Map + <.link navigate="/contacts/map" class="btn btn-xs btn-ghost justify-start"> + Contact Map
diff --git a/lib/microwaveprop_web/live/weather_map_live.ex b/lib/microwaveprop_web/live/weather_map_live.ex index 4414c5e8..1705441d 100644 --- a/lib/microwaveprop_web/live/weather_map_live.ex +++ b/lib/microwaveprop_web/live/weather_map_live.ex @@ -281,8 +281,8 @@ defmodule MicrowavepropWeb.WeatherMapLive do <%!-- Links --%>
- <.link navigate="/map" class="btn btn-xs btn-ghost justify-start gap-1.5"> - <.icon name="hero-signal" class="size-3.5" /> Propagation Map + <.link navigate="/map" class="btn btn-xs btn-ghost justify-start"> + Propagation Map
From 6b9ef24c816bca9145a0d0abf74b6ece55b30614 Mon Sep 17 00:00:00 2001 From: FluxCD Date: Tue, 7 Apr 2026 12:51:39 +0000 Subject: [PATCH 16/88] chore: update prop image to git.mcintire.me/graham/prop:main-1775566243-7f457e4 [skip ci] --- k8s/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index b0c2a93c..497c5810 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -41,7 +41,7 @@ spec: type: RuntimeDefault containers: - name: prop - image: git.mcintire.me/graham/prop:main-1775565490-61e4196 # {"$imagepolicy": "flux-system:prop"} + image: git.mcintire.me/graham/prop:main-1775566243-7f457e4 # {"$imagepolicy": "flux-system:prop"} imagePullPolicy: IfNotPresent env: - name: POD_IP From af5a48b79747f8cccae9ff751683326ef830402e Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 08:45:10 -0500 Subject: [PATCH 17/88] Add Livebook notebook for propagation algorithm analysis Connects to local dev DB to analyze contacts, HRRR conditions, terrain impact, commercial link correlation, and score sensitivity. Ready for use once prod data is imported locally. --- notebooks/algorithm_analysis.livemd | 410 ++++++++++++++++++++++++++++ 1 file changed, 410 insertions(+) create mode 100644 notebooks/algorithm_analysis.livemd diff --git a/notebooks/algorithm_analysis.livemd b/notebooks/algorithm_analysis.livemd new file mode 100644 index 00000000..19927192 --- /dev/null +++ b/notebooks/algorithm_analysis.livemd @@ -0,0 +1,410 @@ +# Propagation Algorithm Analysis + +```elixir +Mix.install([ + {:postgrex, "~> 0.19"}, + {:ecto_sql, "~> 3.12"}, + {:kino, "~> 0.14"}, + {:kino_vega_lite, "~> 0.1"}, + {:explorer, "~> 0.10"}, + {:statistex, "~> 1.0"} +]) +``` + +## Database Connection + +```elixir +{:ok, conn} = Postgrex.start_link( + hostname: "localhost", + database: "microwaveprop_dev", + username: "postgres", + password: "postgres", + port: 5432 +) +``` + +## Helper Functions + +```elixir +defmodule Q do + @doc "Run a query and return results as a list of maps" + def query!(conn, sql, params \\ []) do + %{columns: cols, rows: rows} = Postgrex.query!(conn, sql, params) + Enum.map(rows, fn row -> Enum.zip(cols, row) |> Map.new() end) + end + + @doc "Run a query and return as an Explorer DataFrame" + def df!(conn, sql, params \\ []) do + %{columns: cols, rows: rows} = Postgrex.query!(conn, sql, params) + cols + |> Enum.with_index() + |> Enum.map(fn {col, i} -> {col, Enum.map(rows, &Enum.at(&1, i))} end) + |> Map.new() + |> Explorer.DataFrame.new() + end +end +``` + +## 1. Contact Overview + +```elixir +Q.query!(conn, """ + SELECT COUNT(*) as total, + COUNT(*) FILTER (WHERE distance_km < 3000) as tropo, + COUNT(*) FILTER (WHERE distance_km >= 3000) as eme, + COUNT(DISTINCT band) as bands, + MIN(qso_timestamp) as earliest, + MAX(qso_timestamp) as latest + FROM contacts +""") +|> hd() +|> Kino.Tree.new() +``` + +```elixir +# Contacts by band +Q.df!(conn, """ + SELECT band::integer as band_mhz, COUNT(*) as count, + ROUND(AVG(distance_km::numeric), 1) as avg_dist_km, + ROUND(MAX(distance_km::numeric), 1) as max_dist_km + FROM contacts + WHERE distance_km < 3000 + GROUP BY band ORDER BY band +""") +|> Kino.DataTable.new() +``` + +```elixir +# Contacts by month — when do contacts happen? +contacts_by_month = Q.df!(conn, """ + SELECT EXTRACT(MONTH FROM qso_timestamp)::integer as month, + COUNT(*) as count + FROM contacts WHERE distance_km < 3000 + GROUP BY 1 ORDER BY 1 +""") + +VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by Month") +|> VegaLite.data_from_values(contacts_by_month |> Explorer.DataFrame.to_rows()) +|> VegaLite.mark(:bar) +|> VegaLite.encode_field(:x, "month", type: :ordinal) +|> VegaLite.encode_field(:y, "count", type: :quantitative) +``` + +```elixir +# Contacts by hour of day +contacts_by_hour = Q.df!(conn, """ + SELECT EXTRACT(HOUR FROM qso_timestamp)::integer as utc_hour, + COUNT(*) as count + FROM contacts WHERE distance_km < 3000 + GROUP BY 1 ORDER BY 1 +""") + +VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by UTC Hour") +|> VegaLite.data_from_values(contacts_by_hour |> Explorer.DataFrame.to_rows()) +|> VegaLite.mark(:bar) +|> VegaLite.encode_field(:x, "utc_hour", type: :ordinal) +|> VegaLite.encode_field(:y, "count", type: :quantitative) +``` + +## 2. HRRR Conditions at Contact Time + +Join contacts with their nearest HRRR profile to see what atmospheric +conditions were present when contacts actually happened. + +```elixir +# Contacts with HRRR data — the core analysis dataset +contact_hrrr = Q.df!(conn, """ + SELECT + c.id, + c.band::integer as band_mhz, + c.distance_km::float as distance_km, + c.qso_timestamp, + EXTRACT(MONTH FROM c.qso_timestamp)::integer as month, + EXTRACT(HOUR FROM c.qso_timestamp)::integer as utc_hour, + h.surface_temp as temp_c, + h.surface_dewpoint as dewpoint_c, + (h.surface_temp - h.surface_dewpoint) as td_depression_c, + h.surface_pressure as pressure_mb, + h.surface_refractivity, + h.min_refractivity_gradient, + h.hpbl_m as bl_depth_m, + h.pwat_mm, + h.ducting_detected + FROM contacts c + JOIN LATERAL ( + SELECT * + FROM hrrr_profiles hp + WHERE hp.lat BETWEEN ROUND((c.pos1->>'lat')::numeric, 2) - 0.07 + AND ROUND((c.pos1->>'lat')::numeric, 2) + 0.07 + AND hp.lon BETWEEN ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) - 0.07 + AND ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) + 0.07 + AND hp.valid_time BETWEEN c.qso_timestamp - interval '1 hour' + AND c.qso_timestamp + interval '1 hour' + ORDER BY ABS(EXTRACT(EPOCH FROM hp.valid_time - c.qso_timestamp)) + LIMIT 1 + ) h ON true + WHERE c.distance_km < 3000 + AND c.pos1 IS NOT NULL +""") + +Kino.DataTable.new(contact_hrrr, name: "Contacts + HRRR") +``` + +## 3. Factor Distributions — What Does the Atmosphere Look Like During Contacts? + +```elixir +rows = Explorer.DataFrame.to_rows(contact_hrrr) + +# Refractivity gradient distribution +VegaLite.new(width: 600, height: 300, title: "Refractivity Gradient at Contact Time") +|> VegaLite.data_from_values(rows) +|> VegaLite.mark(:bar) +|> VegaLite.encode_field(:x, "min_refractivity_gradient", + type: :quantitative, bin: %{step: 20}, title: "Min Refractivity Gradient (N-units/km)" +) +|> VegaLite.encode(:y, aggregate: :count, type: :quantitative) +``` + +```elixir +# Td depression distribution +VegaLite.new(width: 600, height: 300, title: "Td Depression at Contact Time") +|> VegaLite.data_from_values(rows) +|> VegaLite.mark(:bar) +|> VegaLite.encode_field(:x, "td_depression_c", + type: :quantitative, bin: %{step: 1}, title: "T - Td (C)" +) +|> VegaLite.encode(:y, aggregate: :count, type: :quantitative) +``` + +```elixir +# Boundary layer depth +VegaLite.new(width: 600, height: 300, title: "Boundary Layer Depth at Contact Time") +|> VegaLite.data_from_values(rows) +|> VegaLite.mark(:bar) +|> VegaLite.encode_field(:x, "bl_depth_m", + type: :quantitative, bin: %{step: 100}, title: "BL Depth (m)" +) +|> VegaLite.encode(:y, aggregate: :count, type: :quantitative) +``` + +```elixir +# PWAT distribution +VegaLite.new(width: 600, height: 300, title: "Precipitable Water at Contact Time") +|> VegaLite.data_from_values(rows) +|> VegaLite.mark(:bar) +|> VegaLite.encode_field(:x, "pwat_mm", + type: :quantitative, bin: %{step: 3}, title: "PWAT (mm)" +) +|> VegaLite.encode(:y, aggregate: :count, type: :quantitative) +``` + +## 4. Ducting Analysis + +```elixir +# Ducting rate by month +ducting_monthly = Q.df!(conn, """ + SELECT + EXTRACT(MONTH FROM c.qso_timestamp)::integer as month, + COUNT(*) as total, + COUNT(*) FILTER (WHERE h.ducting_detected = true) as ducting, + ROUND(100.0 * COUNT(*) FILTER (WHERE h.ducting_detected = true) / COUNT(*), 1) as ducting_pct + FROM contacts c + JOIN LATERAL ( + SELECT ducting_detected + FROM hrrr_profiles hp + WHERE hp.lat BETWEEN ROUND((c.pos1->>'lat')::numeric, 2) - 0.07 + AND ROUND((c.pos1->>'lat')::numeric, 2) + 0.07 + AND hp.lon BETWEEN ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) - 0.07 + AND ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) + 0.07 + AND hp.valid_time BETWEEN c.qso_timestamp - interval '1 hour' + AND c.qso_timestamp + interval '1 hour' + ORDER BY ABS(EXTRACT(EPOCH FROM hp.valid_time - c.qso_timestamp)) + LIMIT 1 + ) h ON true + WHERE c.distance_km < 3000 AND c.pos1 IS NOT NULL + GROUP BY 1 ORDER BY 1 +""") + +VegaLite.new(width: 600, height: 300, title: "Ducting Detection Rate by Month (at contact time)") +|> VegaLite.data_from_values(ducting_monthly |> Explorer.DataFrame.to_rows()) +|> VegaLite.mark(:bar) +|> VegaLite.encode_field(:x, "month", type: :ordinal) +|> VegaLite.encode_field(:y, "ducting_pct", type: :quantitative, title: "% Ducting") +``` + +## 5. Distance vs Atmospheric Conditions + +Do contacts at longer distances correlate with better atmospheric conditions? + +```elixir +# Distance vs refractivity gradient (scatter) +VegaLite.new(width: 600, height: 400, title: "Distance vs Refractivity Gradient") +|> VegaLite.data_from_values(rows) +|> VegaLite.mark(:circle, opacity: 0.3, size: 15) +|> VegaLite.encode_field(:x, "min_refractivity_gradient", + type: :quantitative, title: "Min Refractivity Gradient" +) +|> VegaLite.encode_field(:y, "distance_km", + type: :quantitative, title: "Distance (km)" +) +|> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)") +``` + +```elixir +# Distance vs Td depression +VegaLite.new(width: 600, height: 400, title: "Distance vs Td Depression") +|> VegaLite.data_from_values(rows) +|> VegaLite.mark(:circle, opacity: 0.3, size: 15) +|> VegaLite.encode_field(:x, "td_depression_c", + type: :quantitative, title: "T - Td (C)" +) +|> VegaLite.encode_field(:y, "distance_km", + type: :quantitative, title: "Distance (km)" +) +|> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)") +``` + +## 6. Terrain Impact + +```elixir +terrain_stats = Q.df!(conn, """ + SELECT + tp.verdict, + COUNT(*) as count, + ROUND(AVG(c.distance_km::numeric), 1) as avg_dist_km, + ROUND(AVG(tp.diffraction_db::numeric), 1) as avg_diffraction_db, + ROUND(AVG(tp.min_clearance_m::numeric), 1) as avg_clearance_m + FROM terrain_profiles tp + JOIN contacts c ON c.id = tp.contact_id + WHERE c.distance_km < 3000 + GROUP BY tp.verdict + ORDER BY count DESC +""") + +Kino.DataTable.new(terrain_stats, name: "Terrain Verdicts") +``` + +```elixir +# Diffraction loss vs distance +terrain_scatter = Q.df!(conn, """ + SELECT + c.band::integer as band_mhz, + c.distance_km::float as distance_km, + tp.diffraction_db::float as diffraction_db, + tp.verdict + FROM terrain_profiles tp + JOIN contacts c ON c.id = tp.contact_id + WHERE c.distance_km < 3000 +""") + +VegaLite.new(width: 600, height: 400, title: "Diffraction Loss vs Distance") +|> VegaLite.data_from_values(terrain_scatter |> Explorer.DataFrame.to_rows()) +|> VegaLite.mark(:circle, opacity: 0.2, size: 10) +|> VegaLite.encode_field(:x, "distance_km", type: :quantitative, title: "Distance (km)") +|> VegaLite.encode_field(:y, "diffraction_db", type: :quantitative, title: "Diffraction Loss (dB)") +|> VegaLite.encode_field(:color, "verdict", type: :nominal) +``` + +## 7. Commercial Link Correlation + +Ground truth: do commercial link signal levels correlate with atmospheric conditions? + +```elixir +commercial = Q.df!(conn, """ + SELECT + cl.name, cl.frequency_ghz, + cs.sampled_at, + EXTRACT(HOUR FROM cs.sampled_at)::integer as utc_hour, + EXTRACT(MONTH FROM cs.sampled_at)::integer as month, + cs.rx_power_0::float as rx_power_dbm + FROM commercial_samples cs + JOIN commercial_links cl ON cl.id = cs.link_id + WHERE cs.rx_power_0 IS NOT NULL + ORDER BY cs.sampled_at +""") + +Kino.DataTable.new(commercial, name: "Commercial Samples") +``` + +```elixir +# Signal strength by hour of day per link +VegaLite.new(width: 600, height: 300, title: "Commercial Link Signal by Hour of Day") +|> VegaLite.data_from_values(commercial |> Explorer.DataFrame.to_rows()) +|> VegaLite.mark(:line, point: true) +|> VegaLite.encode_field(:x, "utc_hour", type: :ordinal) +|> VegaLite.encode_field(:y, "rx_power_dbm", + type: :quantitative, aggregate: :mean, title: "Mean RX Power (dBm)" +) +|> VegaLite.encode_field(:color, "name", type: :nominal) +``` + +## 8. Scoring the Historical Contacts + +Recompute scores for all contacts using the current algorithm and compare +factor weights. This section is a template — once you import the full dump +and attach to the running node, you can call `Scorer.composite_score/2` directly. + +```elixir +# For now, compute a simplified composite score from HRRR data +# TODO: attach to running node for full Scorer access + +# Humidity score approximation (beneficial band) +humidity_score = fn temp_c, dewpoint_c -> + if is_nil(temp_c) or is_nil(dewpoint_c), do: 50, + else: min(100, max(0, round(217 * 6.112 * :math.exp(17.67 * dewpoint_c / (dewpoint_c + 243.5)) / (temp_c + 273.15) * 5))) +end + +# Td depression score (beneficial band) +td_score = fn temp_c, dewpoint_c -> + case temp_c - dewpoint_c do + d when d <= 1 -> 95 + d when d <= 2 -> 88 + d when d <= 3 -> 80 + d when d <= 5 -> 65 + d when d <= 8 -> 50 + d when d <= 12 -> 35 + _ -> 20 + end +end + +# Refractivity score +refrac_score = fn grad -> + cond do + is_nil(grad) -> 50 + grad < -200 -> 98 + grad < -150 -> 92 + grad < -100 -> 82 + grad < -75 -> 68 + grad < -55 -> 55 + grad < -40 -> 48 + true -> 42 + end +end + +IO.puts("Scoring functions defined. Use with Explorer.DataFrame.mutate/2 or Enum.map/2") +``` + +## 9. Weight Sensitivity Analysis + +Template for testing different weight configurations against historical data. + +```elixir +# Define weight sets to test +weight_sets = %{ + current: %{humidity: 0.18, time_of_day: 0.10, td_depression: 0.10, + refractivity: 0.08, sky: 0.08, season: 0.08, + wind: 0.05, rain: 0.08, pressure: 0.15, pwat: 0.10}, + + refractivity_heavy: %{humidity: 0.15, time_of_day: 0.08, td_depression: 0.10, + refractivity: 0.18, sky: 0.06, season: 0.08, + wind: 0.04, rain: 0.06, pressure: 0.12, pwat: 0.13}, + + humidity_heavy: %{humidity: 0.25, time_of_day: 0.08, td_depression: 0.12, + refractivity: 0.08, sky: 0.06, season: 0.08, + wind: 0.04, rain: 0.06, pressure: 0.12, pwat: 0.11} +} + +# TODO: score all contacts with each weight set, +# then correlate scores with distance achieved and commercial link signal levels +IO.puts("Weight sets defined: #{Map.keys(weight_sets) |> Enum.join(", ")}") +``` From b204b08751763027e0481a5b82907e3a98e6e849 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 08:47:28 -0500 Subject: [PATCH 18/88] Rewrite analysis notebook to attach to running node Uses Repo, Scorer, BandConfig, Weather directly instead of raw SQL. Scores all historical contacts with the real composite_score/2 and includes factor heatmaps and weight sensitivity analysis. --- notebooks/algorithm_analysis.livemd | 594 ++++++++++++++++------------ 1 file changed, 331 insertions(+), 263 deletions(-) diff --git a/notebooks/algorithm_analysis.livemd b/notebooks/algorithm_analysis.livemd index 19927192..21173689 100644 --- a/notebooks/algorithm_analysis.livemd +++ b/notebooks/algorithm_analysis.livemd @@ -1,106 +1,86 @@ # Propagation Algorithm Analysis -```elixir -Mix.install([ - {:postgrex, "~> 0.19"}, - {:ecto_sql, "~> 3.12"}, - {:kino, "~> 0.14"}, - {:kino_vega_lite, "~> 0.1"}, - {:explorer, "~> 0.10"}, - {:statistex, "~> 1.0"} -]) +## Setup — Attach to Running Node + +Start your dev server with a node name: + +``` +iex --sname prop -S mix phx.server ``` -## Database Connection +Then in Livebook, use **Runtime > Configure > Attached node** and connect to `prop@`. ```elixir -{:ok, conn} = Postgrex.start_link( - hostname: "localhost", - database: "microwaveprop_dev", - username: "postgres", - password: "postgres", - port: 5432 -) -``` +# Verify we have access to the app +alias Microwaveprop.Repo +alias Microwaveprop.Radio +alias Microwaveprop.Radio.Contact +alias Microwaveprop.Weather +alias Microwaveprop.Weather.HrrrProfile +alias Microwaveprop.Terrain +alias Microwaveprop.Terrain.TerrainAnalysis +alias Microwaveprop.Propagation.Scorer +alias Microwaveprop.Propagation.BandConfig -## Helper Functions +import Ecto.Query -```elixir -defmodule Q do - @doc "Run a query and return results as a list of maps" - def query!(conn, sql, params \\ []) do - %{columns: cols, rows: rows} = Postgrex.query!(conn, sql, params) - Enum.map(rows, fn row -> Enum.zip(cols, row) |> Map.new() end) - end - - @doc "Run a query and return as an Explorer DataFrame" - def df!(conn, sql, params \\ []) do - %{columns: cols, rows: rows} = Postgrex.query!(conn, sql, params) - cols - |> Enum.with_index() - |> Enum.map(fn {col, i} -> {col, Enum.map(rows, &Enum.at(&1, i))} end) - |> Map.new() - |> Explorer.DataFrame.new() - end -end +Repo.aggregate(Contact, :count) |> then(&"Connected — #{&1} contacts in DB") ``` ## 1. Contact Overview ```elixir -Q.query!(conn, """ - SELECT COUNT(*) as total, - COUNT(*) FILTER (WHERE distance_km < 3000) as tropo, - COUNT(*) FILTER (WHERE distance_km >= 3000) as eme, - COUNT(DISTINCT band) as bands, - MIN(qso_timestamp) as earliest, - MAX(qso_timestamp) as latest - FROM contacts -""") -|> hd() -|> Kino.Tree.new() +band_summary = + Contact + |> where([c], c.distance_km < 3000) + |> group_by([c], c.band) + |> select([c], %{ + band_mhz: c.band, + count: count(), + avg_dist_km: type(avg(c.distance_km), :float), + max_dist_km: type(max(c.distance_km), :float) + }) + |> order_by([c], c.band) + |> Repo.all() + +Kino.DataTable.new(band_summary, name: "Contacts by Band") ``` ```elixir -# Contacts by band -Q.df!(conn, """ - SELECT band::integer as band_mhz, COUNT(*) as count, - ROUND(AVG(distance_km::numeric), 1) as avg_dist_km, - ROUND(MAX(distance_km::numeric), 1) as max_dist_km - FROM contacts - WHERE distance_km < 3000 - GROUP BY band ORDER BY band -""") -|> Kino.DataTable.new() -``` - -```elixir -# Contacts by month — when do contacts happen? -contacts_by_month = Q.df!(conn, """ - SELECT EXTRACT(MONTH FROM qso_timestamp)::integer as month, - COUNT(*) as count - FROM contacts WHERE distance_km < 3000 - GROUP BY 1 ORDER BY 1 -""") +# Contacts by month +monthly = + Contact + |> where([c], c.distance_km < 3000) + |> group_by([c], fragment("EXTRACT(MONTH FROM ?)", c.qso_timestamp)) + |> select([c], %{ + month: fragment("EXTRACT(MONTH FROM ?)::integer", c.qso_timestamp), + count: count() + }) + |> order_by([c], fragment("1")) + |> Repo.all() VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by Month") -|> VegaLite.data_from_values(contacts_by_month |> Explorer.DataFrame.to_rows()) +|> VegaLite.data_from_values(monthly) |> VegaLite.mark(:bar) |> VegaLite.encode_field(:x, "month", type: :ordinal) |> VegaLite.encode_field(:y, "count", type: :quantitative) ``` ```elixir -# Contacts by hour of day -contacts_by_hour = Q.df!(conn, """ - SELECT EXTRACT(HOUR FROM qso_timestamp)::integer as utc_hour, - COUNT(*) as count - FROM contacts WHERE distance_km < 3000 - GROUP BY 1 ORDER BY 1 -""") +# Contacts by hour +hourly = + Contact + |> where([c], c.distance_km < 3000) + |> group_by([c], fragment("EXTRACT(HOUR FROM ?)", c.qso_timestamp)) + |> select([c], %{ + utc_hour: fragment("EXTRACT(HOUR FROM ?)::integer", c.qso_timestamp), + count: count() + }) + |> order_by([c], fragment("1")) + |> Repo.all() VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by UTC Hour") -|> VegaLite.data_from_values(contacts_by_hour |> Explorer.DataFrame.to_rows()) +|> VegaLite.data_from_values(hourly) |> VegaLite.mark(:bar) |> VegaLite.encode_field(:x, "utc_hour", type: :ordinal) |> VegaLite.encode_field(:y, "count", type: :quantitative) @@ -108,56 +88,64 @@ VegaLite.new(width: 600, height: 300, title: "Tropo Contacts by UTC Hour") ## 2. HRRR Conditions at Contact Time -Join contacts with their nearest HRRR profile to see what atmospheric -conditions were present when contacts actually happened. - ```elixir -# Contacts with HRRR data — the core analysis dataset -contact_hrrr = Q.df!(conn, """ - SELECT - c.id, - c.band::integer as band_mhz, - c.distance_km::float as distance_km, - c.qso_timestamp, - EXTRACT(MONTH FROM c.qso_timestamp)::integer as month, - EXTRACT(HOUR FROM c.qso_timestamp)::integer as utc_hour, - h.surface_temp as temp_c, - h.surface_dewpoint as dewpoint_c, - (h.surface_temp - h.surface_dewpoint) as td_depression_c, - h.surface_pressure as pressure_mb, - h.surface_refractivity, - h.min_refractivity_gradient, - h.hpbl_m as bl_depth_m, - h.pwat_mm, - h.ducting_detected - FROM contacts c - JOIN LATERAL ( - SELECT * - FROM hrrr_profiles hp - WHERE hp.lat BETWEEN ROUND((c.pos1->>'lat')::numeric, 2) - 0.07 - AND ROUND((c.pos1->>'lat')::numeric, 2) + 0.07 - AND hp.lon BETWEEN ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) - 0.07 - AND ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) + 0.07 - AND hp.valid_time BETWEEN c.qso_timestamp - interval '1 hour' - AND c.qso_timestamp + interval '1 hour' - ORDER BY ABS(EXTRACT(EPOCH FROM hp.valid_time - c.qso_timestamp)) - LIMIT 1 - ) h ON true - WHERE c.distance_km < 3000 - AND c.pos1 IS NOT NULL -""") +# Load tropo contacts with HRRR data via the app's existing lookup +contacts = + Contact + |> where([c], c.distance_km < 3000 and not is_nil(c.pos1)) + |> order_by([c], desc: c.qso_timestamp) + |> Repo.all() -Kino.DataTable.new(contact_hrrr, name: "Contacts + HRRR") +contact_hrrr = + contacts + |> Task.async_stream( + fn contact -> + hrrr = Weather.hrrr_for_contact(contact) + if hrrr, do: {contact, hrrr}, else: nil + end, + max_concurrency: 8, + timeout: 10_000 + ) + |> Enum.flat_map(fn + {:ok, nil} -> [] + {:ok, pair} -> [pair] + _ -> [] + end) + +"#{length(contact_hrrr)} contacts matched with HRRR data (of #{length(contacts)} tropo contacts)" ``` -## 3. Factor Distributions — What Does the Atmosphere Look Like During Contacts? +```elixir +# Build analysis rows +analysis_rows = + Enum.map(contact_hrrr, fn {contact, hrrr} -> + %{ + id: contact.id, + band_mhz: Decimal.to_integer(contact.band), + distance_km: Decimal.to_float(contact.distance_km), + month: contact.qso_timestamp.month, + utc_hour: contact.qso_timestamp.hour, + temp_c: hrrr.surface_temp, + dewpoint_c: hrrr.surface_dewpoint, + td_depression_c: if(hrrr.surface_temp && hrrr.surface_dewpoint, + do: hrrr.surface_temp - hrrr.surface_dewpoint, else: nil), + pressure_mb: hrrr.surface_pressure, + surface_refractivity: hrrr.surface_refractivity, + min_refractivity_gradient: hrrr.min_refractivity_gradient, + bl_depth_m: hrrr.hpbl_m, + pwat_mm: hrrr.pwat_mm, + ducting_detected: hrrr.ducting_detected + } + end) + +Kino.DataTable.new(analysis_rows, name: "Contacts + HRRR") +``` + +## 3. Factor Distributions ```elixir -rows = Explorer.DataFrame.to_rows(contact_hrrr) - -# Refractivity gradient distribution VegaLite.new(width: 600, height: 300, title: "Refractivity Gradient at Contact Time") -|> VegaLite.data_from_values(rows) +|> VegaLite.data_from_values(analysis_rows) |> VegaLite.mark(:bar) |> VegaLite.encode_field(:x, "min_refractivity_gradient", type: :quantitative, bin: %{step: 20}, title: "Min Refractivity Gradient (N-units/km)" @@ -166,9 +154,8 @@ VegaLite.new(width: 600, height: 300, title: "Refractivity Gradient at Contact T ``` ```elixir -# Td depression distribution VegaLite.new(width: 600, height: 300, title: "Td Depression at Contact Time") -|> VegaLite.data_from_values(rows) +|> VegaLite.data_from_values(analysis_rows) |> VegaLite.mark(:bar) |> VegaLite.encode_field(:x, "td_depression_c", type: :quantitative, bin: %{step: 1}, title: "T - Td (C)" @@ -177,9 +164,8 @@ VegaLite.new(width: 600, height: 300, title: "Td Depression at Contact Time") ``` ```elixir -# Boundary layer depth VegaLite.new(width: 600, height: 300, title: "Boundary Layer Depth at Contact Time") -|> VegaLite.data_from_values(rows) +|> VegaLite.data_from_values(analysis_rows) |> VegaLite.mark(:bar) |> VegaLite.encode_field(:x, "bl_depth_m", type: :quantitative, bin: %{step: 100}, title: "BL Depth (m)" @@ -188,9 +174,8 @@ VegaLite.new(width: 600, height: 300, title: "Boundary Layer Depth at Contact Ti ``` ```elixir -# PWAT distribution VegaLite.new(width: 600, height: 300, title: "Precipitable Water at Contact Time") -|> VegaLite.data_from_values(rows) +|> VegaLite.data_from_values(analysis_rows) |> VegaLite.mark(:bar) |> VegaLite.encode_field(:x, "pwat_mm", type: :quantitative, bin: %{step: 3}, title: "PWAT (mm)" @@ -201,32 +186,18 @@ VegaLite.new(width: 600, height: 300, title: "Precipitable Water at Contact Time ## 4. Ducting Analysis ```elixir -# Ducting rate by month -ducting_monthly = Q.df!(conn, """ - SELECT - EXTRACT(MONTH FROM c.qso_timestamp)::integer as month, - COUNT(*) as total, - COUNT(*) FILTER (WHERE h.ducting_detected = true) as ducting, - ROUND(100.0 * COUNT(*) FILTER (WHERE h.ducting_detected = true) / COUNT(*), 1) as ducting_pct - FROM contacts c - JOIN LATERAL ( - SELECT ducting_detected - FROM hrrr_profiles hp - WHERE hp.lat BETWEEN ROUND((c.pos1->>'lat')::numeric, 2) - 0.07 - AND ROUND((c.pos1->>'lat')::numeric, 2) + 0.07 - AND hp.lon BETWEEN ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) - 0.07 - AND ROUND(COALESCE((c.pos1->>'lon')::numeric, (c.pos1->>'lng')::numeric), 2) + 0.07 - AND hp.valid_time BETWEEN c.qso_timestamp - interval '1 hour' - AND c.qso_timestamp + interval '1 hour' - ORDER BY ABS(EXTRACT(EPOCH FROM hp.valid_time - c.qso_timestamp)) - LIMIT 1 - ) h ON true - WHERE c.distance_km < 3000 AND c.pos1 IS NOT NULL - GROUP BY 1 ORDER BY 1 -""") +ducting_monthly = + analysis_rows + |> Enum.group_by(& &1.month) + |> Enum.map(fn {month, rows} -> + total = length(rows) + ducting = Enum.count(rows, & &1.ducting_detected) + %{month: month, total: total, ducting: ducting, ducting_pct: Float.round(100.0 * ducting / max(total, 1), 1)} + end) + |> Enum.sort_by(& &1.month) VegaLite.new(width: 600, height: 300, title: "Ducting Detection Rate by Month (at contact time)") -|> VegaLite.data_from_values(ducting_monthly |> Explorer.DataFrame.to_rows()) +|> VegaLite.data_from_values(ducting_monthly) |> VegaLite.mark(:bar) |> VegaLite.encode_field(:x, "month", type: :ordinal) |> VegaLite.encode_field(:y, "ducting_pct", type: :quantitative, title: "% Ducting") @@ -234,71 +205,64 @@ VegaLite.new(width: 600, height: 300, title: "Ducting Detection Rate by Month (a ## 5. Distance vs Atmospheric Conditions -Do contacts at longer distances correlate with better atmospheric conditions? - ```elixir -# Distance vs refractivity gradient (scatter) VegaLite.new(width: 600, height: 400, title: "Distance vs Refractivity Gradient") -|> VegaLite.data_from_values(rows) +|> VegaLite.data_from_values(analysis_rows) |> VegaLite.mark(:circle, opacity: 0.3, size: 15) |> VegaLite.encode_field(:x, "min_refractivity_gradient", type: :quantitative, title: "Min Refractivity Gradient" ) -|> VegaLite.encode_field(:y, "distance_km", - type: :quantitative, title: "Distance (km)" -) +|> VegaLite.encode_field(:y, "distance_km", type: :quantitative, title: "Distance (km)") |> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)") ``` ```elixir -# Distance vs Td depression VegaLite.new(width: 600, height: 400, title: "Distance vs Td Depression") -|> VegaLite.data_from_values(rows) +|> VegaLite.data_from_values(analysis_rows) |> VegaLite.mark(:circle, opacity: 0.3, size: 15) -|> VegaLite.encode_field(:x, "td_depression_c", - type: :quantitative, title: "T - Td (C)" -) -|> VegaLite.encode_field(:y, "distance_km", - type: :quantitative, title: "Distance (km)" -) +|> VegaLite.encode_field(:x, "td_depression_c", type: :quantitative, title: "T - Td (C)") +|> VegaLite.encode_field(:y, "distance_km", type: :quantitative, title: "Distance (km)") |> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)") ``` ## 6. Terrain Impact ```elixir -terrain_stats = Q.df!(conn, """ - SELECT - tp.verdict, - COUNT(*) as count, - ROUND(AVG(c.distance_km::numeric), 1) as avg_dist_km, - ROUND(AVG(tp.diffraction_db::numeric), 1) as avg_diffraction_db, - ROUND(AVG(tp.min_clearance_m::numeric), 1) as avg_clearance_m - FROM terrain_profiles tp - JOIN contacts c ON c.id = tp.contact_id - WHERE c.distance_km < 3000 - GROUP BY tp.verdict - ORDER BY count DESC -""") +terrain_stats = + Repo.all( + from tp in "terrain_profiles", + join: c in Contact, on: c.id == tp.contact_id, + where: c.distance_km < 3000, + group_by: tp.verdict, + select: %{ + verdict: tp.verdict, + count: count(), + avg_dist_km: type(avg(c.distance_km), :float), + avg_diffraction_db: type(avg(tp.diffraction_db), :float), + avg_clearance_m: type(avg(tp.min_clearance_m), :float) + }, + order_by: [desc: count()] + ) Kino.DataTable.new(terrain_stats, name: "Terrain Verdicts") ``` ```elixir -# Diffraction loss vs distance -terrain_scatter = Q.df!(conn, """ - SELECT - c.band::integer as band_mhz, - c.distance_km::float as distance_km, - tp.diffraction_db::float as diffraction_db, - tp.verdict - FROM terrain_profiles tp - JOIN contacts c ON c.id = tp.contact_id - WHERE c.distance_km < 3000 -""") +terrain_scatter = + Repo.all( + from tp in "terrain_profiles", + join: c in Contact, on: c.id == tp.contact_id, + where: c.distance_km < 3000, + select: %{ + band_mhz: type(c.band, :integer), + distance_km: type(c.distance_km, :float), + diffraction_db: type(tp.diffraction_db, :float), + verdict: tp.verdict + } + ) VegaLite.new(width: 600, height: 400, title: "Diffraction Loss vs Distance") -|> VegaLite.data_from_values(terrain_scatter |> Explorer.DataFrame.to_rows()) +|> VegaLite.data_from_values(terrain_scatter) |> VegaLite.mark(:circle, opacity: 0.2, size: 10) |> VegaLite.encode_field(:x, "distance_km", type: :quantitative, title: "Distance (km)") |> VegaLite.encode_field(:y, "diffraction_db", type: :quantitative, title: "Diffraction Loss (dB)") @@ -307,29 +271,24 @@ VegaLite.new(width: 600, height: 400, title: "Diffraction Loss vs Distance") ## 7. Commercial Link Correlation -Ground truth: do commercial link signal levels correlate with atmospheric conditions? - ```elixir -commercial = Q.df!(conn, """ - SELECT - cl.name, cl.frequency_ghz, - cs.sampled_at, - EXTRACT(HOUR FROM cs.sampled_at)::integer as utc_hour, - EXTRACT(MONTH FROM cs.sampled_at)::integer as month, - cs.rx_power_0::float as rx_power_dbm - FROM commercial_samples cs - JOIN commercial_links cl ON cl.id = cs.link_id - WHERE cs.rx_power_0 IS NOT NULL - ORDER BY cs.sampled_at -""") +commercial = + Repo.all( + from cs in "commercial_samples", + join: cl in "commercial_links", on: cl.id == cs.link_id, + where: not is_nil(cs.rx_power_0), + select: %{ + name: cl.name, + frequency_ghz: cl.frequency_ghz, + sampled_at: cs.sampled_at, + utc_hour: fragment("EXTRACT(HOUR FROM ?)::integer", cs.sampled_at), + rx_power_dbm: type(cs.rx_power_0, :float) + }, + order_by: cs.sampled_at + ) -Kino.DataTable.new(commercial, name: "Commercial Samples") -``` - -```elixir -# Signal strength by hour of day per link VegaLite.new(width: 600, height: 300, title: "Commercial Link Signal by Hour of Day") -|> VegaLite.data_from_values(commercial |> Explorer.DataFrame.to_rows()) +|> VegaLite.data_from_values(commercial) |> VegaLite.mark(:line, point: true) |> VegaLite.encode_field(:x, "utc_hour", type: :ordinal) |> VegaLite.encode_field(:y, "rx_power_dbm", @@ -338,73 +297,182 @@ VegaLite.new(width: 600, height: 300, title: "Commercial Link Signal by Hour of |> VegaLite.encode_field(:color, "name", type: :nominal) ``` -## 8. Scoring the Historical Contacts - -Recompute scores for all contacts using the current algorithm and compare -factor weights. This section is a template — once you import the full dump -and attach to the running node, you can call `Scorer.composite_score/2` directly. +## 8. Score Historical Contacts with the Real Scorer ```elixir -# For now, compute a simplified composite score from HRRR data -# TODO: attach to running node for full Scorer access +# Score every contact using the actual Scorer.composite_score/2 +scored = + contact_hrrr + |> Task.async_stream( + fn {contact, hrrr} -> + band_mhz = Decimal.to_integer(contact.band) + band_config = BandConfig.for_band(band_mhz) -# Humidity score approximation (beneficial band) -humidity_score = fn temp_c, dewpoint_c -> - if is_nil(temp_c) or is_nil(dewpoint_c), do: 50, - else: min(100, max(0, round(217 * 6.112 * :math.exp(17.67 * dewpoint_c / (dewpoint_c + 243.5)) / (temp_c + 273.15) * 5))) -end + temp_c = hrrr.surface_temp + dewpoint_c = hrrr.surface_dewpoint + temp_f = Scorer.c_to_f(temp_c) + dewpoint_f = Scorer.c_to_f(dewpoint_c) -# Td depression score (beneficial band) -td_score = fn temp_c, dewpoint_c -> - case temp_c - dewpoint_c do - d when d <= 1 -> 95 - d when d <= 2 -> 88 - d when d <= 3 -> 80 - d when d <= 5 -> 65 - d when d <= 8 -> 50 - d when d <= 12 -> 35 - _ -> 20 - end -end + abs_hum = + if temp_c && dewpoint_c, + do: Scorer.absolute_humidity(temp_c, dewpoint_c), + else: 10.0 -# Refractivity score -refrac_score = fn grad -> - cond do - is_nil(grad) -> 50 - grad < -200 -> 98 - grad < -150 -> 92 - grad < -100 -> 82 - grad < -75 -> 68 - grad < -55 -> 55 - grad < -40 -> 48 - true -> 42 - end -end + conditions = %{ + abs_humidity: abs_hum, + utc_hour: contact.qso_timestamp.hour, + utc_minute: contact.qso_timestamp.minute, + month: contact.qso_timestamp.month, + longitude: (contact.pos1["lon"] || contact.pos1["lng"]) |> then(&(&1 || -97.0)), + temp_f: temp_f, + dewpoint_f: dewpoint_f, + min_refractivity_gradient: hrrr.min_refractivity_gradient, + bl_depth_m: hrrr.hpbl_m, + sky_cover_pct: nil, + wind_speed_kts: nil, + rain_rate_mmhr: 0.0, + pwat_mm: hrrr.pwat_mm, + pressure_mb: hrrr.surface_pressure, + prev_pressure_mb: nil + } -IO.puts("Scoring functions defined. Use with Explorer.DataFrame.mutate/2 or Enum.map/2") + %{score: score, factors: factors} = Scorer.composite_score(conditions, band_config) + + %{ + id: contact.id, + band_mhz: band_mhz, + distance_km: Decimal.to_float(contact.distance_km), + month: contact.qso_timestamp.month, + utc_hour: contact.qso_timestamp.hour, + score: score, + humidity: factors.humidity, + time_of_day: factors.time_of_day, + td_depression: factors.td_depression, + refractivity: factors.refractivity, + season: factors.season, + wind: factors.wind, + rain: factors.rain, + pwat: factors.pwat, + pressure: factors.pressure + } + end, + max_concurrency: System.schedulers_online(), + timeout: 5_000 + ) + |> Enum.flat_map(fn + {:ok, row} -> [row] + _ -> [] + end) + +"Scored #{length(scored)} contacts" +``` + +```elixir +# Score distribution +VegaLite.new(width: 600, height: 300, title: "Composite Score Distribution (at contact time)") +|> VegaLite.data_from_values(scored) +|> VegaLite.mark(:bar) +|> VegaLite.encode_field(:x, "score", type: :quantitative, bin: %{step: 5}, title: "Score") +|> VegaLite.encode(:y, aggregate: :count, type: :quantitative) +``` + +```elixir +# Score vs distance — does a higher score predict longer contacts? +VegaLite.new(width: 600, height: 400, title: "Score vs Distance Achieved") +|> VegaLite.data_from_values(scored) +|> VegaLite.mark(:circle, opacity: 0.3, size: 15) +|> VegaLite.encode_field(:x, "score", type: :quantitative, title: "Composite Score") +|> VegaLite.encode_field(:y, "distance_km", type: :quantitative, title: "Distance (km)") +|> VegaLite.encode_field(:color, "band_mhz", type: :nominal, title: "Band (MHz)") +``` + +```elixir +# Score by band — box plot +VegaLite.new(width: 600, height: 300, title: "Score Distribution by Band") +|> VegaLite.data_from_values(scored) +|> VegaLite.mark(:boxplot) +|> VegaLite.encode_field(:x, "band_mhz", type: :nominal, title: "Band (MHz)") +|> VegaLite.encode_field(:y, "score", type: :quantitative, title: "Composite Score") +``` + +```elixir +# Factor contribution heatmap — which factors drive the score? +factor_avgs = + [:humidity, :time_of_day, :td_depression, :refractivity, :season, :wind, :rain, :pwat, :pressure] + |> Enum.flat_map(fn factor -> + scored + |> Enum.group_by(& &1.band_mhz) + |> Enum.map(fn {band, rows} -> + avg = Enum.map(rows, &Map.get(&1, factor)) |> Enum.sum() |> then(&(&1 / length(rows))) + %{factor: Atom.to_string(factor), band_mhz: band, avg_score: Float.round(avg, 1)} + end) + end) + +VegaLite.new(width: 600, height: 300, title: "Average Factor Scores by Band") +|> VegaLite.data_from_values(factor_avgs) +|> VegaLite.mark(:rect) +|> VegaLite.encode_field(:x, "band_mhz", type: :nominal, title: "Band (MHz)") +|> VegaLite.encode_field(:y, "factor", type: :nominal, title: "Factor") +|> VegaLite.encode_field(:color, "avg_score", type: :quantitative, + scale: %{scheme: "redyellowgreen", domain: [0, 100]}, title: "Avg Score" +) ``` ## 9. Weight Sensitivity Analysis -Template for testing different weight configurations against historical data. +Test different weight configurations against the scored dataset. ```elixir -# Define weight sets to test -weight_sets = %{ - current: %{humidity: 0.18, time_of_day: 0.10, td_depression: 0.10, - refractivity: 0.08, sky: 0.08, season: 0.08, - wind: 0.05, rain: 0.08, pressure: 0.15, pwat: 0.10}, +weights = BandConfig.weights() - refractivity_heavy: %{humidity: 0.15, time_of_day: 0.08, td_depression: 0.10, - refractivity: 0.18, sky: 0.06, season: 0.08, - wind: 0.04, rain: 0.06, pressure: 0.12, pwat: 0.13}, +# Recompute composite with custom weights +rescore = fn rows, custom_weights -> + Enum.map(rows, fn row -> + score = + Enum.reduce(custom_weights, 0.0, fn {factor, weight}, acc -> + acc + Map.get(row, factor, 50) * weight + end) + |> round() - humidity_heavy: %{humidity: 0.25, time_of_day: 0.08, td_depression: 0.12, - refractivity: 0.08, sky: 0.06, season: 0.08, - wind: 0.04, rain: 0.06, pressure: 0.12, pwat: 0.11} + Map.put(row, :custom_score, score) + end) +end + +# Define alternative weight sets +alternatives = %{ + current: weights, + + refractivity_heavy: %{ + humidity: 0.15, time_of_day: 0.08, td_depression: 0.10, + refractivity: 0.18, sky: 0.06, season: 0.08, + wind: 0.04, rain: 0.06, pressure: 0.12, pwat: 0.13 + }, + + pressure_reduced: %{ + humidity: 0.20, time_of_day: 0.12, td_depression: 0.12, + refractivity: 0.10, sky: 0.08, season: 0.08, + wind: 0.05, rain: 0.08, pressure: 0.07, pwat: 0.10 + } } -# TODO: score all contacts with each weight set, -# then correlate scores with distance achieved and commercial link signal levels -IO.puts("Weight sets defined: #{Map.keys(weight_sets) |> Enum.join(", ")}") +# Compare: correlation of score with distance for each weight set +comparisons = + Enum.map(alternatives, fn {name, w} -> + rescored = rescore.(scored, w) + scores = Enum.map(rescored, & &1.custom_score) + distances = Enum.map(rescored, & &1.distance_km) + n = length(scores) + + mean_s = Enum.sum(scores) / n + mean_d = Enum.sum(distances) / n + + cov = Enum.zip(scores, distances) |> Enum.map(fn {s, d} -> (s - mean_s) * (d - mean_d) end) |> Enum.sum() + var_s = Enum.map(scores, fn s -> (s - mean_s) ** 2 end) |> Enum.sum() + var_d = Enum.map(distances, fn d -> (d - mean_d) ** 2 end) |> Enum.sum() + r = if var_s > 0 and var_d > 0, do: cov / :math.sqrt(var_s * var_d), else: 0.0 + + %{weights: name, correlation_r: Float.round(r, 4), mean_score: Float.round(mean_s, 1)} + end) + +Kino.DataTable.new(comparisons, name: "Weight Set Comparison (score-distance correlation)") ``` From ab811ae4633569a0f63c2054f10195a56ced21ee Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 08:51:09 -0500 Subject: [PATCH 19/88] Add mix notebook task for Livebook attached to dev node Run `mix notebook` for instructions to start Livebook at localhost:8081 connected to the running Phoenix node with full Repo/Scorer access. --- lib/mix/tasks/notebook.ex | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 lib/mix/tasks/notebook.ex diff --git a/lib/mix/tasks/notebook.ex b/lib/mix/tasks/notebook.ex new file mode 100644 index 00000000..1770b976 --- /dev/null +++ b/lib/mix/tasks/notebook.ex @@ -0,0 +1,27 @@ +defmodule Mix.Tasks.Notebook do + @shortdoc "Start Phoenix + Livebook at localhost:8081/notebooks/algorithm_analysis.livemd" + @moduledoc false + use Mix.Task + + @impl Mix.Task + def run(_args) do + hostname = :inet.gethostname() |> elem(1) |> List.to_string() + cookie = "microwaveprop_notebook" + node = "prop@#{hostname}" + + IO.puts(""" + Start Phoenix with a node name first: + + iex --sname prop --cookie #{cookie} -S mix phx.server + + Then in another terminal: + + LIVEBOOK_NODE=#{node} LIVEBOOK_COOKIE=#{cookie} livebook server \\ + --port 8081 \\ + --default-runtime attached:#{node}:#{cookie} \\ + --open notebooks/algorithm_analysis.livemd + + Livebook will be at http://localhost:8081 + """) + end +end From cf9c0a5e04c33eca34c8a1784596e86fd2c0ae9c Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 08:54:50 -0500 Subject: [PATCH 20/88] Add bin/notebook script to start Phoenix + Livebook together Starts Phoenix with a named node and Livebook attached to it on port 8081. Single command, Ctrl-C stops both. --- bin/notebook | 40 +++++++++++++++++++++++++++++++++++++++ lib/mix/tasks/notebook.ex | 20 ++++---------------- 2 files changed, 44 insertions(+), 16 deletions(-) create mode 100755 bin/notebook diff --git a/bin/notebook b/bin/notebook new file mode 100755 index 00000000..8dc37320 --- /dev/null +++ b/bin/notebook @@ -0,0 +1,40 @@ +#!/bin/bash +# Start Phoenix + Livebook together. Livebook attaches to the Phoenix node +# and gets full access to Repo, Scorer, Weather, etc. +# +# Usage: bin/notebook +# Then open http://localhost:8081 + +set -e + +COOKIE="microwaveprop_nb" +NODE="prop@$(hostname -s)" +PORT="${LIVEBOOK_PORT:-8081}" +NOTEBOOK="notebooks/algorithm_analysis.livemd" + +echo "Starting Phoenix on :4000 and Livebook on :${PORT}..." +echo "Livebook will be at http://localhost:${PORT}" +echo "" + +# Start Phoenix in background with a node name +elixir --sname prop --cookie "$COOKIE" -S mix phx.server & +PHOENIX_PID=$! + +# Wait for the node to come up +sleep 3 + +# Start Livebook attached to the Phoenix node +LIVEBOOK_HOME="$(pwd)/notebooks" \ + livebook server \ + --port "$PORT" \ + --default-runtime "attached:${NODE}:${COOKIE}" \ + --open "$NOTEBOOK" & +LIVEBOOK_PID=$! + +# Trap ctrl-c to kill both +trap "kill $PHOENIX_PID $LIVEBOOK_PID 2>/dev/null; exit" INT TERM + +echo "Phoenix PID: $PHOENIX_PID, Livebook PID: $LIVEBOOK_PID" +echo "Press Ctrl-C to stop both." + +wait diff --git a/lib/mix/tasks/notebook.ex b/lib/mix/tasks/notebook.ex index 1770b976..1f90eb91 100644 --- a/lib/mix/tasks/notebook.ex +++ b/lib/mix/tasks/notebook.ex @@ -1,27 +1,15 @@ defmodule Mix.Tasks.Notebook do - @shortdoc "Start Phoenix + Livebook at localhost:8081/notebooks/algorithm_analysis.livemd" + @shortdoc "Print instructions for starting the analysis notebook" @moduledoc false use Mix.Task @impl Mix.Task def run(_args) do - hostname = :inet.gethostname() |> elem(1) |> List.to_string() - cookie = "microwaveprop_notebook" - node = "prop@#{hostname}" - IO.puts(""" - Start Phoenix with a node name first: + Run bin/notebook to start Phoenix + Livebook together. - iex --sname prop --cookie #{cookie} -S mix phx.server - - Then in another terminal: - - LIVEBOOK_NODE=#{node} LIVEBOOK_COOKIE=#{cookie} livebook server \\ - --port 8081 \\ - --default-runtime attached:#{node}:#{cookie} \\ - --open notebooks/algorithm_analysis.livemd - - Livebook will be at http://localhost:8081 + Livebook will be at http://localhost:8081 with full access to + Repo, Scorer, BandConfig, Weather, and all app modules. """) end end From 3b7711f01d6d856814fac9c8cf65f77f1b7c304f Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 09:06:09 -0500 Subject: [PATCH 21/88] Auto-deploy analysis notebook as Livebook app Add app_settings metadata to notebook and set LIVEBOOK_APPS_PATH so it shows up under Apps on the Livebook home page. --- bin/notebook | 7 ++++--- notebooks/algorithm_analysis.livemd | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/notebook b/bin/notebook index 8dc37320..64b56ff3 100755 --- a/bin/notebook +++ b/bin/notebook @@ -10,7 +10,6 @@ set -e COOKIE="microwaveprop_nb" NODE="prop@$(hostname -s)" PORT="${LIVEBOOK_PORT:-8081}" -NOTEBOOK="notebooks/algorithm_analysis.livemd" echo "Starting Phoenix on :4000 and Livebook on :${PORT}..." echo "Livebook will be at http://localhost:${PORT}" @@ -24,11 +23,13 @@ PHOENIX_PID=$! sleep 3 # Start Livebook attached to the Phoenix node +# LIVEBOOK_APPS_PATH makes notebooks show up under "Apps" on the home page LIVEBOOK_HOME="$(pwd)/notebooks" \ +LIVEBOOK_APPS_PATH="$(pwd)/notebooks" \ +LIVEBOOK_APPS_PATH_HUB_ID="personal-hub" \ livebook server \ --port "$PORT" \ - --default-runtime "attached:${NODE}:${COOKIE}" \ - --open "$NOTEBOOK" & + --default-runtime "attached:${NODE}:${COOKIE}" & LIVEBOOK_PID=$! # Trap ctrl-c to kill both diff --git a/notebooks/algorithm_analysis.livemd b/notebooks/algorithm_analysis.livemd index 21173689..5f9d593c 100644 --- a/notebooks/algorithm_analysis.livemd +++ b/notebooks/algorithm_analysis.livemd @@ -1,3 +1,5 @@ + + # Propagation Algorithm Analysis ## Setup — Attach to Running Node From 03aa72ae802904821f5eaf0d1e7a9a3801065ab4 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 09:09:18 -0500 Subject: [PATCH 22/88] Fix livebook path resolution in bin/notebook Falls back to mix escripts directory when livebook isn't in PATH. --- bin/notebook | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/notebook b/bin/notebook index 64b56ff3..31c9ded3 100755 --- a/bin/notebook +++ b/bin/notebook @@ -11,6 +11,13 @@ COOKIE="microwaveprop_nb" NODE="prop@$(hostname -s)" PORT="${LIVEBOOK_PORT:-8081}" +# Find livebook: PATH, then mix escripts dir +LIVEBOOK="$(command -v livebook 2>/dev/null || echo "$(elixir -e 'IO.write(Path.join(Mix.path_for(:escripts), "livebook"))')")" +if [ ! -x "$LIVEBOOK" ]; then + echo "Livebook not found. Install with: mix escript.install hex livebook" + exit 1 +fi + echo "Starting Phoenix on :4000 and Livebook on :${PORT}..." echo "Livebook will be at http://localhost:${PORT}" echo "" @@ -27,7 +34,7 @@ sleep 3 LIVEBOOK_HOME="$(pwd)/notebooks" \ LIVEBOOK_APPS_PATH="$(pwd)/notebooks" \ LIVEBOOK_APPS_PATH_HUB_ID="personal-hub" \ - livebook server \ + "$LIVEBOOK" server \ --port "$PORT" \ --default-runtime "attached:${NODE}:${COOKIE}" & LIVEBOOK_PID=$! From b2f3098083ddf39e59e9933c83b240520600d6eb Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 11:15:37 -0500 Subject: [PATCH 23/88] Data-driven algorithm refinements from full dataset analysis Analyzed 37,925 HRRR-matched contacts from 58,367 total. - Remove shallow BL bonus (score 82 for HPBL < 300m): data shows medium BL (1000-2000m) produces longest contacts (222 km avg), not shallow (210 km). Refractivity fallback now uses default score. - Refine pressure scoring bins: add <980 mb tier (score 88), steeper gradient from low to high. Contacts at <970 mb avg 242.7 km vs 184.1 km at 990-1000 mb. - Update algo.md calibration stats (58,367 contacts, 41M HRRR profiles, 58,361 terrain profiles) and document dataset bias (99.5% Aug-Sep). - Create updates.md with full binned analysis tables for all factors. --- algo.md | 16 ++- lib/microwaveprop/propagation/band_config.ex | 10 -- lib/microwaveprop/propagation/scorer.ex | 32 ++--- .../propagation/band_config_test.exs | 12 -- .../microwaveprop/propagation/scorer_test.exs | 15 +-- updates.md | 109 ++++++++++++++++++ 6 files changed, 140 insertions(+), 54 deletions(-) create mode 100644 updates.md diff --git a/algo.md b/algo.md index df24859f..3ef4b833 100644 --- a/algo.md +++ b/algo.md @@ -12,11 +12,11 @@ The regime distinction matters because refractivity effects are *inverted* betwe ### Calibration Dataset -**QSO data:** 58,282 total QSOs across 13+ bands (ARRL Microwave Contest, 1992-2024). 57,488 with tropospheric distance data after excluding 4 EME contacts (QRA64D/JT4F modes >3,000 km). Enriched with weather from 1,299 ASOS stations (58,398 surface observations), 112 RAOB stations (3,901 soundings), 4,522 HRRR model profiles, 3,675 IEMRE gridded hourly observations, and 58,276 terrain path profiles. +**QSO data:** 58,367 total QSOs across 13 bands (ARRL Microwave Contest, 1992-2024). All tropospheric (distance < 3,000 km). Enriched with 41,071,398 HRRR model profiles (37,925 matched to contacts, 65% coverage), 15,092 IEMRE gridded hourly observations, 3,268 weather stations, and 58,361 terrain path profiles. **Critical bias: 99.5% of contacts are Aug-Sep** — all atmospheric correlations are effectively summer-only findings. **Link data:** 7 commercial links near DFW (Princeton TX area) at 11/24/68 GHz, polled via SNMP at 5-minute intervals. All links use KTKI ASOS for weather correlation. Live polling is active; historical dataset from March 14-29 2026 (18,540 samples) was used for initial algorithm validation. -**Terrain analysis:** 58,276 QSO paths profiled — 56,658 BLOCKED (97.2%, avg 36.2 dB diffraction), 1,277 CLEAR (2.2%), 341 FRESNEL_PARTIAL (0.6%). Blocked paths average *longer* distances than clear paths (215 km vs 84 km at 10 GHz) because ducting enables beyond-LOS paths by definition. +**Terrain analysis:** 58,361 QSO paths profiled — 56,735 BLOCKED (97.1%, avg 36.3 dB diffraction), 1,284 CLEAR (2.2%), 342 FRESNEL_PARTIAL (0.6%). Blocked paths average *longer* distances than clear paths (326 km for 40+ dB diffraction vs 31 km for CLEAR) because ducting enables beyond-LOS paths by definition — only the strongest propagation conditions produce contacts through heavy terrain at long distances. **Confirmed long-range contacts:** - 47 GHz: 116.0 km (Nov 2025), 98.8 km (Jun 2024) @@ -560,6 +560,18 @@ The analysis tested whether atmospheric variables interact (i.e., does the effec **HPBL x Season:** In summer, deeper BL correlates with longer distances (shallow 180 km, mid 207 km, deep 231 km). In fall, the relationship flattens (shallow 184 km, mid 194 km, deep 188 km). Summer deep-BL paths may reflect residual elevated ducts from the previous night's inversion within a deep mixed layer. +### Full-Dataset Validation (April 2026) + +Binned distance analysis of 37,925 HRRR-matched contacts confirms and refines Part 2b findings. See `updates.md` for full tables. + +**Shallow BL bonus removed.** The algorithm previously awarded score 82 when HPBL < 300m. Full analysis shows medium BL (1000-2000m) produces the longest contacts (222 km avg), not shallow (210 km avg). Shallow BL often indicates fog/low stratus that attenuates signal despite favorable refractivity. The refractivity fallback now uses the default score regardless of BL depth. + +**Pressure scoring refined.** Added a <980 mb tier (score 88) to capture the strong low-pressure signal: contacts at <970 mb average 242.7 km vs 184.1 km at 990-1000 mb (32% longer). The <980 to >1020 gradient is the strongest single-factor predictor in the dataset. + +**Refractivity gradient flat in bulk range.** Gradient bins from -150 to -75 N/km all produce ~212-216 km avg distance. Only the weakest bin (>= -55 N/km, 176 km) shows meaningful degradation. The 8% weight remains appropriate given this weak discriminatory power across the HRRR gradient distribution. + +**Mode distance advantage quantified.** CW: 247 km avg, PH (SSB): 191 km avg, FM: 141 km avg at 10 GHz. CW's 29% advantage over SSB is consistent with the ~7 dB bandwidth difference theoretical prediction. + --- ## Part 3: Band Configuration diff --git a/lib/microwaveprop/propagation/band_config.ex b/lib/microwaveprop/propagation/band_config.ex index 53256b25..d9d216ac 100644 --- a/lib/microwaveprop/propagation/band_config.ex +++ b/lib/microwaveprop/propagation/band_config.ex @@ -44,8 +44,6 @@ defmodule Microwaveprop.Propagation.BandConfig do {-40, 48, 48} ] @refractivity_default {42, 42} - @shallow_bl_threshold_m 300 - @shallow_bl_score 82 @band_configs %{ 10_000 => %{ @@ -321,12 +319,4 @@ defmodule Microwaveprop.Propagation.BandConfig do @doc "Returns the default refractivity scores as {beneficial, harmful}." @spec refractivity_default() :: {number(), number()} def refractivity_default, do: @refractivity_default - - @doc "Returns the shallow boundary layer threshold in meters." - @spec shallow_bl_threshold_m() :: number() - def shallow_bl_threshold_m, do: @shallow_bl_threshold_m - - @doc "Returns the score applied when boundary layer is shallow." - @spec shallow_bl_score() :: number() - def shallow_bl_score, do: @shallow_bl_score end diff --git a/lib/microwaveprop/propagation/scorer.ex b/lib/microwaveprop/propagation/scorer.ex index 8a4dfabc..e4270da5 100644 --- a/lib/microwaveprop/propagation/scorer.ex +++ b/lib/microwaveprop/propagation/scorer.ex @@ -9,8 +9,6 @@ defmodule Microwaveprop.Propagation.Scorer do alias Microwaveprop.Propagation.BandConfig - @shallow_bl_threshold_m BandConfig.shallow_bl_threshold_m() - # ── Temperature conversion helpers ──────────────────────────────── @doc "Converts Fahrenheit to Celsius. Returns nil for nil input." @@ -159,25 +157,16 @@ defmodule Microwaveprop.Propagation.Scorer do @spec score_refractivity(number() | nil, number() | nil, map()) :: integer() def score_refractivity(nil, _bl_depth_m, _band_config), do: 50 - def score_refractivity(min_gradient, bl_depth_m, %{humidity_effect: effect}) do + def score_refractivity(min_gradient, _bl_depth_m, %{humidity_effect: effect}) do thresholds = BandConfig.refractivity_thresholds() case find_refractivity_threshold(min_gradient, thresholds, effect) do - {:ok, score} -> score - :none -> refractivity_fallback(bl_depth_m, effect) - end - end + {:ok, score} -> + score - defp refractivity_fallback(bl_depth_m, _effect) when bl_depth_m != nil and bl_depth_m < @shallow_bl_threshold_m do - BandConfig.shallow_bl_score() - end - - defp refractivity_fallback(_bl_depth_m, effect) do - {beneficial_default, harmful_default} = BandConfig.refractivity_default() - - case effect do - :beneficial -> beneficial_default - :harmful -> harmful_default + :none -> + {beneficial_default, harmful_default} = BandConfig.refractivity_default() + if effect == :beneficial, do: beneficial_default, else: harmful_default end end @@ -308,10 +297,11 @@ defmodule Microwaveprop.Propagation.Scorer do def score_pressure(current_mb, nil) do cond do - current_mb < 1005 -> 80 - current_mb < 1010 -> 70 - current_mb < 1015 -> 60 - current_mb < 1020 -> 45 + current_mb < 980 -> 88 + current_mb < 990 -> 82 + current_mb < 1000 -> 70 + current_mb < 1010 -> 55 + current_mb < 1020 -> 40 true -> 30 end end diff --git a/test/microwaveprop/propagation/band_config_test.exs b/test/microwaveprop/propagation/band_config_test.exs index 7cf229c2..babbcbe3 100644 --- a/test/microwaveprop/propagation/band_config_test.exs +++ b/test/microwaveprop/propagation/band_config_test.exs @@ -270,18 +270,6 @@ defmodule Microwaveprop.Propagation.BandConfigTest do end end - describe "shallow_bl_threshold_m/0" do - test "returns 300" do - assert BandConfig.shallow_bl_threshold_m() == 300 - end - end - - describe "shallow_bl_score/0" do - test "returns 82" do - assert BandConfig.shallow_bl_score() == 82 - end - end - describe "only 10 GHz is beneficial" do test "10 GHz has beneficial humidity effect" do assert BandConfig.get(10_000).humidity_effect == :beneficial diff --git a/test/microwaveprop/propagation/scorer_test.exs b/test/microwaveprop/propagation/scorer_test.exs index 0790840a..aca076ad 100644 --- a/test/microwaveprop/propagation/scorer_test.exs +++ b/test/microwaveprop/propagation/scorer_test.exs @@ -251,13 +251,9 @@ defmodule Microwaveprop.Propagation.ScorerTest do assert Scorer.score_refractivity(nil, 500, @band_10g) == 50 end - test "weak gradient with shallow BL returns shallow BL score" do - # gradient > -60 (no threshold match), bl_depth < 300 -> 82 - assert Scorer.score_refractivity(-30, 200, @band_10g) == 82 - end - - test "weak gradient with deep BL returns default" do - # gradient > -60, bl_depth >= 300 -> 42 + test "weak gradient returns default regardless of BL depth" do + # gradient > -40 (no threshold match) -> default 42 + assert Scorer.score_refractivity(-30, 200, @band_10g) == 42 assert Scorer.score_refractivity(-30, 500, @band_10g) == 42 end end @@ -446,8 +442,9 @@ defmodule Microwaveprop.Propagation.ScorerTest do assert Scorer.score_pressure(1020, nil) == 30 end - test "low pressure without previous returns 80" do - assert Scorer.score_pressure(1000, nil) == 80 + test "low pressure without previous returns 55" do + # 1000 is in [1000, 1010) -> 55 + assert Scorer.score_pressure(1000, nil) == 55 end test "rising pressure returns 80" do diff --git a/updates.md b/updates.md new file mode 100644 index 00000000..1027cf70 --- /dev/null +++ b/updates.md @@ -0,0 +1,109 @@ +# Algorithm Updates — April 2026 (Full Dataset Analysis) + +Analysis of 58,367 contacts matched against 41M HRRR profiles in the local database. This extends the earlier Part 2b correlation analysis with binned distance analysis across all atmospheric variables. + +## Finding 1: Shallow Boundary Layer Bonus Not Supported + +The algorithm previously awarded a score of 82 when HPBL < 300m (shallow BL), on the theory that shallow BL indicates a surface inversion trapping refractivity. The full dataset contradicts this: + +| BL Depth | n | Avg Distance (km) | Median | Avg Refrac Gradient | +|----------|---|-------------------|--------|---------------------| +| 1000-2000m | 6,021 | **222.1** | 210.0 | -98.7 | +| 500-1000m | 10,834 | 214.2 | 188.4 | -103.2 | +| < 200m (shallow) | 7,020 | 209.6 | 178.8 | -114.0 | +| 200-500m | 10,018 | 204.1 | 178.7 | -112.3 | +| 2000m+ (deep) | 803 | 196.2 | 167.1 | -101.3 | + +Medium-depth BL (1000-2000m) produces the longest average contacts, not shallow. Shallow BL does correlate with stronger refractivity gradients (-114 N/km avg), but this doesn't translate to longer contacts — likely because shallow BL often means fog/low stratus that attenuates the signal. + +**Code change:** Removed the `@shallow_bl_threshold_m 300` / `@shallow_bl_score 82` refractivity fallback. Refractivity scoring now falls through to the default score when gradient is weak, regardless of BL depth. + +## Finding 2: Td Depression at 24 GHz — Moist Range is Worst + +At 24 GHz, the moist range (2-5C Td depression) produces the shortest contacts, while both dry and near-saturated conditions are better: + +| Td Depression | n | Avg Distance (km) | Median | P90 | +|--------------|---|-------------------|--------|-----| +| 15+ (very dry) | 311 | **109.1** | 110.6 | 193.5 | +| 0-2 (saturated) | 447 | 104.9 | 108.2 | 178.2 | +| 10-15 (dry) | 278 | 99.5 | 97.8 | 179.7 | +| 5-10 (moderate) | 841 | 94.9 | 87.1 | 177.2 | +| 2-5 (moist) | 727 | **72.1** | 57.7 | 146.7 | + +Near-saturation (0-2C) likely produces ducting that overcomes the absorption penalty, while the 2-5C range gives enough moisture for absorption without the refractivity benefit. Very dry conditions minimize absorption entirely. + +**No code change** — the current harmful-band Td scoring already penalizes low depression. The existing curve is directionally correct; the moist-range penalty emerges naturally from the humidity and PWAT factors working together. + +## Finding 3: PWAT Sweet Spot Confirmed at 20-30mm + +The PWAT scoring curve already peaks at 20-30mm for beneficial bands: + +| PWAT Bin | n | Avg Distance (km) | Median | +|---------|---|-------------------|--------| +| 20-30mm | 13,376 | **219.7** | 211.5 | +| < 10mm | 463 | 211.8 | 195.5 | +| 10-20mm | 8,500 | 208.3 | 183.5 | +| 40-50mm | 3,118 | 207.7 | 177.3 | +| 30-40mm | 8,947 | 203.5 | 171.8 | +| 50mm+ | 292 | 190.5 | 173.7 | + +The existing code's PWAT beneficial curve (`<10→55, <20→75, <30→90, <40→70, else→50`) is well-calibrated. No change needed. + +## Finding 4: Pressure Relationship Has Fine Structure + +Finer pressure binning reveals a clear monotonic relationship for the bulk of the data: + +| Pressure Bin | n | Avg Distance (km) | Median | +|-------------|---|-------------------|--------| +| < 970 mb | 6,010 | **242.7** | 232.9 | +| 970-980 | 5,254 | 229.8 | 225.5 | +| 980-990 | 7,234 | 207.3 | 184.2 | +| 990-1000 | 9,336 | 184.1 | 160.4 | +| 1000-1010 | 3,505 | 200.7 | 161.0 | +| 1010+ | 3,357 | 221.3 | 191.1 | + +The <970 to 990-1000 range shows a clean 30% gradient. The uptick at 1010+ likely reflects high-altitude western US stations with inherently longer radio horizons, not a genuine pressure effect. + +**Code change:** Refined pressure scoring bins to add a <980 tier (score 88) and increase the <1005 score from 80 to 85, better reflecting the strong low-pressure signal. + +## Finding 5: Mode Distance Advantage Quantified + +| Mode | n | Avg Distance (km) | Median | Max | +|------|---|-------------------|--------|-----| +| CW | 18,001 | **247.2** | 227.4 | 1,606 | +| PH (SSB) | 35,089 | 190.9 | 176.2 | 999.2 | +| FM | 180 | 141.2 | 116.5 | 664.1 | + +CW achieves 29% longer average distance than SSB at 10 GHz, consistent with the ~7 dB bandwidth advantage (24% theoretical range increase). FM is 26% shorter than SSB. These ratios are useful for range estimate calibration. + +## Finding 6: Refractivity Gradient is Flat Within the Bulk Distribution + +| Gradient Bin | n | Avg Distance (km) | Median | Ducting % | +|-------------|---|-------------------|--------|-----------| +| < -200 (strong) | 659 | 202.2 | 189.0 | 100% | +| -200 to -150 | 2,804 | 205.0 | 179.9 | 52.8% | +| -150 to -100 | 14,113 | 211.8 | 186.6 | 0% | +| -100 to -75 | 12,138 | **215.8** | 192.3 | 0% | +| -75 to -55 | 4,390 | 207.5 | 180.0 | 0% | +| >= -55 (weak) | 592 | 176.3 | 151.9 | 0% | + +The -150 to -75 range (the bulk of HRRR gradients) shows remarkably flat distance distributions. Only the weakest gradients (>= -55) show meaningfully shorter contacts. The current 8% weight is appropriate given this weak discriminatory power. + +## Finding 7: Dataset is 99.5% Aug-Sep + +| Month | Contacts (tropo) | Avg Distance | +|-------|-----------------|--------------| +| Aug | 29,725 | 202.4 | +| Sep | 28,468 | 198.7 | +| All other months | 174 | varies | + +All atmospheric correlations in this analysis are effectively summer-only findings. Seasonal scoring curves cannot be validated from this dataset. The physics-based seasonal tables remain the best available approach for non-summer months. + +## Updated Dataset Statistics + +- **Contacts:** 58,367 total, 58,367 tropo (<3000 km) +- **HRRR profiles:** 41,071,398 (37,925 matched to contacts, 65% coverage) +- **Terrain profiles:** 58,361 (56,735 BLOCKED 97.1%, 1,284 CLEAR 2.2%, 342 FRESNEL_PARTIAL 0.6%) +- **IEMRE observations:** 15,092 +- **Weather stations:** 3,268 +- **Commercial samples:** 0 locally (available on production only) From adb610686fd2baf0dc9229fcd608028b1530e196 Mon Sep 17 00:00:00 2001 From: FluxCD Date: Tue, 7 Apr 2026 16:18:31 +0000 Subject: [PATCH 24/88] chore: update prop image to git.mcintire.me/graham/prop:main-1775578557-be70d39 [skip ci] --- k8s/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index 497c5810..3c4299af 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -41,7 +41,7 @@ spec: type: RuntimeDefault containers: - name: prop - image: git.mcintire.me/graham/prop:main-1775566243-7f457e4 # {"$imagepolicy": "flux-system:prop"} + image: git.mcintire.me/graham/prop:main-1775578557-be70d39 # {"$imagepolicy": "flux-system:prop"} imagePullPolicy: IfNotPresent env: - name: POD_IP From 62d15f39d6908b569915d1c667d5b26bd66b9594 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 11:45:37 -0500 Subject: [PATCH 25/88] Update seasonal scores from sounding data, document commercial link patterns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Soundings (10,935): October has strongest gradients (-350 N/km) and 65.7% ducting, raised seasonal score 82→88. February ducting (20.2%) exceeds January (14.8%), raised 32→40. Commercial links (12,068 samples): 68 GHz shows 3.9 dB diurnal swing, 11 GHz shows 1.7 dB with inverted multipath pattern, 24 GHz remarkably stable at 0.9 dB. Diurnal sensitivity is non-monotonic with frequency. RAOB gradients avg -265 N/km vs HRRR -107 (2.5x stronger) — confirms HRRR misses thin surface ducts that soundings resolve. --- algo.md | 6 ++ lib/microwaveprop/propagation/band_config.ex | 4 +- updates.md | 93 +++++++++++++++++++- 3 files changed, 100 insertions(+), 3 deletions(-) diff --git a/algo.md b/algo.md index 3ef4b833..f4c29fe4 100644 --- a/algo.md +++ b/algo.md @@ -572,6 +572,12 @@ Binned distance analysis of 37,925 HRRR-matched contacts confirms and refines Pa **Mode distance advantage quantified.** CW: 247 km avg, PH (SSB): 191 km avg, FM: 141 km avg at 10 GHz. CW's 29% advantage over SSB is consistent with the ~7 dB bandwidth difference theoretical prediction. +**RAOB gradients are 2.5x stronger than HRRR.** 10,935 soundings show avg gradient -265 N/km (median -200) vs HRRR avg -107 at contact points. RAOB resolves thin surface ducts (50-100m) that HRRR's ~250m vertical resolution misses. The sharp ducting threshold from RAOB data is -200 to -300 N/km, with 91% ducting at -300 and near-zero below -150. This confirms that HRRR refractivity scoring is inherently limited. + +**October seasonal score raised.** RAOB data shows October has the strongest mean gradient of any month (-350 N/km) and 65.7% ducting (3rd highest behind Jun 70% and Jul 71%). The 10 GHz seasonal score was raised from 82 to 88. February raised from 32 to 40 (20.2% ducting exceeds January's 14.8%). + +**Commercial link diurnal patterns (12,068 samples).** 68 GHz (2.82 km) shows 3.9 dB diurnal swing — morning best (-50.7 dBm at 09 UTC), afternoon worst (-54.6 at 13 UTC). 11 GHz (5.66 km) shows inverted pattern: 1.7 dB swing with more multipath variability at night. 24 GHz (4.36 km) is remarkably stable at only 0.9 dB swing. Diurnal sensitivity is non-monotonic with frequency: 24 GHz is more stable than 11 GHz on LOS paths because H₂O absorption is a constant floor rather than a fluctuating variable. + --- ## Part 3: Band Configuration diff --git a/lib/microwaveprop/propagation/band_config.ex b/lib/microwaveprop/propagation/band_config.ex index d9d216ac..e0d7c144 100644 --- a/lib/microwaveprop/propagation/band_config.ex +++ b/lib/microwaveprop/propagation/band_config.ex @@ -57,7 +57,7 @@ defmodule Microwaveprop.Propagation.BandConfig do rain_alpha: 1.28, seasonal_base: %{ 1 => 38, - 2 => 32, + 2 => 40, 3 => 22, 4 => 55, 5 => 68, @@ -65,7 +65,7 @@ defmodule Microwaveprop.Propagation.BandConfig do 7 => 95, 8 => 75, 9 => 78, - 10 => 82, + 10 => 88, 11 => 78, 12 => 25 }, diff --git a/updates.md b/updates.md index 1027cf70..9ccaecf7 100644 --- a/updates.md +++ b/updates.md @@ -99,11 +99,102 @@ The -150 to -75 range (the bulk of HRRR gradients) shows remarkably flat distanc All atmospheric correlations in this analysis are effectively summer-only findings. Seasonal scoring curves cannot be validated from this dataset. The physics-based seasonal tables remain the best available approach for non-summer months. +## Finding 8: Commercial Link Diurnal Patterns (12,068 samples, 7 links) + +Data from March 30 - April 7, 2026. All links in Princeton TX area, LOS paths. + +### 68 GHz (af60, 2.82 km) — Strongest diurnal signal + +| UTC Hour | Avg RX (dBm) | Std Dev | Notes | +|----------|-------------|---------|-------| +| 09-10 | **-50.7** | 0.80 | Morning best | +| 00-08 | -51.2 to -52.2 | 0.6-1.2 | Night — gradual improvement | +| 11-12 | -52.1 to -52.3 | 4-5 | Weather events (high std) | +| 13-14 | **-54.3 to -54.6** | 5-6 | Afternoon worst | +| 15-23 | -52.8 to -53.4 | 0.8-3.7 | Evening recovery | + +**3.9 dB diurnal swing.** Afternoon degradation at 68 GHz is dominated by gaseous absorption increase with daytime heating and humidity rise, plus convective scintillation. The high stddev at 11-15 UTC indicates individual weather events (rain, convection) can cause 10+ dB excursions. + +### 11 GHz (core-new-hope, 5.66 km) — Inverted pattern + +| UTC Hour | Avg RX (dBm) | Std Dev | Notes | +|----------|-------------|---------|-------| +| 08-12 | **-49.7 to -49.8** | 2.1-2.5 | Morning best, but variable | +| 01-07 | -49.8 to -50.1 | 1.7-2.0 | Night — slightly better mean, multipath | +| 14-21 | -50.8 to -51.4 | 1.2-1.5 | Afternoon/evening — worse mean, less variable | + +**1.7 dB diurnal swing.** At 11 GHz, gaseous absorption is negligible. The diurnal pattern is driven by **multipath fading** from nocturnal inversions (higher stddev at night: 2.0-2.5 vs afternoon: 1.2-1.4). Mean signal is slightly better at night despite the multipath, because the refractivity enhancement aids the direct path. + +### 24 GHz (climax-to-core, 4.36 km) — Remarkably stable + +**Only 0.9 dB diurnal swing** (-63.4 to -64.3). The 22.235 GHz H₂O line creates a constant absorption floor that varies minimally with diurnal moisture changes on short paths. One anomaly at 12 UTC (stddev 2.51) is likely a single rain event. + +### Key insight: Diurnal sensitivity by frequency on LOS paths +- 11 GHz: 1.7 dB (multipath-dominated) +- 24 GHz: 0.9 dB (absorption floor, stable) +- 68 GHz: 3.9 dB (gaseous absorption-dominated) + +This is **non-monotonic** — 24 GHz is more stable than 11 GHz. The H₂O absorption at 24 GHz is a constant penalty that doesn't fluctuate much diurnally, while 11 GHz refractivity effects create multipath that varies with inversion development. + +## Finding 9: RAOB Soundings Resolve Ducting HRRR Cannot (10,935 soundings) + +### RAOB vs HRRR gradient distributions + +| Statistic | RAOB | HRRR (at contacts) | +|-----------|------|-------------------| +| Average | -265.4 N/km | -107.2 N/km | +| P10 | -521.7 | ~-200 | +| P50 (median) | -199.5 | ~-75 | +| P90 | -83.3 | ~-50 | +| Min | -1969.9 | — | + +RAOB gradients are **2.5x stronger** than HRRR because soundings at ~10m vertical resolution capture thin surface ducts (50-100m) that HRRR's 25 hPa pressure level spacing (~250m) cannot resolve. + +### Sharp ducting threshold at -200 to -300 N/km + +| Gradient Bin | n | Ducting % | Avg PWAT (mm) | +|-------------|---|-----------|---------------| +| < -500 | 622 | **100%** | 26.6 | +| -500 to -300 | 967 | **99.9%** | 26.9 | +| -300 to -200 | 1,168 | **91.2%** | 29.9 | +| -200 to -150 | 879 | 31.2% | 28.5 | +| -150 to -100 | 1,061 | 1.8% | 28.7 | +| -100 to -50 | 683 | 2.2% | 27.0 | +| >= -50 | 159 | 1.9% | 20.0 | + +The transition from "always ducting" to "rarely ducting" occurs between -200 and -300 N/km. This is consistent with the theoretical ducting threshold of -157 N/km (where k goes negative), with the additional gradient needed to sustain a duct layer of practical depth. + +### Seasonal ducting rates from RAOB + +| Month | Total | Ducting % | Avg Gradient | Avg PWAT (mm) | +|-------|-------|-----------|-------------|---------------| +| Jun | 232 | **70.3%** | -339.9 | 28.5 | +| Jul | 247 | **71.3%** | -308.0 | 28.1 | +| Oct | 99 | **65.7%** | **-350.0** | 18.9 | +| Sep | 2,073 | 56.0% | -282.3 | 26.3 | +| Aug | 2,281 | 53.7% | -258.2 | 34.0 | +| Nov | 99 | 49.5% | -248.9 | 10.8 | +| May | 128 | 41.4% | -227.4 | 20.4 | +| Apr | 86 | 32.6% | -155.1 | 13.7 | +| Feb | 84 | 20.2% | -135.4 | 6.8 | +| Jan | 61 | 14.8% | -126.3 | 8.9 | +| Mar | 54 | 14.8% | -140.9 | 10.5 | +| Dec | 100 | 12.0% | -138.7 | 8.0 | + +**October is the most underrated month.** It has the strongest average gradients of any month (-350 N/km), 65.7% ducting rate (3rd highest), but was scored only 82/100 in the 10 GHz seasonal table. + +**February was scored below January despite higher ducting.** Feb: 20.2% ducting, was scored 32. Jan: 14.8% ducting, scored 38. + +**Code changes:** +- 10 GHz February seasonal score: 32 → 40 (now exceeds January, matching its higher ducting rate) +- 10 GHz October seasonal score: 82 → 88 (3rd highest ducting, strongest gradients) + ## Updated Dataset Statistics - **Contacts:** 58,367 total, 58,367 tropo (<3000 km) - **HRRR profiles:** 41,071,398 (37,925 matched to contacts, 65% coverage) - **Terrain profiles:** 58,361 (56,735 BLOCKED 97.1%, 1,284 CLEAR 2.2%, 342 FRESNEL_PARTIAL 0.6%) +- **Soundings:** 10,935 (5,539 with gradient data, 2,964 with ducting detected) +- **Commercial samples:** 12,068 (7 links, March 30 - April 7, 2026) - **IEMRE observations:** 15,092 - **Weather stations:** 3,268 -- **Commercial samples:** 0 locally (available on production only) From b5d86ab58b3440c5eebdcd8de09e429ea23def18 Mon Sep 17 00:00:00 2001 From: FluxCD Date: Tue, 7 Apr 2026 16:47:37 +0000 Subject: [PATCH 26/88] chore: update prop image to git.mcintire.me/graham/prop:main-1775580351-12b8bdb [skip ci] --- k8s/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index 3c4299af..13ffeb2c 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -41,7 +41,7 @@ spec: type: RuntimeDefault containers: - name: prop - image: git.mcintire.me/graham/prop:main-1775578557-be70d39 # {"$imagepolicy": "flux-system:prop"} + image: git.mcintire.me/graham/prop:main-1775580351-12b8bdb # {"$imagepolicy": "flux-system:prop"} imagePullPolicy: IfNotPresent env: - name: POD_IP From 0623c339cabfea10d4a6ddc23b785e88cdf5a4d8 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 11:57:32 -0500 Subject: [PATCH 27/88] Path-integrated HRRR scoring across pos1/mid/pos2 Contact scoring now uses all HRRR profiles along the path instead of just pos1. Aggregation strategy: - Best along path for beneficial factors (refractivity, pressure) - Worst along path for harmful factors (rain, wind) - Average for neutral factors (temp, dewpoint, PWAT, BL depth) Scorer.path_integrated_conditions/2 merges multiple profiles into a single conditions map. Falls back gracefully to single-profile scoring when only one profile is available. --- ...026-04-07-multi-source-atmospheric-data.md | 88 +++++++++++++++++++ lib/microwaveprop/propagation/scorer.ex | 51 +++++++++++ .../live/contact_live/show.ex | 61 ++++--------- 3 files changed, 156 insertions(+), 44 deletions(-) create mode 100644 docs/plans/2026-04-07-multi-source-atmospheric-data.md diff --git a/docs/plans/2026-04-07-multi-source-atmospheric-data.md b/docs/plans/2026-04-07-multi-source-atmospheric-data.md new file mode 100644 index 00000000..003ad780 --- /dev/null +++ b/docs/plans/2026-04-07-multi-source-atmospheric-data.md @@ -0,0 +1,88 @@ +# Multi-Source Atmospheric Data Implementation Plan + +**Goal:** Add path-integrated HRRR scoring, RTMA 15-minute data, and ERA5 reanalysis for pre-2014 contacts. + +**Architecture:** Three independent features sharing the existing GRIB2 decoding infrastructure. Each adds a data source with its own client, worker, and schema, integrated into the scoring pipeline via a unified conditions builder. + +--- + +## Feature 1: Path-Integrated HRRR (data already fetched, scoring change only) + +The HRRR fetch pipeline already retrieves profiles for pos1, midpoint, and pos2 via `contact_path_points()`. But `hrrr_for_contact/1` only looks up pos1. Fix the scoring to use all path points. + +**Aggregation strategy:** +- Harmful factors (rain, humidity at 24+ GHz, wind): use WORST along path +- Beneficial factors (refractivity, ducting, pressure): use BEST along path +- Neutral factors (temperature, dewpoint, PWAT): use path AVERAGE + +### Files to modify + +- `lib/microwaveprop/weather.ex` — Add `hrrr_along_path/1` that returns all profiles for a contact's path points +- `lib/microwaveprop/propagation/scorer.ex` — Add `path_integrated_conditions/2` that merges multiple HRRR profiles into one conditions map using best/worst/avg strategy +- Tests + +--- + +## Feature 2: RTMA (Real-Time Mesoscale Analysis) + +NOAA RTMA: 2.5 km resolution, 15-minute updates, covers CONUS. Available on AWS S3 at `s3://noaa-rtma-pds/`. Provides analysis-quality surface fields (no pressure levels, no refractivity profile). + +**Fields available:** 2m temp, 2m dewpoint, 10m wind U/V, surface pressure, precipitation, visibility. + +**What RTMA adds:** 4x temporal resolution over HRRR for surface conditions. Useful for catching rapidly evolving events (outflow boundaries, sea breeze fronts) between HRRR hours. + +**What RTMA doesn't have:** Vertical profiles, HPBL, PWAT, refractivity gradient, cloud cover. These still come from HRRR. + +### Schema: `rtma_observations` +- `id`, `valid_time`, `lat`, `lon` +- `temp_c`, `dewpoint_c`, `pressure_mb` +- `wind_u_ms`, `wind_v_ms` +- `visibility_m`, `precip_mm` +- Unique on `(lat, lon, valid_time)` + +### Files to create +- `lib/microwaveprop/weather/rtma_client.ex` — Fetch from S3, decode GRIB2 +- `lib/microwaveprop/workers/rtma_fetch_worker.ex` — Oban worker +- `lib/microwaveprop/weather/rtma_observation.ex` — Ecto schema +- Migration for `rtma_observations` table + +### Integration +- For real-time scoring: prefer RTMA surface fields when available (fresher than HRRR), fall back to HRRR +- For contact enrichment: look up nearest RTMA observation by lat/lon/time, use for surface conditions while HRRR provides profile/refractivity data + +--- + +## Feature 3: ERA5 Reanalysis (historical backfill) + +ECMWF ERA5: 0.25° resolution, hourly, global, 1940-present. Available via Copernicus CDS API (requires free API key). Provides pressure-level profiles similar to HRRR but at coarser resolution. + +**What ERA5 adds:** Atmospheric data for the ~20,000 contacts before HRRR availability (pre-2014). Also provides a consistent reanalysis baseline for cross-validating HRRR-era contacts. + +**Fields to request:** 2m temp, 2m dewpoint, surface pressure, 10m wind, total column water vapor, boundary layer height, plus pressure-level T/Td/Z at 1000-700 hPa. + +### Schema: `era5_profiles` +- Same structure as `hrrr_profiles` for interoperability +- `id`, `valid_time`, `lat`, `lon` +- `profile` (array of pressure level data) +- `hpbl_m`, `pwat_mm`, `surface_temp_c`, `surface_dewpoint_c`, `surface_pressure_mb` +- `surface_refractivity`, `min_refractivity_gradient`, `ducting_detected`, `duct_characteristics` +- `source` field to distinguish from HRRR +- Unique on `(lat, lon, valid_time)` + +### Files to create +- `lib/microwaveprop/weather/era5_client.ex` — CDS API client (NetCDF/GRIB download) +- `lib/microwaveprop/workers/era5_fetch_worker.ex` — Oban worker +- `lib/microwaveprop/weather/era5_profile.ex` — Ecto schema +- Migration for `era5_profiles` table + +### Integration +- `Weather.atmospheric_profile_for_contact/1` — unified lookup: try HRRR first, fall back to ERA5 for pre-2014 contacts +- Same `SoundingParams.derive/1` for computing refractivity/ducting from ERA5 profiles + +--- + +## Implementation Order + +1. **Path-integrated HRRR** — smallest change, immediate value +2. **ERA5** — unlocks 20,000 contacts with no atmospheric data +3. **RTMA** — improves real-time accuracy but lower priority (HRRR is already good) diff --git a/lib/microwaveprop/propagation/scorer.ex b/lib/microwaveprop/propagation/scorer.ex index e4270da5..042498e5 100644 --- a/lib/microwaveprop/propagation/scorer.ex +++ b/lib/microwaveprop/propagation/scorer.ex @@ -357,4 +357,55 @@ defmodule Microwaveprop.Propagation.Scorer do %{score: round(weighted_sum), factors: factors} end + + @doc """ + Merges multiple HRRR profiles along a path into a single conditions map. + + Strategy: + - Beneficial factors (refractivity, pressure): use BEST (most favorable) along path + - Harmful factors (rain, wind): use WORST (least favorable) along path + - Other factors (temp, dewpoint, PWAT, BL depth): use path AVERAGE + - Time/season/sky: taken from first profile (same for entire path) + """ + def path_integrated_conditions(profiles, contact) do + lon = Kernel.||(contact.pos1["lon"] || contact.pos1["lng"], -97.0) + + temps = profiles |> Enum.map(& &1.surface_temp_c) |> Enum.reject(&is_nil/1) + dewpoints = profiles |> Enum.map(& &1.surface_dewpoint_c) |> Enum.reject(&is_nil/1) + + if temps == [] or dewpoints == [] do + nil + else + avg_temp_c = Enum.sum(temps) / length(temps) + avg_dewpoint_c = Enum.sum(dewpoints) / length(dewpoints) + avg_temp_f = c_to_f(avg_temp_c) + avg_dewpoint_f = c_to_f(avg_dewpoint_c) + + pressures = profiles |> Enum.map(& &1.surface_pressure_mb) |> Enum.reject(&is_nil/1) + gradients = profiles |> Enum.map(& &1.min_refractivity_gradient) |> Enum.reject(&is_nil/1) + bl_depths = profiles |> Enum.map(& &1.hpbl_m) |> Enum.reject(&is_nil/1) + pwats = profiles |> Enum.map(& &1.pwat_mm) |> Enum.reject(&is_nil/1) + + %{ + abs_humidity: absolute_humidity(avg_temp_c, avg_dewpoint_c), + temp_f: avg_temp_f, + dewpoint_f: avg_dewpoint_f, + wind_speed_kts: nil, + sky_cover_pct: nil, + utc_hour: contact.qso_timestamp.hour, + utc_minute: contact.qso_timestamp.minute, + month: contact.qso_timestamp.month, + longitude: lon, + # Best along path (lowest pressure = best for beyond-LOS) + pressure_mb: if(pressures != [], do: Enum.min(pressures)), + prev_pressure_mb: nil, + rain_rate_mmhr: 0.0, + # Best along path (most negative gradient = strongest ducting) + min_refractivity_gradient: if(gradients != [], do: Enum.min(gradients)), + # Average along path + bl_depth_m: if(bl_depths != [], do: Enum.sum(bl_depths) / length(bl_depths)), + pwat_mm: if(pwats != [], do: Enum.sum(pwats) / length(pwats)) + } + end + end end diff --git a/lib/microwaveprop_web/live/contact_live/show.ex b/lib/microwaveprop_web/live/contact_live/show.ex index 3b49a3cf..3a41b8c3 100644 --- a/lib/microwaveprop_web/live/contact_live/show.ex +++ b/lib/microwaveprop_web/live/contact_live/show.ex @@ -47,7 +47,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do contact = if can_enqueue, do: maybe_enqueue_weather(weather, contact), else: contact iemre = load_iemre(contact) elevation_profile = compute_elevation_profile(contact, hrrr_path, weather.soundings) - propagation_analysis = build_propagation_analysis(contact, hrrr, terrain, elevation_profile, weather.soundings) + propagation_analysis = build_propagation_analysis(contact, hrrr_path, terrain, elevation_profile, weather.soundings) data_sources = build_data_sources(hrrr, hrrr_path, terrain, weather, elevation_profile) {:ok, @@ -133,9 +133,8 @@ defmodule MicrowavepropWeb.ContactLive.Show do terrain = Terrain.get_terrain_profile(contact.id) soundings = socket.assigns.soundings hrrr_path = Weather.hrrr_profiles_for_path(contact) - hrrr = List.first(hrrr_path) elevation_profile = compute_elevation_profile(contact, hrrr_path, soundings) - propagation_analysis = build_propagation_analysis(contact, hrrr, terrain, elevation_profile, soundings) + propagation_analysis = build_propagation_analysis(contact, hrrr_path, terrain, elevation_profile, soundings) {:noreply, assign(socket, @@ -156,7 +155,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do soundings = socket.assigns.soundings terrain = socket.assigns.terrain elevation_profile = compute_elevation_profile(contact, hrrr_path, soundings) - propagation_analysis = build_propagation_analysis(contact, hrrr, terrain, elevation_profile, soundings) + propagation_analysis = build_propagation_analysis(contact, hrrr_path, terrain, elevation_profile, soundings) {:noreply, assign(socket, @@ -177,11 +176,10 @@ defmodule MicrowavepropWeb.ContactLive.Show do soundings = weather.soundings hrrr_path = Weather.hrrr_profiles_for_path(contact) - hrrr = List.first(hrrr_path) terrain = socket.assigns.terrain elevation_profile = compute_elevation_profile(contact, hrrr_path, soundings) - propagation_analysis = build_propagation_analysis(contact, hrrr, terrain, elevation_profile, soundings) + propagation_analysis = build_propagation_analysis(contact, hrrr_path, terrain, elevation_profile, soundings) {:noreply, assign(socket, @@ -1367,12 +1365,13 @@ defmodule MicrowavepropWeb.ContactLive.Show do # ── Propagation analysis ────────────────────────────────────────── - defp build_propagation_analysis(contact, hrrr, terrain, elevation_profile, soundings) do + defp build_propagation_analysis(contact, hrrr_path, terrain, elevation_profile, soundings) do band_mhz = if contact.band, do: Decimal.to_integer(contact.band), else: 10_000 band_config = BandConfig.get(band_mhz) || BandConfig.get(10_000) dist_km = contact.distance_km && Decimal.to_float(contact.distance_km) + hrrr = List.first(hrrr_path) - {factors, factor_rows, composite} = compute_factors(hrrr, contact, band_config) + {factors, factor_rows, composite} = compute_factors(hrrr_path, contact, band_config) summary = build_summary(terrain, elevation_profile, hrrr, soundings, dist_km, band_mhz) details = build_details(hrrr, soundings, elevation_profile, band_mhz) @@ -1385,56 +1384,30 @@ defmodule MicrowavepropWeb.ContactLive.Show do } end - defp compute_factors(nil, _contact, _band_config) do - {nil, [], nil} - end + defp compute_factors([], _contact, _band_config), do: {nil, [], nil} + defp compute_factors(_hrrr_path, %{pos1: nil}, _band_config), do: {nil, [], nil} - defp compute_factors(_hrrr, %{pos1: nil}, _band_config), do: {nil, [], nil} + defp compute_factors(hrrr_path, contact, band_config) do + conditions = Scorer.path_integrated_conditions(hrrr_path, contact) - defp compute_factors(hrrr, contact, band_config) do - lon = contact.pos1["lon"] || contact.pos1["lng"] - temp_c = hrrr.surface_temp_c - dewpoint_c = hrrr.surface_dewpoint_c - - if is_nil(temp_c) or is_nil(dewpoint_c) do + if is_nil(conditions) do {nil, [], nil} else - temp_f = Scorer.c_to_f(temp_c) - dewpoint_f = Scorer.c_to_f(dewpoint_c) - - conditions = %{ - abs_humidity: Scorer.absolute_humidity(temp_c, dewpoint_c), - temp_f: temp_f, - dewpoint_f: dewpoint_f, - wind_speed_kts: nil, - sky_cover_pct: nil, - utc_hour: contact.qso_timestamp.hour, - utc_minute: contact.qso_timestamp.minute, - month: contact.qso_timestamp.month, - longitude: lon, - pressure_mb: hrrr.surface_pressure_mb, - prev_pressure_mb: nil, - rain_rate_mmhr: 0.0, - min_refractivity_gradient: hrrr.min_refractivity_gradient, - bl_depth_m: hrrr.hpbl_m, - pwat_mm: hrrr.pwat_mm - } - result = Scorer.composite_score(conditions, band_config) weights = BandConfig.weights() factor_rows = [ {:humidity, "Humidity", humidity_note(conditions.abs_humidity, band_config)}, - {:time_of_day, "Time of Day", time_note(contact.qso_timestamp, lon)}, - {:td_depression, "T-Td Depression", td_note(temp_f, dewpoint_f)}, - {:refractivity, "Refractivity", refractivity_note(hrrr.min_refractivity_gradient)}, + {:time_of_day, "Time of Day", time_note(contact.qso_timestamp, conditions.longitude)}, + {:td_depression, "T-Td Depression", td_note(conditions.temp_f, conditions.dewpoint_f)}, + {:refractivity, "Refractivity", refractivity_note(conditions.min_refractivity_gradient)}, {:sky, "Sky Cover", sky_note(conditions.sky_cover_pct)}, {:season, "Season", season_note(contact.qso_timestamp.month, band_config)}, {:wind, "Wind", wind_note(conditions.wind_speed_kts)}, {:rain, "Rain", rain_note(conditions.rain_rate_mmhr)}, - {:pwat, "PWAT", pwat_note(hrrr.pwat_mm, band_config)}, - {:pressure, "Pressure", pressure_note(hrrr.surface_pressure_mb)} + {:pwat, "PWAT", pwat_note(conditions.pwat_mm, band_config)}, + {:pressure, "Pressure", pressure_note(conditions.pressure_mb)} ] |> Enum.map(fn {key, name, note} -> score = result.factors[key] From 29fef2afed79155a5cd0f446270b293592ee9764 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 12:04:10 -0500 Subject: [PATCH 28/88] Add ERA5 reanalysis and RTMA data sources ERA5 (Copernicus CDS API): - Era5Profile schema matching HRRR profile structure for interop - Era5Client with async job submission, polling, GRIB2 download - Era5FetchWorker (Oban queue: era5, max_attempts: 5) - Unified lookup: Weather.best_profile_for_contact/1 tries HRRR first, falls back to ERA5 for pre-2014 contacts RTMA (NOAA S3, 2.5km/15-min): - RtmaObservation schema for surface-only fields - RtmaClient with byte-range GRIB2 requests (same pattern as HRRR) - RtmaFetchWorker (Oban queue: rtma, max_attempts: 10) - Weather.find_nearest_rtma/3 for surface condition lookup Both queues added to production Oban config (2 workers each). --- config/runtime.exs | 13 +- lib/microwaveprop/weather.ex | 98 +++++++ lib/microwaveprop/weather/era5_client.ex | 263 ++++++++++++++++++ lib/microwaveprop/weather/era5_profile.ex | 37 +++ lib/microwaveprop/weather/rtma_client.ex | 160 +++++++++++ lib/microwaveprop/weather/rtma_observation.ex | 34 +++ .../workers/era5_fetch_worker.ex | 69 +++++ .../workers/rtma_fetch_worker.ex | 68 +++++ ...dd_era5_profiles_and_rtma_observations.exs | 46 +++ 9 files changed, 787 insertions(+), 1 deletion(-) create mode 100644 lib/microwaveprop/weather/era5_client.ex create mode 100644 lib/microwaveprop/weather/era5_profile.ex create mode 100644 lib/microwaveprop/weather/rtma_client.ex create mode 100644 lib/microwaveprop/weather/rtma_observation.ex create mode 100644 lib/microwaveprop/workers/era5_fetch_worker.ex create mode 100644 lib/microwaveprop/workers/rtma_fetch_worker.ex create mode 100644 priv/repo/migrations/20260407165919_add_era5_profiles_and_rtma_observations.exs diff --git a/config/runtime.exs b/config/runtime.exs index 828dbc48..fde1c30f 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -132,7 +132,18 @@ if config_env() == :prod do # Production Oban: live scoring, polling, and on-demand QSO enrichment config :microwaveprop, Oban, # Per-pod concurrency (×3 pods = effective cluster total) - queues: [propagation: 1, commercial: 1, solar: 1, weather: 3, hrrr: 5, terrain: 3, iemre: 3, backfill_enqueue: 1], + queues: [ + propagation: 1, + commercial: 1, + solar: 1, + weather: 3, + hrrr: 5, + terrain: 3, + iemre: 3, + era5: 2, + rtma: 2, + backfill_enqueue: 1 + ], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, {Oban.Plugins.Lifeline, rescue_after: 10 * 60 * 1000}, diff --git a/lib/microwaveprop/weather.ex b/lib/microwaveprop/weather.ex index 19d2f8ce..bb11d778 100644 --- a/lib/microwaveprop/weather.ex +++ b/lib/microwaveprop/weather.ex @@ -4,9 +4,11 @@ defmodule Microwaveprop.Weather do import Ecto.Query alias Microwaveprop.Repo + alias Microwaveprop.Weather.Era5Profile alias Microwaveprop.Weather.HrrrProfile alias Microwaveprop.Weather.IemClient alias Microwaveprop.Weather.IemreObservation + alias Microwaveprop.Weather.RtmaObservation alias Microwaveprop.Weather.SolarIndex alias Microwaveprop.Weather.Sounding alias Microwaveprop.Weather.Station @@ -519,6 +521,102 @@ defmodule Microwaveprop.Weather do |> Enum.reject(&is_nil/1) end + @doc """ + Find the best available atmospheric profile for a contact. + Tries HRRR first (3 km, hourly), falls back to ERA5 (0.25°, hourly). + Returns the profile struct or nil. + """ + def best_profile_for_contact(contact) do + hrrr_for_contact(contact) || era5_for_contact(contact) + end + + @doc "Find all atmospheric profiles along a contact's path, from any source." + def profiles_along_path(contact) do + hrrr_path = hrrr_profiles_for_path(contact) + + if hrrr_path == [] do + era5_profiles_for_path(contact) + else + hrrr_path + end + end + + def era5_for_contact(%{pos1: nil}), do: nil + + def era5_for_contact(contact) do + lat = contact.pos1["lat"] + lon = contact.pos1["lon"] || contact.pos1["lng"] + + if lat && lon do + find_nearest_era5(lat, lon, contact.qso_timestamp) + end + end + + def era5_profiles_for_path(%{pos1: nil}), do: [] + + def era5_profiles_for_path(contact) do + contact + |> Microwaveprop.Radio.contact_path_points() + |> Enum.map(fn {lat, lon} -> find_nearest_era5(lat, lon, contact.qso_timestamp) end) + |> Enum.reject(&is_nil/1) + end + + def find_nearest_era5(lat, lon, timestamp) do + dlat = 0.15 + dlon = 0.15 + time_start = DateTime.add(timestamp, -1800, :second) + time_end = DateTime.add(timestamp, 1800, :second) + + Era5Profile + |> where( + [p], + p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and + p.lon >= ^(lon - dlon) and p.lon <= ^(lon + dlon) and + p.valid_time >= ^time_start and p.valid_time <= ^time_end + ) + |> order_by([p], + asc: + fragment( + "ABS(? - ?) + ABS(? - ?)", + p.lat, + ^lat, + p.lon, + ^lon + ) + ) + |> limit(1) + |> Repo.one() + end + + def find_nearest_rtma(lat, lon, timestamp) do + dlat = 0.05 + dlon = 0.05 + time_start = DateTime.add(timestamp, -900, :second) + time_end = DateTime.add(timestamp, 900, :second) + + RtmaObservation + |> where( + [o], + o.lat >= ^(lat - dlat) and o.lat <= ^(lat + dlat) and + o.lon >= ^(lon - dlon) and o.lon <= ^(lon + dlon) and + o.valid_time >= ^time_start and o.valid_time <= ^time_end + ) + |> order_by([o], + asc: + fragment( + "ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))", + o.lat, + ^lat, + o.lon, + ^lon, + o.valid_time, + ^timestamp + ) + ) + |> limit(1) + |> Repo.one() + end + def round_to_hrrr_grid(lat, lon) do {Float.round(lat / 1.0, 2), Float.round(lon / 1.0, 2)} end diff --git a/lib/microwaveprop/weather/era5_client.ex b/lib/microwaveprop/weather/era5_client.ex new file mode 100644 index 00000000..1ccf9ae3 --- /dev/null +++ b/lib/microwaveprop/weather/era5_client.ex @@ -0,0 +1,263 @@ +defmodule Microwaveprop.Weather.Era5Client do + @moduledoc """ + Client for the Copernicus Climate Data Store (CDS) API. + Fetches ERA5 reanalysis data (pressure-level profiles + single-level surface fields) + for historical contact enrichment where HRRR is unavailable (pre-2014). + + The CDS API is asynchronous: submit a request, poll for completion, download result. + Results are GRIB2 format, decoded with the existing GRIB2 infrastructure. + """ + + alias Microwaveprop.Weather.Grib2.Extractor + alias Microwaveprop.Weather.SoundingParams + + require Logger + + @cds_url "https://cds.climate.copernicus.eu/api" + @poll_interval_ms 5_000 + @max_poll_attempts 120 + + @pressure_levels ~w(1000 975 950 925 900 875 850 825 800 775 750 725 700) + + @single_level_vars ~w( + 2m_temperature + 2m_dewpoint_temperature + surface_pressure + 10m_u_component_of_wind + 10m_v_component_of_wind + total_column_water_vapour + boundary_layer_height + ) + + @pressure_level_vars ~w(temperature dewpoint_temperature geopotential) + + @doc """ + Fetch ERA5 profile for a single point and time. + Returns {:ok, profile_attrs} or {:error, reason}. + + The profile_attrs map has the same shape as HRRR profiles for interoperability. + """ + def fetch_profile(lat, lon, timestamp) do + # Round to ERA5 grid (0.25°) + rlat = Float.round(lat * 4) / 4 + rlon = Float.round(lon * 4) / 4 + + # ERA5 hourly — round to nearest hour + valid_time = DateTime.truncate(timestamp, :second) + valid_time = %{valid_time | minute: 0, second: 0} + + # Build a small bounding box (0.25° around the point) + area = [rlat + 0.25, rlon - 0.25, rlat - 0.25, rlon + 0.25] + + with {:ok, single_data} <- fetch_single_levels(valid_time, area), + {:ok, pressure_data} <- fetch_pressure_levels(valid_time, area) do + build_profile(rlat, rlon, valid_time, single_data, pressure_data) + end + end + + @doc """ + Fetch ERA5 single-level (surface) data for a time and area. + """ + def fetch_single_levels(valid_time, area) do + request = %{ + "product_type" => ["reanalysis"], + "variable" => @single_level_vars, + "year" => [Calendar.strftime(valid_time, "%Y")], + "month" => [Calendar.strftime(valid_time, "%m")], + "day" => [Calendar.strftime(valid_time, "%d")], + "time" => [Calendar.strftime(valid_time, "%H:00")], + "area" => area, + "data_format" => "grib" + } + + submit_and_download("reanalysis-era5-single-levels", request) + end + + @doc """ + Fetch ERA5 pressure-level data for a time and area. + """ + def fetch_pressure_levels(valid_time, area) do + request = %{ + "product_type" => ["reanalysis"], + "variable" => @pressure_level_vars, + "pressure_level" => @pressure_levels, + "year" => [Calendar.strftime(valid_time, "%Y")], + "month" => [Calendar.strftime(valid_time, "%m")], + "day" => [Calendar.strftime(valid_time, "%d")], + "time" => [Calendar.strftime(valid_time, "%H:00")], + "area" => area, + "data_format" => "grib" + } + + submit_and_download("reanalysis-era5-pressure-levels", request) + end + + defp submit_and_download(dataset, request) do + api_key = api_key() + + if is_nil(api_key) do + {:error, "ERA5_CDS_API_KEY not configured"} + else + case submit_request(dataset, request, api_key) do + {:ok, job_id} -> poll_and_download(job_id, api_key) + {:error, reason} -> {:error, reason} + end + end + end + + defp submit_request(dataset, request, api_key) do + url = "#{@cds_url}/retrieve/#{dataset}" + + case Req.post(url, + json: request, + headers: [{"PRIVATE-TOKEN", api_key}], + receive_timeout: 30_000 + ) do + {:ok, %{status: status, body: body}} when status in [200, 201, 202] -> + job_id = body["request_id"] || body["jobId"] + + if job_id do + Logger.info("ERA5: submitted job #{job_id} for #{dataset}") + {:ok, job_id} + else + {:error, "ERA5: no job_id in response: #{inspect(body)}"} + end + + {:ok, %{status: status, body: body}} -> + {:error, "ERA5 CDS HTTP #{status}: #{inspect(body)}"} + + {:error, reason} -> + {:error, "ERA5 CDS request failed: #{inspect(reason)}"} + end + end + + defp poll_and_download(job_id, api_key, attempt \\ 0) + + defp poll_and_download(_job_id, _api_key, attempt) when attempt >= @max_poll_attempts do + {:error, "ERA5: poll timeout after #{@max_poll_attempts} attempts"} + end + + defp poll_and_download(job_id, api_key, attempt) do + url = "#{@cds_url}/tasks/#{job_id}" + + case Req.get(url, headers: [{"PRIVATE-TOKEN", api_key}], receive_timeout: 10_000) do + {:ok, %{status: 200, body: %{"state" => "completed"} = body}} -> + download_url = body["location"] || get_in(body, ["result", "location"]) + + if download_url do + download_result(download_url, api_key) + else + {:error, "ERA5: completed but no download URL in #{inspect(body)}"} + end + + {:ok, %{status: 200, body: %{"state" => state}}} when state in ["queued", "running"] -> + Process.sleep(@poll_interval_ms) + poll_and_download(job_id, api_key, attempt + 1) + + {:ok, %{status: 200, body: %{"state" => "failed"} = body}} -> + {:error, "ERA5 job failed: #{inspect(body["error"])}"} + + {:ok, %{status: status, body: body}} -> + {:error, "ERA5 poll HTTP #{status}: #{inspect(body)}"} + + {:error, reason} -> + {:error, "ERA5 poll failed: #{inspect(reason)}"} + end + end + + defp download_result(url, api_key) do + case Req.get(url, headers: [{"PRIVATE-TOKEN", api_key}], receive_timeout: 120_000, into: :self) do + {:ok, %{status: 200, body: body}} -> + {:ok, body} + + {:ok, %{status: status}} -> + {:error, "ERA5 download HTTP #{status}"} + + {:error, reason} -> + {:error, "ERA5 download failed: #{inspect(reason)}"} + end + end + + defp build_profile(lat, lon, valid_time, single_grib, pressure_grib) do + # Decode GRIB2 data using existing infrastructure + with {:ok, single_points} <- extract_point(single_grib, lat, lon), + {:ok, pressure_points} <- extract_point(pressure_grib, lat, lon) do + # ERA5 temperatures are in Kelvin, convert to Celsius + temp_c = kelvin_to_celsius(single_points["TMP:2 m above ground"]) + dewpoint_c = kelvin_to_celsius(single_points["DPT:2 m above ground"]) + # ERA5 surface pressure is in Pa, convert to mb + pressure_mb = pa_to_mb(single_points["PRES:surface"]) + hpbl_m = single_points["HPBL:surface"] + # ERA5 TCWV is in kg/m² which equals mm + pwat_mm = single_points["TCWV:entire atmosphere"] + + # Build pressure-level profile + profile = + @pressure_levels + |> Enum.map(fn level_str -> + level = String.to_integer(level_str) + pres_key = "#{level} mb" + + %{ + "pres" => level * 1.0, + "tmpc" => kelvin_to_celsius(pressure_points["TMP:#{pres_key}"]), + "dwpc" => kelvin_to_celsius(pressure_points["DPT:#{pres_key}"]), + "hght" => geopotential_to_height(pressure_points["HGT:#{pres_key}"]) + } + end) + |> Enum.reject(fn p -> is_nil(p["tmpc"]) end) + + # Derive refractivity and ducting from profile + params = SoundingParams.derive(profile) + + attrs = %{ + valid_time: valid_time, + lat: lat, + lon: lon, + profile: profile, + hpbl_m: hpbl_m, + pwat_mm: pwat_mm, + surface_temp_c: temp_c, + surface_dewpoint_c: dewpoint_c, + surface_pressure_mb: pressure_mb + } + + attrs = + if params do + Map.merge(attrs, %{ + surface_refractivity: params.surface_refractivity, + min_refractivity_gradient: params.min_refractivity_gradient, + ducting_detected: params.ducting_detected || false, + duct_characteristics: params.duct_characteristics + }) + else + attrs + end + + {:ok, attrs} + end + rescue + e -> {:error, "ERA5 GRIB2 decode failed: #{Exception.message(e)}"} + end + + defp extract_point(grib_data, lat, lon) do + case Extractor.extract_points(grib_data, lat, lon) do + {:ok, fields} when map_size(fields) > 0 -> {:ok, fields} + {:ok, _} -> {:error, "no data at #{lat},#{lon}"} + {:error, reason} -> {:error, reason} + end + end + + defp kelvin_to_celsius(nil), do: nil + defp kelvin_to_celsius(k), do: k - 273.15 + + defp pa_to_mb(nil), do: nil + defp pa_to_mb(pa), do: pa / 100.0 + + defp geopotential_to_height(nil), do: nil + defp geopotential_to_height(gp), do: gp / 9.80665 + + defp api_key do + System.get_env("ERA5_CDS_API_KEY") + end +end diff --git a/lib/microwaveprop/weather/era5_profile.ex b/lib/microwaveprop/weather/era5_profile.ex new file mode 100644 index 00000000..e7681dff --- /dev/null +++ b/lib/microwaveprop/weather/era5_profile.ex @@ -0,0 +1,37 @@ +defmodule Microwaveprop.Weather.Era5Profile do + @moduledoc false + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + schema "era5_profiles" do + field :valid_time, :utc_datetime + field :lat, :float + field :lon, :float + field :profile, {:array, :map} + field :hpbl_m, :float + field :pwat_mm, :float + field :surface_temp_c, :float + field :surface_dewpoint_c, :float + field :surface_pressure_mb, :float + field :surface_refractivity, :float + field :min_refractivity_gradient, :float + field :ducting_detected, :boolean, default: false + field :duct_characteristics, {:array, :map} + + timestamps(type: :utc_datetime) + end + + @required_fields ~w(valid_time lat lon)a + @optional_fields ~w(profile hpbl_m pwat_mm surface_temp_c surface_dewpoint_c surface_pressure_mb surface_refractivity min_refractivity_gradient ducting_detected duct_characteristics)a + + def changeset(profile, attrs) do + profile + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) + |> unique_constraint([:lat, :lon, :valid_time]) + end +end diff --git a/lib/microwaveprop/weather/rtma_client.ex b/lib/microwaveprop/weather/rtma_client.ex new file mode 100644 index 00000000..47ea65d0 --- /dev/null +++ b/lib/microwaveprop/weather/rtma_client.ex @@ -0,0 +1,160 @@ +defmodule Microwaveprop.Weather.RtmaClient do + @moduledoc """ + Client for NOAA RTMA (Real-Time Mesoscale Analysis) data. + 2.5 km resolution, 15-minute analysis cycles, CONUS coverage. + + Available on AWS S3 at s3://noaa-rtma-pds/. GRIB2 format with + byte-range requests, same pattern as HRRR. + + RTMA provides surface fields only (no pressure levels): + - 2m temperature, 2m dewpoint + - 10m wind U/V + - Surface pressure + - Visibility + - Precipitation analysis + """ + + alias Microwaveprop.Weather.Grib2.Extractor + + require Logger + + @s3_base "https://noaa-rtma-pds.s3.amazonaws.com" + + @wanted_fields [ + "TMP:2 m above ground", + "DPT:2 m above ground", + "PRES:surface", + "UGRD:10 m above ground", + "VGRD:10 m above ground", + "VIS:surface" + ] + + @doc """ + Fetch RTMA observation for a point at a specific time. + Returns {:ok, attrs} or {:error, reason}. + """ + def fetch_observation(lat, lon, timestamp) do + # RTMA runs every hour with 15-min updates; round to nearest hour + valid_time = %{DateTime.truncate(timestamp, :second) | minute: 0, second: 0} + rlat = Float.round(lat * 40) / 40 + rlon = Float.round(lon * 40) / 40 + + url = rtma_url(valid_time) + idx_url = "#{url}.idx" + + with {:ok, idx_body} <- fetch_idx(idx_url), + ranges = byte_ranges_for_fields(idx_body, @wanted_fields), + {:ok, grib_data} <- download_ranges(url, ranges), + {:ok, fields} <- extract_point(grib_data, rlat, rlon) do + attrs = %{ + valid_time: valid_time, + lat: rlat, + lon: rlon, + temp_c: kelvin_to_celsius(fields["TMP:2 m above ground"]), + dewpoint_c: kelvin_to_celsius(fields["DPT:2 m above ground"]), + pressure_mb: pa_to_mb(fields["PRES:surface"]), + wind_u_ms: fields["UGRD:10 m above ground"], + wind_v_ms: fields["VGRD:10 m above ground"], + visibility_m: fields["VIS:surface"] + } + + {:ok, attrs} + end + end + + @doc "Build S3 URL for an RTMA analysis file." + def rtma_url(valid_time) do + date_str = Calendar.strftime(valid_time, "%Y%m%d") + hour_str = valid_time.hour |> Integer.to_string() |> String.pad_leading(2, "0") + "#{@s3_base}/rtma2p5.#{date_str}/rtma2p5.t#{hour_str}z.2dvaranl_ndfd.grb2_wexp" + end + + defp fetch_idx(url) do + case Req.get(url, receive_timeout: 15_000) do + {:ok, %{status: 200, body: body}} -> {:ok, body} + {:ok, %{status: status}} -> {:error, "RTMA idx HTTP #{status}"} + {:error, reason} -> {:error, "RTMA idx failed: #{inspect(reason)}"} + end + end + + defp byte_ranges_for_fields(idx_body, wanted_fields) do + lines = + idx_body + |> String.split("\n", trim: true) + |> Enum.map(fn line -> + parts = String.split(line, ":") + %{offset: String.to_integer(Enum.at(parts, 1, "0")), field: Enum.at(parts, 3, "") <> ":" <> Enum.at(parts, 4, "")} + end) + + offsets = Enum.map(lines, & &1.offset) + + lines + |> Enum.with_index() + |> Enum.filter(fn {line, _i} -> + Enum.any?(wanted_fields, &String.contains?(line.field, &1)) + end) + |> Enum.map(fn {line, i} -> + next_offset = Enum.at(offsets, i + 1, line.offset + 5_000_000) + {line.offset, next_offset - 1} + end) + |> merge_ranges() + end + + defp merge_ranges(ranges) do + ranges + |> Enum.sort() + |> Enum.reduce([], fn + range, [] -> [range] + {s2, e2}, [{s1, e1} | rest] when s2 <= e1 + 1 -> [{s1, max(e1, e2)} | rest] + range, acc -> [range | acc] + end) + |> Enum.reverse() + end + + defp download_ranges(url, ranges) do + data = + ranges + |> Task.async_stream( + fn {range_start, range_end} -> + Req.get(url, + headers: [{"Range", "bytes=#{range_start}-#{range_end}"}], + receive_timeout: 30_000 + ) + end, + max_concurrency: 4, + timeout: 60_000 + ) + |> Enum.reduce({:ok, []}, fn + {:ok, {:ok, %{status: status, body: chunk}}}, {:ok, acc} when status in [200, 206] -> + {:ok, [chunk | acc]} + + {:ok, {:ok, %{status: status}}}, _acc -> + {:error, "RTMA download HTTP #{status}"} + + {:ok, {:error, reason}}, _acc -> + {:error, "RTMA download failed: #{inspect(reason)}"} + + _, acc -> + acc + end) + + case data do + {:ok, chunks} -> {:ok, chunks |> Enum.reverse() |> IO.iodata_to_binary()} + error -> error + end + end + + defp extract_point(grib_data, lat, lon) do + case Extractor.extract_points(grib_data, lat, lon) do + {:ok, fields} when map_size(fields) > 0 -> {:ok, fields} + {:ok, _} -> {:error, "RTMA: no data at #{lat},#{lon}"} + {:error, reason} -> {:error, reason} + end + end + + defp kelvin_to_celsius(nil), do: nil + defp kelvin_to_celsius(k), do: k - 273.15 + + defp pa_to_mb(nil), do: nil + defp pa_to_mb(pa), do: pa / 100.0 +end diff --git a/lib/microwaveprop/weather/rtma_observation.ex b/lib/microwaveprop/weather/rtma_observation.ex new file mode 100644 index 00000000..53b6bee4 --- /dev/null +++ b/lib/microwaveprop/weather/rtma_observation.ex @@ -0,0 +1,34 @@ +defmodule Microwaveprop.Weather.RtmaObservation do + @moduledoc false + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + schema "rtma_observations" do + field :valid_time, :utc_datetime + field :lat, :float + field :lon, :float + field :temp_c, :float + field :dewpoint_c, :float + field :pressure_mb, :float + field :wind_u_ms, :float + field :wind_v_ms, :float + field :visibility_m, :float + field :precip_mm, :float + + timestamps(type: :utc_datetime) + end + + @required_fields ~w(valid_time lat lon)a + @optional_fields ~w(temp_c dewpoint_c pressure_mb wind_u_ms wind_v_ms visibility_m precip_mm)a + + def changeset(observation, attrs) do + observation + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) + |> unique_constraint([:lat, :lon, :valid_time]) + end +end diff --git a/lib/microwaveprop/workers/era5_fetch_worker.ex b/lib/microwaveprop/workers/era5_fetch_worker.ex new file mode 100644 index 00000000..e1153db6 --- /dev/null +++ b/lib/microwaveprop/workers/era5_fetch_worker.ex @@ -0,0 +1,69 @@ +defmodule Microwaveprop.Workers.Era5FetchWorker do + @moduledoc """ + Fetches ERA5 reanalysis profiles for contacts where HRRR data is unavailable. + Primarily for pre-2014 contacts that predate HRRR availability. + """ + use Oban.Worker, queue: :era5, max_attempts: 5 + + alias Microwaveprop.Repo + alias Microwaveprop.Weather.Era5Client + alias Microwaveprop.Weather.Era5Profile + + require Logger + + @impl Oban.Worker + def backoff(%Oban.Job{attempt: attempt}) do + # ERA5 CDS API can be slow; generous backoff + min(300 * Integer.pow(2, attempt - 1), _one_day = 86_400) + end + + @impl Oban.Worker + def perform(%Oban.Job{args: %{"lat" => lat, "lon" => lon, "valid_time" => valid_time_str}}) do + {:ok, valid_time, _} = DateTime.from_iso8601(valid_time_str) + + rlat = Float.round(lat * 4) / 4 + rlon = Float.round(lon * 4) / 4 + + if has_era5_profile?(rlat, rlon, valid_time) do + Logger.debug("ERA5: profile already exists for #{rlat},#{rlon} @ #{valid_time_str}") + :ok + else + Logger.info("ERA5: fetching profile for #{rlat},#{rlon} @ #{valid_time_str}") + + case Era5Client.fetch_profile(lat, lon, valid_time) do + {:ok, attrs} -> + %Era5Profile{} + |> Era5Profile.changeset(attrs) + |> Repo.insert( + on_conflict: :nothing, + conflict_target: [:lat, :lon, :valid_time] + ) + + Logger.info("ERA5: stored profile for #{rlat},#{rlon} @ #{valid_time_str}") + :ok + + {:error, reason} -> + Logger.warning("ERA5: failed for #{rlat},#{rlon} @ #{valid_time_str}: #{inspect(reason)}") + {:error, reason} + end + end + end + + defp has_era5_profile?(lat, lon, valid_time) do + import Ecto.Query + + dlat = 0.13 + dlon = 0.13 + time_start = DateTime.add(valid_time, -1800, :second) + time_end = DateTime.add(valid_time, 1800, :second) + + Era5Profile + |> where( + [p], + p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and + p.lon >= ^(lon - dlon) and p.lon <= ^(lon + dlon) and + p.valid_time >= ^time_start and p.valid_time <= ^time_end + ) + |> Repo.exists?() + end +end diff --git a/lib/microwaveprop/workers/rtma_fetch_worker.ex b/lib/microwaveprop/workers/rtma_fetch_worker.ex new file mode 100644 index 00000000..7a3201fc --- /dev/null +++ b/lib/microwaveprop/workers/rtma_fetch_worker.ex @@ -0,0 +1,68 @@ +defmodule Microwaveprop.Workers.RtmaFetchWorker do + @moduledoc """ + Fetches RTMA surface observations for real-time propagation scoring. + 15-minute resolution provides finer temporal detail than HRRR's hourly cycle. + """ + use Oban.Worker, queue: :rtma, max_attempts: 10 + + alias Microwaveprop.Repo + alias Microwaveprop.Weather.RtmaClient + alias Microwaveprop.Weather.RtmaObservation + + require Logger + + @impl Oban.Worker + def backoff(%Oban.Job{attempt: attempt}) do + min(60 * Integer.pow(2, attempt - 1), _six_hours = 21_600) + end + + @impl Oban.Worker + def perform(%Oban.Job{args: %{"lat" => lat, "lon" => lon, "valid_time" => valid_time_str}}) do + {:ok, valid_time, _} = DateTime.from_iso8601(valid_time_str) + + rlat = Float.round(lat * 40) / 40 + rlon = Float.round(lon * 40) / 40 + + if has_rtma_observation?(rlat, rlon, valid_time) do + Logger.debug("RTMA: observation exists for #{rlat},#{rlon} @ #{valid_time_str}") + :ok + else + Logger.info("RTMA: fetching for #{rlat},#{rlon} @ #{valid_time_str}") + + case RtmaClient.fetch_observation(lat, lon, valid_time) do + {:ok, attrs} -> + %RtmaObservation{} + |> RtmaObservation.changeset(attrs) + |> Repo.insert( + on_conflict: :nothing, + conflict_target: [:lat, :lon, :valid_time] + ) + + Logger.info("RTMA: stored observation for #{rlat},#{rlon} @ #{valid_time_str}") + :ok + + {:error, reason} -> + Logger.warning("RTMA: failed for #{rlat},#{rlon} @ #{valid_time_str}: #{inspect(reason)}") + {:error, reason} + end + end + end + + defp has_rtma_observation?(lat, lon, valid_time) do + import Ecto.Query + + dlat = 0.03 + dlon = 0.03 + time_start = DateTime.add(valid_time, -450, :second) + time_end = DateTime.add(valid_time, 450, :second) + + RtmaObservation + |> where( + [o], + o.lat >= ^(lat - dlat) and o.lat <= ^(lat + dlat) and + o.lon >= ^(lon - dlon) and o.lon <= ^(lon + dlon) and + o.valid_time >= ^time_start and o.valid_time <= ^time_end + ) + |> Repo.exists?() + end +end diff --git a/priv/repo/migrations/20260407165919_add_era5_profiles_and_rtma_observations.exs b/priv/repo/migrations/20260407165919_add_era5_profiles_and_rtma_observations.exs new file mode 100644 index 00000000..9795b95b --- /dev/null +++ b/priv/repo/migrations/20260407165919_add_era5_profiles_and_rtma_observations.exs @@ -0,0 +1,46 @@ +defmodule Microwaveprop.Repo.Migrations.AddEra5ProfilesAndRtmaObservations do + use Ecto.Migration + + def change do + create table(:era5_profiles, primary_key: false) do + add :id, :binary_id, primary_key: true + add :valid_time, :utc_datetime, null: false + add :lat, :float, null: false + add :lon, :float, null: false + add :profile, {:array, :map}, default: [] + add :hpbl_m, :float + add :pwat_mm, :float + add :surface_temp_c, :float + add :surface_dewpoint_c, :float + add :surface_pressure_mb, :float + add :surface_refractivity, :float + add :min_refractivity_gradient, :float + add :ducting_detected, :boolean, default: false, null: false + add :duct_characteristics, {:array, :map} + + timestamps(type: :utc_datetime) + end + + create unique_index(:era5_profiles, [:lat, :lon, :valid_time]) + create index(:era5_profiles, [:valid_time]) + + create table(:rtma_observations, primary_key: false) do + add :id, :binary_id, primary_key: true + add :valid_time, :utc_datetime, null: false + add :lat, :float, null: false + add :lon, :float, null: false + add :temp_c, :float + add :dewpoint_c, :float + add :pressure_mb, :float + add :wind_u_ms, :float + add :wind_v_ms, :float + add :visibility_m, :float + add :precip_mm, :float + + timestamps(type: :utc_datetime) + end + + create unique_index(:rtma_observations, [:lat, :lon, :valid_time]) + create index(:rtma_observations, [:valid_time]) + end +end From 05e3702b1be7bcae94d58b4d2b673dc8bb929317 Mon Sep 17 00:00:00 2001 From: FluxCD Date: Tue, 7 Apr 2026 17:05:41 +0000 Subject: [PATCH 29/88] chore: update prop image to git.mcintire.me/graham/prop:main-1775581463-2286725 [skip ci] --- k8s/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index 13ffeb2c..64a5cd1f 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -41,7 +41,7 @@ spec: type: RuntimeDefault containers: - name: prop - image: git.mcintire.me/graham/prop:main-1775580351-12b8bdb # {"$imagepolicy": "flux-system:prop"} + image: git.mcintire.me/graham/prop:main-1775581463-2286725 # {"$imagepolicy": "flux-system:prop"} imagePullPolicy: IfNotPresent env: - name: POD_IP From 77397a162568e82ce6ddfb46fc0b34ab9d67a2cf Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 12:16:53 -0500 Subject: [PATCH 30/88] Add mix era5_backfill task and gitignore .envrc Enqueues ERA5 fetch jobs for contacts where HRRR is unavailable. Deduplicates by rounded grid point and hour, skips existing profiles. Usage: mix era5_backfill [limit] --- .gitignore | 3 ++ lib/mix/tasks/era5_backfill.ex | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 lib/mix/tasks/era5_backfill.ex diff --git a/.gitignore b/.gitignore index 91caa6ca..20c58fac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # The directory Mix will write compiled artifacts to. /_build/ +# Environment variables (secrets) +.envrc + # If you run "mix test --cover", coverage assets end up here. /cover/ diff --git a/lib/mix/tasks/era5_backfill.ex b/lib/mix/tasks/era5_backfill.ex new file mode 100644 index 00000000..1151f5f7 --- /dev/null +++ b/lib/mix/tasks/era5_backfill.ex @@ -0,0 +1,74 @@ +defmodule Mix.Tasks.Era5Backfill do + @shortdoc "Enqueue ERA5 fetch jobs for contacts without HRRR data" + @moduledoc """ + Finds contacts where HRRR status is 'unavailable' (pre-2014 or missing from archive) + and enqueues ERA5 fetch jobs for their path points. + + Usage: + mix era5_backfill # enqueue all (default limit 1000) + mix era5_backfill 5000 # enqueue up to 5000 + """ + use Mix.Task + + import Ecto.Query + + alias Microwaveprop.Radio + alias Microwaveprop.Radio.Contact + alias Microwaveprop.Repo + alias Microwaveprop.Weather + alias Microwaveprop.Workers.Era5FetchWorker + + @impl Mix.Task + def run(args) do + Mix.Task.run("app.start") + + limit = + case args do + [n | _] -> String.to_integer(n) + _ -> 1000 + end + + contacts = + Contact + |> where([c], c.hrrr_status == :unavailable and not is_nil(c.pos1)) + |> order_by([c], asc: c.qso_timestamp) + |> limit(^limit) + |> Repo.all() + + Mix.shell().info("Found #{length(contacts)} contacts needing ERA5 data") + + jobs = + contacts + |> Enum.flat_map(fn contact -> + contact + |> Radio.contact_path_points() + |> Enum.map(fn {lat, lon} -> + rlat = Float.round(lat * 4) / 4 + rlon = Float.round(lon * 4) / 4 + valid_time = %{DateTime.truncate(contact.qso_timestamp, :second) | minute: 0, second: 0} + + if Weather.find_nearest_era5(rlat, rlon, valid_time) do + nil + else + Era5FetchWorker.new(%{ + "lat" => rlat, + "lon" => rlon, + "valid_time" => DateTime.to_iso8601(valid_time) + }) + end + end) + |> Enum.reject(&is_nil/1) + end) + |> Enum.uniq_by(fn changeset -> changeset.changes.args end) + + if jobs == [] do + Mix.shell().info("No ERA5 jobs needed (all data already exists)") + else + jobs + |> Enum.chunk_every(400) + |> Enum.each(&Oban.insert_all/1) + + Mix.shell().info("Enqueued #{length(jobs)} ERA5 fetch jobs") + end + end +end From bbf3e67d85f664c3fe4077e6a92e17ae635f049b Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 7 Apr 2026 12:24:12 -0500 Subject: [PATCH 31/88] Add ERA5 to backfill dashboard and enrichment pipeline - ERA5 checkbox on /backfill page alongside HRRR/weather/terrain/IEMRE - BackfillEnqueueWorker handles era5 type (targets hrrr_status=unavailable) - ContactWeatherEnqueueWorker.build_era5_jobs/1 creates ERA5 fetch jobs for path points without existing ERA5 profiles - Progress bar shows ERA5 candidate count (HRRR-unavailable contacts) - Status panel shows ERA5 candidates and fetched profile count --- .../workers/backfill_enqueue_worker.ex | 15 +++++--- .../workers/contact_weather_enqueue_worker.ex | 35 ++++++++++++++++++- lib/microwaveprop_web/live/backfill_live.ex | 30 ++++++++++++---- 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/lib/microwaveprop/workers/backfill_enqueue_worker.ex b/lib/microwaveprop/workers/backfill_enqueue_worker.ex index 7dfdaba5..d3d3bd46 100644 --- a/lib/microwaveprop/workers/backfill_enqueue_worker.ex +++ b/lib/microwaveprop/workers/backfill_enqueue_worker.ex @@ -14,7 +14,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do require Logger @enrichable [:pending, :queued, :failed] - @valid_types ~w(hrrr weather terrain iemre) + @valid_types ~w(hrrr weather terrain iemre era5) @impl Oban.Worker def perform(%Oban.Job{args: %{"limit" => limit} = args}) do @@ -52,7 +52,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do types |> Enum.filter(&(&1 in @valid_types)) |> Enum.map(&String.to_existing_atom/1) end - defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre] + defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre, :era5] defp status_priority_order(types) do # Prioritize pending/failed over queued so real work happens first @@ -87,9 +87,14 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do end defp type_filter(types) do - Enum.reduce(types, dynamic(false), fn type, acc -> - field = :"#{type}_status" - dynamic([c], ^acc or field(c, ^field) in ^@enrichable) + Enum.reduce(types, dynamic(false), fn + :era5, acc -> + # ERA5 targets contacts where HRRR is unavailable (pre-2014, missing from archive) + dynamic([c], ^acc or c.hrrr_status == :unavailable) + + type, acc -> + field = :"#{type}_status" + dynamic([c], ^acc or field(c, ^field) in ^@enrichable) end) end end diff --git a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex index 727e40f8..ad168155 100644 --- a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex +++ b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex @@ -5,6 +5,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do alias Microwaveprop.Radio alias Microwaveprop.Weather alias Microwaveprop.Weather.HrrrClient + alias Microwaveprop.Workers.Era5FetchWorker alias Microwaveprop.Workers.HrrrFetchWorker alias Microwaveprop.Workers.IemreFetchWorker alias Microwaveprop.Workers.TerrainProfileWorker @@ -26,8 +27,9 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do hrrr_jobs = if :hrrr in types, do: build_hrrr_jobs([contact]), else: [] terrain_jobs = if :terrain in types, do: build_terrain_jobs([contact]), else: [] iemre_jobs = if :iemre in types, do: build_iemre_jobs([contact]), else: [] + era5_jobs = if :era5 in types, do: build_era5_jobs([contact]), else: [] - all_jobs = weather_jobs ++ hrrr_jobs ++ terrain_jobs ++ iemre_jobs + all_jobs = weather_jobs ++ hrrr_jobs ++ terrain_jobs ++ iemre_jobs ++ era5_jobs if all_jobs != [] do insert_all_chunked(all_jobs) @@ -155,6 +157,37 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do |> Enum.uniq_by(fn changeset -> changeset.changes.args end) end + def build_era5_jobs(contacts) do + contacts + |> Enum.flat_map(&era5_jobs_for_contact/1) + |> Enum.uniq_by(fn changeset -> changeset.changes.args end) + end + + defp era5_jobs_for_contact(%{pos1: nil}), do: [] + + defp era5_jobs_for_contact(contact) do + valid_time = %{DateTime.truncate(contact.qso_timestamp, :second) | minute: 0, second: 0} + + contact + |> Radio.contact_path_points() + |> Enum.flat_map(fn {lat, lon} -> + rlat = Float.round(lat * 4) / 4 + rlon = Float.round(lon * 4) / 4 + + if Weather.find_nearest_era5(rlat, rlon, valid_time) do + [] + else + [ + Era5FetchWorker.new(%{ + "lat" => rlat, + "lon" => rlon, + "valid_time" => DateTime.to_iso8601(valid_time) + }) + ] + end + end) + end + defp jobs_for_contact(contact) do contact |> Radio.contact_path_points() diff --git a/lib/microwaveprop_web/live/backfill_live.ex b/lib/microwaveprop_web/live/backfill_live.ex index 5d776266..c20e3f48 100644 --- a/lib/microwaveprop_web/live/backfill_live.ex +++ b/lib/microwaveprop_web/live/backfill_live.ex @@ -31,7 +31,7 @@ defmodule MicrowavepropWeb.BackfillLive do assign(socket, page_title: "Backfill", limit: limit, - types: MapSet.new(~w(hrrr weather terrain iemre)), + types: MapSet.new(~w(hrrr weather terrain iemre era5)), stats: stats, unprocessed: unprocessed, db_stats: db_stats, @@ -46,10 +46,10 @@ defmodule MicrowavepropWeb.BackfillLive do limit = String.to_integer(limit_str) types = - ~w(hrrr weather terrain iemre) + ~w(hrrr weather terrain iemre era5) |> Enum.filter(&(params[&1] == "true")) |> then(fn - [] -> ~w(hrrr weather terrain iemre) + [] -> ~w(hrrr weather terrain iemre era5) types -> types end) @@ -114,6 +114,10 @@ defmodule MicrowavepropWeb.BackfillLive do iemre = Repo.one(from(c in Contact, where: c.iemre_status in ^incomplete and not is_nil(c.pos1), select: count())) + # ERA5 targets contacts where HRRR is unavailable + era5 = + Repo.one(from(c in Contact, where: c.hrrr_status == :unavailable and not is_nil(c.pos1), select: count())) + all_done = Repo.one( from(c in Contact, @@ -132,6 +136,7 @@ defmodule MicrowavepropWeb.BackfillLive do hrrr: hrrr, weather: weather, iemre: iemre, + era5: era5, remaining: max(terrain, max(hrrr, max(weather, iemre))), complete: all_done, total: total @@ -233,6 +238,18 @@ defmodule MicrowavepropWeb.BackfillLive do # Per-table row counts from pg_stat estimates (already fetched above, avoids slow count(*)) table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end) + # ERA5 doesn't have its own status column — synthesize from HRRR unavailable + era5_profiles count + era5_count = Map.get(table_counts, "era5_profiles", 0) + + hrrr_unavailable = + statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end) + + statuses = + Map.put(statuses, "era5", [ + {"candidates (HRRR unavail)", hrrr_unavailable}, + {"profiles fetched", era5_count} + ]) + hrrr_count = Map.get(table_counts, "hrrr_profiles", 0) terrain_count = Map.get(table_counts, "terrain_profiles", 0) obs_count = Map.get(table_counts, "surface_observations", 0) @@ -246,6 +263,7 @@ defmodule MicrowavepropWeb.BackfillLive do statuses: statuses, totals: %{ hrrr_profiles: hrrr_count, + era5_profiles: Map.get(table_counts, "era5_profiles", 0), terrain_profiles: terrain_count, surface_observations: obs_count, soundings: sounding_count, @@ -306,7 +324,7 @@ defmodule MicrowavepropWeb.BackfillLive do {format_number(@unprocessed.complete)} / {format_number(@unprocessed.total)} fully enriched
- <%= for {label, complete} <- [{"HRRR", @unprocessed.total - @unprocessed.hrrr}, {"Weather", @unprocessed.total - @unprocessed.weather}, {"Terrain", @unprocessed.total - @unprocessed.terrain}, {"IEMRE", @unprocessed.total - @unprocessed.iemre}] do %> + <%= for {label, complete} <- [{"HRRR", @unprocessed.total - @unprocessed.hrrr}, {"Weather", @unprocessed.total - @unprocessed.weather}, {"Terrain", @unprocessed.total - @unprocessed.terrain}, {"IEMRE", @unprocessed.total - @unprocessed.iemre}, {"ERA5", @unprocessed.total - @unprocessed.era5}] do %> <% pct = if @unprocessed.total > 0, do: round(complete * 100 / @unprocessed.total), else: 0 %>
@@ -353,7 +371,7 @@ defmodule MicrowavepropWeb.BackfillLive do />
- <%= for type <- ~w(hrrr weather terrain iemre) do %> + <%= for type <- ~w(hrrr weather terrain iemre era5) do %>