From d275e9e7c4e61305776f16710a1a59a7a9899dbe Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 2 Apr 2026 15:30:41 -0500 Subject: [PATCH] fix enrichment --- config/dev.exs | 10 +-- config/runtime.exs | 73 +++++++++---------- lib/microwaveprop/markdown.ex | 8 +- lib/microwaveprop/radio.ex | 18 ++--- lib/microwaveprop/radio/contact.ex | 6 +- lib/microwaveprop/radio/csv_import.ex | 66 ++++++++--------- lib/microwaveprop/weather/hrrr_client.ex | 24 +++++- .../workers/terrain_profile_worker.ex | 72 +++++++++--------- .../live/contact_live/index.ex | 1 + .../live/contact_live/show.ex | 1 - lib/microwaveprop_web/live/map_live.ex | 5 +- lib/microwaveprop_web/live/submit_live.ex | 12 ++- lib/microwaveprop_web/plugs/remote_ip.ex | 3 +- lib/mix/tasks/hrrr_backfill.ex | 42 ++++++----- mix.exs | 2 +- ...replace_enrichment_booleans_with_enums.exs | 23 +++++- test/microwaveprop/radio/csv_import_test.exs | 58 ++++++++------- .../contact_weather_enqueue_worker_test.exs | 4 +- test/mix/tasks/reset_enrichment_test.exs | 4 +- 19 files changed, 239 insertions(+), 193 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index 9ec51295..86a963e8 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -85,17 +85,17 @@ config :microwaveprop, Oban, # Enable dev routes for dashboard and mailbox config :microwaveprop, dev_routes: true +# HRRR caching handled by nginx proxy on skippy — no local filesystem cache + +# Proxy HRRR downloads through local caching nginx (set to nil to use S3 directly) +config :microwaveprop, hrrr_base_url: "http://skippy.w5isp.com:8080" + # Load ML model at startup (Nx/Axon/EXLA only available in dev/test) config :microwaveprop, load_ml_model: true # Use local SRTM1 tiles for elevation lookups instead of the Open-Meteo API config :microwaveprop, srtm_tiles_dir: Path.expand("~/srtm/tiles") -# HRRR caching handled by nginx proxy on skippy — no local filesystem cache - -# Proxy HRRR downloads through local caching nginx (set to nil to use S3 directly) -config :microwaveprop, hrrr_base_url: "http://skippy.w5isp.com:8080" - # Freshness monitor watches for stale propagation scores config :microwaveprop, start_freshness_monitor: true diff --git a/config/runtime.exs b/config/runtime.exs index 01ed22fb..c12d17e0 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -46,43 +46,6 @@ if config_env() == :prod do host = System.get_env("PHX_HOST") || "example.com" - config :microwaveprop, Microwaveprop.Repo, - # ssl: true, - url: database_url, - pool_size: String.to_integer(System.get_env("POOL_SIZE") || "20"), - # For machines with several cores, consider starting multiple pools of `pool_size` - # pool_count: 4, - socket_options: maybe_ipv6 - - config :microwaveprop, MicrowavepropWeb.Endpoint, - # SSL terminated by Cloudflare tunnel; generated URLs still use https - url: [host: host, port: 443, scheme: "https"], - http: [ - # Enable IPv6 and bind on all interfaces. - # Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access. - # See the documentation on https://hexdocs.pm/bandit/Bandit.html#t:options/0 - # for details about using IPv6 vs IPv4 and loopback vs public addresses. - ip: {0, 0, 0, 0, 0, 0, 0, 0} - ], - secret_key_base: secret_key_base - - # Production Oban: live scoring, polling, and on-demand QSO enrichment (no cron backfill) - config :microwaveprop, Oban, - queues: [propagation: 1, commercial: 2, solar: 1, weather: 5, hrrr: 5, terrain: 2, iemre: 5], - plugins: [ - {Oban.Plugins.Pruner, max_age: 3600 * 24}, - {Oban.Plugins.Lifeline, rescue_after: 30 * 60 * 1000}, - {Oban.Plugins.Cron, - crontab: [ - {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, - {"*/5 * * * *", Microwaveprop.Commercial.PollWorker}, - {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker} - ]} - ] - - config :microwaveprop, srtm_tiles_dir: "/srtm" - config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080") - # ## SSL Support # # To get SSL working, you will need to add the `https` key @@ -132,5 +95,41 @@ if config_env() == :prod do auth: :always, ssl: false + config :microwaveprop, Microwaveprop.Repo, + # ssl: true, + url: database_url, + pool_size: String.to_integer(System.get_env("POOL_SIZE") || "20"), + # For machines with several cores, consider starting multiple pools of `pool_size` + # pool_count: 4, + socket_options: maybe_ipv6 + + config :microwaveprop, MicrowavepropWeb.Endpoint, + # SSL terminated by Cloudflare tunnel; generated URLs still use https + url: [host: host, port: 443, scheme: "https"], + http: [ + # Enable IPv6 and bind on all interfaces. + # Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access. + # See the documentation on https://hexdocs.pm/bandit/Bandit.html#t:options/0 + # for details about using IPv6 vs IPv4 and loopback vs public addresses. + ip: {0, 0, 0, 0, 0, 0, 0, 0} + ], + secret_key_base: secret_key_base + + # Production Oban: live scoring, polling, and on-demand QSO enrichment (no cron backfill) + config :microwaveprop, Oban, + queues: [propagation: 1, commercial: 2, solar: 1, weather: 5, hrrr: 5, terrain: 2, iemre: 5], + plugins: [ + {Oban.Plugins.Pruner, max_age: 3600 * 24}, + {Oban.Plugins.Lifeline, rescue_after: 30 * 60 * 1000}, + {Oban.Plugins.Cron, + crontab: [ + {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, + {"*/5 * * * *", Microwaveprop.Commercial.PollWorker}, + {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker} + ]} + ] + config :microwaveprop, :email_from, {"NTMS Propagation", "graham@w5isp.com"} + config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080") + config :microwaveprop, srtm_tiles_dir: "/srtm" end diff --git a/lib/microwaveprop/markdown.ex b/lib/microwaveprop/markdown.ex index 151ef7ce..f44e538a 100644 --- a/lib/microwaveprop/markdown.ex +++ b/lib/microwaveprop/markdown.ex @@ -23,7 +23,7 @@ defmodule Microwaveprop.Markdown do defp parse_blocks(["```" <> lang | rest], acc) do {code_lines, remaining} = take_until_fence(rest, []) lang = String.trim(lang) - lang_attr = if lang != "", do: " class=\"language-#{lang}\"", else: "" + lang_attr = if lang == "", do: "", else: " class=\"language-#{lang}\"" parse_blocks(remaining, [{:code, lang_attr, code_lines} | acc]) end @@ -123,11 +123,11 @@ defmodule Microwaveprop.Markdown do end defp render_block({:paragraph, lines}) do - "

#{lines |> Enum.map_join(" ", &inline/1)}

" + "

#{Enum.map_join(lines, " ", &inline/1)}

" end defp render_block({:code, lang_attr, lines}) do - escaped = lines |> Enum.map_join("\n", &escape_html/1) + escaped = Enum.map_join(lines, "\n", &escape_html/1) "
#{escaped}
" end @@ -146,7 +146,7 @@ defmodule Microwaveprop.Markdown do end defp render_block({:table, lines}) do - rows = lines |> Enum.map(&parse_table_row/1) + rows = Enum.map(lines, &parse_table_row/1) case rows do [header, _separator | body] -> diff --git a/lib/microwaveprop/radio.ex b/lib/microwaveprop/radio.ex index 52e0cfa3..84f61b32 100644 --- a/lib/microwaveprop/radio.ex +++ b/lib/microwaveprop/radio.ex @@ -112,17 +112,14 @@ defmodule Microwaveprop.Radio do Repo.all(query) end - def unprocessed_contacts(limit \\ 500), - do: contacts_needing_enrichment(:weather_status, limit) + def unprocessed_contacts(limit \\ 500), do: contacts_needing_enrichment(:weather_status, limit) - def unprocessed_hrrr_contacts(limit \\ 500), - do: contacts_needing_enrichment(:hrrr_status, limit) + def unprocessed_hrrr_contacts(limit \\ 500), do: contacts_needing_enrichment(:hrrr_status, limit) def unprocessed_terrain_contacts(limit \\ 500), do: contacts_needing_enrichment(:terrain_status, limit, &where(&1, [q], not is_nil(q.pos2))) - def unprocessed_iemre_contacts(limit \\ 500), - do: contacts_needing_enrichment(:iemre_status, limit) + def unprocessed_iemre_contacts(limit \\ 500), do: contacts_needing_enrichment(:iemre_status, limit) def set_enrichment_status!(ids, field, status) do Contact @@ -231,7 +228,8 @@ defmodule Microwaveprop.Radio do distance = if pos1 && pos2 do - haversine_km(pos1["lat"], pos1["lon"], pos2["lat"], pos2["lon"]) + pos1["lat"] + |> haversine_km(pos1["lon"], pos2["lat"], pos2["lon"]) |> round() |> Decimal.new() else @@ -244,12 +242,12 @@ defmodule Microwaveprop.Radio do |> then(fn c -> if pos2 && is_nil(contact.pos2), do: Map.put(c, :pos2, pos2), else: c end) |> then(fn c -> if distance && is_nil(contact.distance_km), do: Map.put(c, :distance_km, distance), else: c end) - if changes != %{} do + if changes == %{} do + contact + else contact |> Ecto.Changeset.change(changes) |> Repo.update!() - else - contact end else contact diff --git a/lib/microwaveprop/radio/contact.ex b/lib/microwaveprop/radio/contact.ex index 971b671b..43b40d29 100644 --- a/lib/microwaveprop/radio/contact.ex +++ b/lib/microwaveprop/radio/contact.ex @@ -20,6 +20,7 @@ defmodule Microwaveprop.Radio.Contact do field :mode, :string field :band, :decimal field :distance_km, :decimal + field :hrrr_status, Ecto.Enum, values: [:pending, :queued, :processing, :complete, :failed, :unavailable], default: :pending @@ -35,6 +36,7 @@ defmodule Microwaveprop.Radio.Contact do field :iemre_status, Ecto.Enum, values: [:pending, :queued, :processing, :complete, :failed, :unavailable], default: :pending + field :user_submitted, :boolean, default: false field :submitter_email, :string field :flagged_invalid, :boolean, default: false @@ -93,9 +95,7 @@ defmodule Microwaveprop.Radio.Contact do # Callsigns: letters, digits, and / only (e.g. W5XD, KG5CCI/P, VE3/W5XD) defp validate_callsign(changeset, field) do - validate_format(changeset, field, ~r/^[A-Z0-9\/]+$/, - message: "must contain only letters, digits, and /" - ) + validate_format(changeset, field, ~r/^[A-Z0-9\/]+$/, message: "must contain only letters, digits, and /") end defp validate_grid_format(changeset, field) do diff --git a/lib/microwaveprop/radio/csv_import.ex b/lib/microwaveprop/radio/csv_import.ex index b90711a1..ff5c80d9 100644 --- a/lib/microwaveprop/radio/csv_import.ex +++ b/lib/microwaveprop/radio/csv_import.ex @@ -91,40 +91,38 @@ defmodule Microwaveprop.Radio.CsvImport do # Each returns {year, month, day, hour, minute, second} or nil. defp timestamp_parsers do [ - # ISO-ish: 2024-06-15T14:30:00Z, 2024-06-15 14:30, 2024/06/15 14:30:00, etc. - # Accepts T or space separator, optional seconds, optional timezone suffix - {~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})[T\s]+(\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$"i, - fn - [_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s} - [_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"} - end}, - # Date only: 2024-06-15, 2024/06/15 - {~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})$", - fn [_, y, m, d] -> {y, m, d, "00", "00", "00"} end}, - # US with AM/PM: 6/15/2024 2:30 PM, 6-15-2024 2:30PM, 06.15.2024 02:30 am - {~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(AM|PM)$"i, - fn - [_, m, d, y, h, mi, s, ampm] when byte_size(s) > 0 -> - {y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, s} + # ISO-ish: 2024-06-15T14:30:00Z, 2024-06-15 14:30, 2024/06/15 14:30:00, etc. + # Accepts T or space separator, optional seconds, optional timezone suffix + {~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})[T\s]+(\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$"i, + fn + [_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s} + [_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"} + end}, + # Date only: 2024-06-15, 2024/06/15 + {~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})$", fn [_, y, m, d] -> {y, m, d, "00", "00", "00"} end}, + # US with AM/PM: 6/15/2024 2:30 PM, 6-15-2024 2:30PM, 06.15.2024 02:30 am + {~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(AM|PM)$"i, + fn + [_, m, d, y, h, mi, s, ampm] when byte_size(s) > 0 -> + {y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, s} - [_, m, d, y, h, mi, _, ampm] -> - {y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, "00"} - end}, - # US 24h: 6/15/2024 14:30, 06-15-2024 14:30:00, 6.15.2024 14:30 - {~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?$", - fn - [_, m, d, y, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s} - [_, m, d, y, h, mi | _] -> {y, m, d, h, mi, "00"} - end}, - # US date only: 6/15/2024, 06-15-2024 - {~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})$", - fn [_, m, d, y] -> {y, m, d, "00", "00", "00"} end}, - # Compact: 20240615T143000, 20240615 1430, 20240615T1430 - {~r"^(\d{4})(\d{2})(\d{2})[T\s]?(\d{2})(\d{2})(\d{2})?$", - fn - [_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s} - [_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"} - end} + [_, m, d, y, h, mi, _, ampm] -> + {y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, "00"} + end}, + # US 24h: 6/15/2024 14:30, 06-15-2024 14:30:00, 6.15.2024 14:30 + {~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?$", + fn + [_, m, d, y, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s} + [_, m, d, y, h, mi | _] -> {y, m, d, h, mi, "00"} + end}, + # US date only: 6/15/2024, 06-15-2024 + {~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})$", fn [_, m, d, y] -> {y, m, d, "00", "00", "00"} end}, + # Compact: 20240615T143000, 20240615 1430, 20240615T1430 + {~r"^(\d{4})(\d{2})(\d{2})[T\s]?(\d{2})(\d{2})(\d{2})?$", + fn + [_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s} + [_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"} + end} ] end @@ -204,7 +202,7 @@ defmodule Microwaveprop.Radio.CsvImport do defp parse_quoted_field(<<"\"", rest::binary>>, acc, current) do # End of quoted field — skip to next comma or end case rest do - <<"," , rest2::binary>> -> do_parse_fields(rest2, [current | acc], "") + <<",", rest2::binary>> -> do_parse_fields(rest2, [current | acc], "") "" -> [current | acc] _ -> do_parse_fields(rest, [current | acc], "") end diff --git a/lib/microwaveprop/weather/hrrr_client.ex b/lib/microwaveprop/weather/hrrr_client.ex index bdcce945..105f9612 100644 --- a/lib/microwaveprop/weather/hrrr_client.ex +++ b/lib/microwaveprop/weather/hrrr_client.ex @@ -14,8 +14,19 @@ defmodule Microwaveprop.Weather.HrrrClient do # Fine-grained levels below 900mb (~1km) for duct detection, plus standard upper levels. # Every 25mb from 1000-900 for ~80m vertical spacing near surface. @pressure_levels [ - 1000, 975, 950, 925, 900, - 875, 850, 825, 800, 775, 750, 725, 700 + 1000, + 975, + 950, + 925, + 900, + 875, + 850, + 825, + 800, + 775, + 750, + 725, + 700 ] @surface_messages [ @@ -371,9 +382,12 @@ defmodule Microwaveprop.Weather.HrrrClient do defp read_cache(key) do case hrrr_cache_dir() do - nil -> :miss + nil -> + :miss + dir -> path = Path.join(dir, key) + case File.read(path) do {:ok, binary} -> {:ok, binary} {:error, _} -> :miss @@ -383,7 +397,9 @@ defmodule Microwaveprop.Weather.HrrrClient do defp write_cache(key, binary) do case hrrr_cache_dir() do - nil -> :ok + nil -> + :ok + dir -> File.mkdir_p!(dir) path = Path.join(dir, key) diff --git a/lib/microwaveprop/workers/terrain_profile_worker.ex b/lib/microwaveprop/workers/terrain_profile_worker.ex index bf7193f4..b435c312 100644 --- a/lib/microwaveprop/workers/terrain_profile_worker.ex +++ b/lib/microwaveprop/workers/terrain_profile_worker.ex @@ -27,49 +27,49 @@ defmodule Microwaveprop.Workers.TerrainProfileWorker do dist_km = Decimal.to_float(contact.distance_km || Decimal.new(0)) freq_ghz = Decimal.to_float(contact.band) / 1000 - # Look up HRRR refractivity gradient for dynamic k-factor - k = lookup_k_factor(contact) + # Look up HRRR refractivity gradient for dynamic k-factor + k = lookup_k_factor(contact) - case ElevationClient.fetch_elevation_profile(lat1, lon1, lat2, lon2, 64, download: true) do - {:ok, profile} -> - analysis = TerrainAnalysis.analyse(profile, dist_km, freq_ghz, k_factor: k) + case ElevationClient.fetch_elevation_profile(lat1, lon1, lat2, lon2, 64, download: true) do + {:ok, profile} -> + analysis = TerrainAnalysis.analyse(profile, dist_km, freq_ghz, k_factor: k) - path_points = - Enum.map(profile, fn p -> - %{ - "lat" => p.lat, - "lon" => p.lon, - "d" => p.d, - "elev" => p.elev, - "dist_km" => p.dist_km - } - end) + path_points = + Enum.map(profile, fn p -> + %{ + "lat" => p.lat, + "lon" => p.lon, + "d" => p.d, + "elev" => p.elev, + "dist_km" => p.dist_km + } + end) - Terrain.upsert_terrain_profile(%{ - qso_id: qso_id, - sample_count: length(profile), - path_points: path_points, - max_elevation_m: analysis.max_elevation_m, - min_clearance_m: analysis.min_clearance_m, - diffraction_db: analysis.diffraction_db, - fresnel_hit_count: analysis.fresnel_hit_count, - obstructed_count: analysis.obstructed_count, - verdict: analysis.verdict - }) + Terrain.upsert_terrain_profile(%{ + qso_id: qso_id, + sample_count: length(profile), + path_points: path_points, + max_elevation_m: analysis.max_elevation_m, + min_clearance_m: analysis.min_clearance_m, + diffraction_db: analysis.diffraction_db, + fresnel_hit_count: analysis.fresnel_hit_count, + obstructed_count: analysis.obstructed_count, + verdict: analysis.verdict + }) - Radio.set_enrichment_status!([qso_id], :terrain_status, :complete) + Radio.set_enrichment_status!([qso_id], :terrain_status, :complete) - Phoenix.PubSub.broadcast( - Microwaveprop.PubSub, - "contact_enrichment:#{qso_id}", - {:terrain_ready, qso_id} - ) + Phoenix.PubSub.broadcast( + Microwaveprop.PubSub, + "contact_enrichment:#{qso_id}", + {:terrain_ready, qso_id} + ) - :ok + :ok - {:error, reason} -> - {:error, reason} - end + {:error, reason} -> + {:error, reason} + end else _ -> Radio.set_enrichment_status!([qso_id], :terrain_status, :unavailable) diff --git a/lib/microwaveprop_web/live/contact_live/index.ex b/lib/microwaveprop_web/live/contact_live/index.ex index c88659c5..63078558 100644 --- a/lib/microwaveprop_web/live/contact_live/index.ex +++ b/lib/microwaveprop_web/live/contact_live/index.ex @@ -20,6 +20,7 @@ defmodule MicrowavepropWeb.ContactLive.Index do {n, _} -> max(n, 1) :error -> 1 end + sort_by = validate_sort_field(Map.get(params, "sort_by", @default_sort_by)) sort_order = validate_sort_order(Map.get(params, "sort_order", @default_sort_order)) search = Map.get(params, "search", "") diff --git a/lib/microwaveprop_web/live/contact_live/show.ex b/lib/microwaveprop_web/live/contact_live/show.ex index 578f5a39..661b55e9 100644 --- a/lib/microwaveprop_web/live/contact_live/show.ex +++ b/lib/microwaveprop_web/live/contact_live/show.ex @@ -346,7 +346,6 @@ defmodule MicrowavepropWeb.ContactLive.Show do ) end - defp load_solar(contact) do contact.qso_timestamp |> DateTime.to_date() diff --git a/lib/microwaveprop_web/live/map_live.ex b/lib/microwaveprop_web/live/map_live.ex index b312cc2d..9bf8fb68 100644 --- a/lib/microwaveprop_web/live/map_live.ex +++ b/lib/microwaveprop_web/live/map_live.ex @@ -277,7 +277,10 @@ defmodule MicrowavepropWeb.MapLive do <%!-- Top-left control panel --%> -
+
diff --git a/lib/microwaveprop_web/live/submit_live.ex b/lib/microwaveprop_web/live/submit_live.ex index 3d625282..3e95f309 100644 --- a/lib/microwaveprop_web/live/submit_live.ex +++ b/lib/microwaveprop_web/live/submit_live.ex @@ -168,7 +168,10 @@ defmodule MicrowavepropWeb.SubmitLive do