graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix enrichment

Browse source
This commit is contained in:
Graham McIntire 2026-04-02 15:30:41 -05:00
parent d95098daa9
commit d275e9e7c4
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
19 changed files with 239 additions and 193 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
config/dev.exs
View file

@ -85,17 +85,17 @@ config :microwaveprop, Oban,
# Enable dev routes for dashboard and mailbox
config :microwaveprop, dev_routes: true
# HRRR caching handled by nginx proxy on skippy — no local filesystem cache
# Proxy HRRR downloads through local caching nginx (set to nil to use S3 directly)
config :microwaveprop, hrrr_base_url: "http://skippy.w5isp.com:8080"
# Load ML model at startup (Nx/Axon/EXLA only available in dev/test)
config :microwaveprop, load_ml_model: true
# Use local SRTM1 tiles for elevation lookups instead of the Open-Meteo API
config :microwaveprop, srtm_tiles_dir: Path.expand("~/srtm/tiles")
# HRRR caching handled by nginx proxy on skippy — no local filesystem cache
# Proxy HRRR downloads through local caching nginx (set to nil to use S3 directly)
config :microwaveprop, hrrr_base_url: "http://skippy.w5isp.com:8080"
# Freshness monitor watches for stale propagation scores
config :microwaveprop, start_freshness_monitor: true

73
config/runtime.exs
View file

@ -46,43 +46,6 @@ if config_env() == :prod do
host = System.get_env("PHX_HOST") || "example.com"
config :microwaveprop, Microwaveprop.Repo,
# ssl: true,
url: database_url,
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "20"),
# For machines with several cores, consider starting multiple pools of `pool_size`
# pool_count: 4,
socket_options: maybe_ipv6
config :microwaveprop, MicrowavepropWeb.Endpoint,
# SSL terminated by Cloudflare tunnel; generated URLs still use https
url: [host: host, port: 443, scheme: "https"],
http: [
# Enable IPv6 and bind on all interfaces.
# Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access.
# See the documentation on https://hexdocs.pm/bandit/Bandit.html#t:options/0
# for details about using IPv6 vs IPv4 and loopback vs public addresses.
ip: {0, 0, 0, 0, 0, 0, 0, 0}
],
secret_key_base: secret_key_base
# Production Oban: live scoring, polling, and on-demand QSO enrichment (no cron backfill)
config :microwaveprop, Oban,
queues: [propagation: 1, commercial: 2, solar: 1, weather: 5, hrrr: 5, terrain: 2, iemre: 5],
plugins: [
{Oban.Plugins.Pruner, max_age: 3600 * 24},
{Oban.Plugins.Lifeline, rescue_after: 30 * 60 * 1000},
{Oban.Plugins.Cron,
crontab: [
{"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
{"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker}
]}
]
config :microwaveprop, srtm_tiles_dir: "/srtm"
config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080")
# ## SSL Support
#
# To get SSL working, you will need to add the `https` key
@ -132,5 +95,41 @@ if config_env() == :prod do
auth: :always,
ssl: false
config :microwaveprop, Microwaveprop.Repo,
# ssl: true,
url: database_url,
pool_size: String.to_integer(System.get_env("POOL_SIZE") || "20"),
# For machines with several cores, consider starting multiple pools of `pool_size`
# pool_count: 4,
socket_options: maybe_ipv6
config :microwaveprop, MicrowavepropWeb.Endpoint,
# SSL terminated by Cloudflare tunnel; generated URLs still use https
url: [host: host, port: 443, scheme: "https"],
http: [
# Enable IPv6 and bind on all interfaces.
# Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access.
# See the documentation on https://hexdocs.pm/bandit/Bandit.html#t:options/0
# for details about using IPv6 vs IPv4 and loopback vs public addresses.
ip: {0, 0, 0, 0, 0, 0, 0, 0}
],
secret_key_base: secret_key_base
# Production Oban: live scoring, polling, and on-demand QSO enrichment (no cron backfill)
config :microwaveprop, Oban,
queues: [propagation: 1, commercial: 2, solar: 1, weather: 5, hrrr: 5, terrain: 2, iemre: 5],
plugins: [
{Oban.Plugins.Pruner, max_age: 3600 * 24},
{Oban.Plugins.Lifeline, rescue_after: 30 * 60 * 1000},
{Oban.Plugins.Cron,
crontab: [
{"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
{"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker}
]}
]
config :microwaveprop, :email_from, {"NTMS Propagation", "graham@w5isp.com"}
config :microwaveprop, hrrr_base_url: System.get_env("HRRR_BASE_URL", "http://skippy.w5isp.com:8080")
config :microwaveprop, srtm_tiles_dir: "/srtm"
end

8
lib/microwaveprop/markdown.ex
View file

@ -23,7 +23,7 @@ defmodule Microwaveprop.Markdown do
defp parse_blocks(["```" <> lang | rest], acc) do
{code_lines, remaining} = take_until_fence(rest, [])
lang = String.trim(lang)
lang_attr = if lang != "", do: " class=\"language-#{lang}\"", else: ""
lang_attr = if lang == "", do: "", else: " class=\"language-#{lang}\""
parse_blocks(remaining, [{:code, lang_attr, code_lines} | acc])
end
@ -123,11 +123,11 @@ defmodule Microwaveprop.Markdown do
end
defp render_block({:paragraph, lines}) do
"<p>#{lines |> Enum.map_join(" ", &inline/1)}</p>"
"<p>#{Enum.map_join(lines, " ", &inline/1)}</p>"
end
defp render_block({:code, lang_attr, lines}) do
escaped = lines |> Enum.map_join("\n", &escape_html/1)
escaped = Enum.map_join(lines, "\n", &escape_html/1)
"<pre><code#{lang_attr}>#{escaped}</code></pre>"
end
@ -146,7 +146,7 @@ defmodule Microwaveprop.Markdown do
end
defp render_block({:table, lines}) do
rows = lines |> Enum.map(&parse_table_row/1)
rows = Enum.map(lines, &parse_table_row/1)
case rows do
[header, _separator | body] ->

18
lib/microwaveprop/radio.ex
View file

@ -112,17 +112,14 @@ defmodule Microwaveprop.Radio do
Repo.all(query)
end
def unprocessed_contacts(limit \\ 500),
do: contacts_needing_enrichment(:weather_status, limit)
def unprocessed_contacts(limit \\ 500), do: contacts_needing_enrichment(:weather_status, limit)
def unprocessed_hrrr_contacts(limit \\ 500),
do: contacts_needing_enrichment(:hrrr_status, limit)
def unprocessed_hrrr_contacts(limit \\ 500), do: contacts_needing_enrichment(:hrrr_status, limit)
def unprocessed_terrain_contacts(limit \\ 500),
do: contacts_needing_enrichment(:terrain_status, limit, &where(&1, [q], not is_nil(q.pos2)))
def unprocessed_iemre_contacts(limit \\ 500),
do: contacts_needing_enrichment(:iemre_status, limit)
def unprocessed_iemre_contacts(limit \\ 500), do: contacts_needing_enrichment(:iemre_status, limit)
def set_enrichment_status!(ids, field, status) do
Contact
@ -231,7 +228,8 @@ defmodule Microwaveprop.Radio do
distance =
if pos1 && pos2 do
haversine_km(pos1["lat"], pos1["lon"], pos2["lat"], pos2["lon"])
pos1["lat"]
|> haversine_km(pos1["lon"], pos2["lat"], pos2["lon"])
|> round()
|> Decimal.new()
else
@ -244,12 +242,12 @@ defmodule Microwaveprop.Radio do
|> then(fn c -> if pos2 && is_nil(contact.pos2), do: Map.put(c, :pos2, pos2), else: c end)
|> then(fn c -> if distance && is_nil(contact.distance_km), do: Map.put(c, :distance_km, distance), else: c end)
if changes != %{} do
if changes == %{} do
contact
else
contact
|> Ecto.Changeset.change(changes)
|> Repo.update!()
else
contact
end
else
contact

6
lib/microwaveprop/radio/contact.ex
View file

@ -20,6 +20,7 @@ defmodule Microwaveprop.Radio.Contact do
field :mode, :string
field :band, :decimal
field :distance_km, :decimal
field :hrrr_status, Ecto.Enum,
values: [:pending, :queued, :processing, :complete, :failed, :unavailable],
default: :pending
@ -35,6 +36,7 @@ defmodule Microwaveprop.Radio.Contact do
field :iemre_status, Ecto.Enum,
values: [:pending, :queued, :processing, :complete, :failed, :unavailable],
default: :pending
field :user_submitted, :boolean, default: false
field :submitter_email, :string
field :flagged_invalid, :boolean, default: false
@ -93,9 +95,7 @@ defmodule Microwaveprop.Radio.Contact do
# Callsigns: letters, digits, and / only (e.g. W5XD, KG5CCI/P, VE3/W5XD)
defp validate_callsign(changeset, field) do
validate_format(changeset, field, ~r/^[A-Z0-9\/]+$/,
message: "must contain only letters, digits, and /"
)
validate_format(changeset, field, ~r/^[A-Z0-9\/]+$/, message: "must contain only letters, digits, and /")
end
defp validate_grid_format(changeset, field) do

66
lib/microwaveprop/radio/csv_import.ex
View file

@ -91,40 +91,38 @@ defmodule Microwaveprop.Radio.CsvImport do
# Each returns {year, month, day, hour, minute, second} or nil.
defp timestamp_parsers do
[
# ISO-ish: 2024-06-15T14:30:00Z, 2024-06-15 14:30, 2024/06/15 14:30:00, etc.
# Accepts T or space separator, optional seconds, optional timezone suffix
{~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})[T\s]+(\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$"i,
fn
[_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s}
[_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"}
end},
# Date only: 2024-06-15, 2024/06/15
{~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})$",
fn [_, y, m, d] -> {y, m, d, "00", "00", "00"} end},
# US with AM/PM: 6/15/2024 2:30 PM, 6-15-2024 2:30PM, 06.15.2024 02:30 am
{~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(AM|PM)$"i,
fn
[_, m, d, y, h, mi, s, ampm] when byte_size(s) > 0 ->
{y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, s}
# ISO-ish: 2024-06-15T14:30:00Z, 2024-06-15 14:30, 2024/06/15 14:30:00, etc.
# Accepts T or space separator, optional seconds, optional timezone suffix
{~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})[T\s]+(\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$"i,
fn
[_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s}
[_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"}
end},
# Date only: 2024-06-15, 2024/06/15
{~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})$", fn [_, y, m, d] -> {y, m, d, "00", "00", "00"} end},
# US with AM/PM: 6/15/2024 2:30 PM, 6-15-2024 2:30PM, 06.15.2024 02:30 am
{~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(AM|PM)$"i,
fn
[_, m, d, y, h, mi, s, ampm] when byte_size(s) > 0 ->
{y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, s}
[_, m, d, y, h, mi, _, ampm] ->
{y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, "00"}
end},
# US 24h: 6/15/2024 14:30, 06-15-2024 14:30:00, 6.15.2024 14:30
{~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?$",
fn
[_, m, d, y, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s}
[_, m, d, y, h, mi | _] -> {y, m, d, h, mi, "00"}
end},
# US date only: 6/15/2024, 06-15-2024
{~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})$",
fn [_, m, d, y] -> {y, m, d, "00", "00", "00"} end},
# Compact: 20240615T143000, 20240615 1430, 20240615T1430
{~r"^(\d{4})(\d{2})(\d{2})[T\s]?(\d{2})(\d{2})(\d{2})?$",
fn
[_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s}
[_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"}
end}
[_, m, d, y, h, mi, _, ampm] ->
{y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, "00"}
end},
# US 24h: 6/15/2024 14:30, 06-15-2024 14:30:00, 6.15.2024 14:30
{~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?$",
fn
[_, m, d, y, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s}
[_, m, d, y, h, mi | _] -> {y, m, d, h, mi, "00"}
end},
# US date only: 6/15/2024, 06-15-2024
{~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})$", fn [_, m, d, y] -> {y, m, d, "00", "00", "00"} end},
# Compact: 20240615T143000, 20240615 1430, 20240615T1430
{~r"^(\d{4})(\d{2})(\d{2})[T\s]?(\d{2})(\d{2})(\d{2})?$",
fn
[_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s}
[_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"}
end}
]
end
@ -204,7 +202,7 @@ defmodule Microwaveprop.Radio.CsvImport do
defp parse_quoted_field(<<"\"", rest::binary>>, acc, current) do
# End of quoted field — skip to next comma or end
case rest do
<<"," , rest2::binary>> -> do_parse_fields(rest2, [current | acc], "")
<<",", rest2::binary>> -> do_parse_fields(rest2, [current | acc], "")
"" -> [current | acc]
_ -> do_parse_fields(rest, [current | acc], "")
end

24
lib/microwaveprop/weather/hrrr_client.ex
View file

@ -14,8 +14,19 @@ defmodule Microwaveprop.Weather.HrrrClient do
# Fine-grained levels below 900mb (~1km) for duct detection, plus standard upper levels.
# Every 25mb from 1000-900 for ~80m vertical spacing near surface.
@pressure_levels [
1000, 975, 950, 925, 900,
875, 850, 825, 800, 775, 750, 725, 700
1000,
975,
950,
925,
900,
875,
850,
825,
800,
775,
750,
725,
700
]
@surface_messages [
@ -371,9 +382,12 @@ defmodule Microwaveprop.Weather.HrrrClient do
defp read_cache(key) do
case hrrr_cache_dir() do
nil -> :miss
nil ->
:miss
dir ->
path = Path.join(dir, key)
case File.read(path) do
{:ok, binary} -> {:ok, binary}
{:error, _} -> :miss
@ -383,7 +397,9 @@ defmodule Microwaveprop.Weather.HrrrClient do
defp write_cache(key, binary) do
case hrrr_cache_dir() do
nil -> :ok
nil ->
:ok
dir ->
File.mkdir_p!(dir)
path = Path.join(dir, key)

72
lib/microwaveprop/workers/terrain_profile_worker.ex
View file

@ -27,49 +27,49 @@ defmodule Microwaveprop.Workers.TerrainProfileWorker do
dist_km = Decimal.to_float(contact.distance_km || Decimal.new(0))
freq_ghz = Decimal.to_float(contact.band) / 1000
# Look up HRRR refractivity gradient for dynamic k-factor
k = lookup_k_factor(contact)
# Look up HRRR refractivity gradient for dynamic k-factor
k = lookup_k_factor(contact)
case ElevationClient.fetch_elevation_profile(lat1, lon1, lat2, lon2, 64, download: true) do
{:ok, profile} ->
analysis = TerrainAnalysis.analyse(profile, dist_km, freq_ghz, k_factor: k)
case ElevationClient.fetch_elevation_profile(lat1, lon1, lat2, lon2, 64, download: true) do
{:ok, profile} ->
analysis = TerrainAnalysis.analyse(profile, dist_km, freq_ghz, k_factor: k)
path_points =
Enum.map(profile, fn p ->
%{
"lat" => p.lat,
"lon" => p.lon,
"d" => p.d,
"elev" => p.elev,
"dist_km" => p.dist_km
}
end)
path_points =
Enum.map(profile, fn p ->
%{
"lat" => p.lat,
"lon" => p.lon,
"d" => p.d,
"elev" => p.elev,
"dist_km" => p.dist_km
}
end)
Terrain.upsert_terrain_profile(%{
qso_id: qso_id,
sample_count: length(profile),
path_points: path_points,
max_elevation_m: analysis.max_elevation_m,
min_clearance_m: analysis.min_clearance_m,
diffraction_db: analysis.diffraction_db,
fresnel_hit_count: analysis.fresnel_hit_count,
obstructed_count: analysis.obstructed_count,
verdict: analysis.verdict
})
Terrain.upsert_terrain_profile(%{
qso_id: qso_id,
sample_count: length(profile),
path_points: path_points,
max_elevation_m: analysis.max_elevation_m,
min_clearance_m: analysis.min_clearance_m,
diffraction_db: analysis.diffraction_db,
fresnel_hit_count: analysis.fresnel_hit_count,
obstructed_count: analysis.obstructed_count,
verdict: analysis.verdict
})
Radio.set_enrichment_status!([qso_id], :terrain_status, :complete)
Radio.set_enrichment_status!([qso_id], :terrain_status, :complete)
Phoenix.PubSub.broadcast(
Microwaveprop.PubSub,
"contact_enrichment:#{qso_id}",
{:terrain_ready, qso_id}
)
Phoenix.PubSub.broadcast(
Microwaveprop.PubSub,
"contact_enrichment:#{qso_id}",
{:terrain_ready, qso_id}
)
:ok
:ok
{:error, reason} ->
{:error, reason}
end
{:error, reason} ->
{:error, reason}
end
else
_ ->
Radio.set_enrichment_status!([qso_id], :terrain_status, :unavailable)

1
lib/microwaveprop_web/live/contact_live/index.ex
View file

@ -20,6 +20,7 @@ defmodule MicrowavepropWeb.ContactLive.Index do
{n, _} -> max(n, 1)
:error -> 1
end
sort_by = validate_sort_field(Map.get(params, "sort_by", @default_sort_by))
sort_order = validate_sort_order(Map.get(params, "sort_order", @default_sort_order))
search = Map.get(params, "search", "")

1
lib/microwaveprop_web/live/contact_live/show.ex
View file

@ -346,7 +346,6 @@ defmodule MicrowavepropWeb.ContactLive.Show do
)
end
defp load_solar(contact) do
contact.qso_timestamp
|> DateTime.to_date()

5
lib/microwaveprop_web/live/map_live.ex
View file

@ -277,7 +277,10 @@ defmodule MicrowavepropWeb.MapLive do
</div>
<%!-- Top-left control panel --%>
<div id="map-controls" class="absolute top-2 left-12 z-[1000] flex flex-col gap-2 max-w-[calc(100vw-4rem)] md:left-14 md:top-3 md:max-w-none">
<div
id="map-controls"
class="absolute top-2 left-12 z-[1000] flex flex-col gap-2 max-w-[calc(100vw-4rem)] md:left-14 md:top-3 md:max-w-none"
>
<div class="bg-base-100/90 shadow rounded-box border border-base-300 p-2 md:p-3 flex flex-col gap-2">
<div class="font-bold text-sm leading-tight px-1 flex items-center justify-between gap-2">
<div class="min-w-0">

12
lib/microwaveprop_web/live/submit_live.ex
View file

@ -168,7 +168,10 @@ defmodule MicrowavepropWeb.SubmitLive do
<button
class={[
"px-4 py-2 text-sm font-medium rounded-md transition-colors",
if(@active_tab == :single, do: "bg-primary text-primary-content", else: "hover:bg-base-300")
if(@active_tab == :single,
do: "bg-primary text-primary-content",
else: "hover:bg-base-300"
)
]}
phx-click="switch_tab"
phx-value-tab="single"
@ -275,10 +278,11 @@ defmodule MicrowavepropWeb.SubmitLive do
<code class="text-xs">station1, station2, grid1, grid2, band, mode, qso_timestamp</code>
<ul class="list-disc list-inside mt-2 space-y-1 text-base-content/60">
<li>
Timestamps in most formats accepted (e.g. <code>2024-06-15T14:30:00Z</code>,
<code>6/15/2024 2:30 PM</code>, <code>2024-06-15 14:30</code>). All times assumed UTC.
Timestamps in most formats accepted (e.g. <code>2024-06-15T14:30:00Z</code>, <code>6/15/2024 2:30 PM</code>, <code>2024-06-15 14:30</code>). All times assumed UTC.
</li>
<li>
Grid squares should be as detailed as possible (6 characters preferred, e.g. EM12kp)
</li>
<li>Grid squares should be as detailed as possible (6 characters preferred, e.g. EM12kp)</li>
<li>Band in MHz (e.g. 10000, 24000)</li>
</ul>
<p class="mt-2">

3
lib/microwaveprop_web/plugs/remote_ip.ex
View file

@ -16,8 +16,7 @@ defmodule MicrowavepropWeb.Plugs.RemoteIp do
def call(conn, _opts) do
ip =
@ip_headers
|> Enum.find_value(fn header ->
Enum.find_value(@ip_headers, fn header ->
case Plug.Conn.get_req_header(conn, header) do
[value | _] ->
value

42
lib/mix/tasks/hrrr_backfill.ex
View file

@ -35,12 +35,13 @@ defmodule Mix.Tasks.HrrrBackfill do
# Get all distinct (rounded_hour, path_points) combinations for contacts
contacts =
from(c in Contact,
where: not is_nil(c.pos1) and c.qso_timestamp >= ^~U[2014-01-01 00:00:00Z],
select: %{id: c.id, pos1: c.pos1, pos2: c.pos2, qso_timestamp: c.qso_timestamp},
order_by: [desc: c.qso_timestamp]
Repo.all(
from(c in Contact,
where: not is_nil(c.pos1) and c.qso_timestamp >= ^~U[2014-01-01 00:00:00Z],
select: %{id: c.id, pos1: c.pos1, pos2: c.pos2, qso_timestamp: c.qso_timestamp},
order_by: [desc: c.qso_timestamp]
)
)
|> Repo.all()
# Group by rounded HRRR hour to batch requests
by_hour =
@ -48,7 +49,8 @@ defmodule Mix.Tasks.HrrrBackfill do
|> Enum.flat_map(fn c ->
hour = HrrrClient.nearest_hrrr_hour(c.qso_timestamp)
Radio.contact_path_points(c)
c
|> Radio.contact_path_points()
|> Enum.map(fn {lat, lon} ->
{rlat, rlon} = Weather.round_to_hrrr_grid(lat, lon)
{hour, {rlat, rlon}}
@ -129,19 +131,21 @@ defmodule Mix.Tasks.HrrrBackfill do
params = SoundingParams.derive(data.profile)
attrs =
%{
valid_time: valid_time,
lat: lat,
lon: lon,
run_time: data.run_time,
profile: data.profile,
hpbl_m: data.hpbl_m,
pwat_mm: data.pwat_mm,
surface_temp_c: data.surface_temp_c,
surface_dewpoint_c: data.surface_dewpoint_c,
surface_pressure_mb: data.surface_pressure_mb
}
|> maybe_add_derived(params)
maybe_add_derived(
%{
valid_time: valid_time,
lat: lat,
lon: lon,
run_time: data.run_time,
profile: data.profile,
hpbl_m: data.hpbl_m,
pwat_mm: data.pwat_mm,
surface_temp_c: data.surface_temp_c,
surface_dewpoint_c: data.surface_dewpoint_c,
surface_pressure_mb: data.surface_pressure_mb
},
params
)
Weather.upsert_hrrr_profile(attrs)
end

2
mix.exs
View file

@ -70,7 +70,7 @@ defmodule Microwaveprop.MixProject do
{:axon, "~> 0.7", only: [:dev, :test]},
{:exla, "~> 0.9", only: [:dev, :test]},
{:polaris, "~> 0.1", only: [:dev, :test]},
{:tidewave, "~> 0.5", only: :dev},
{:tidewave, "~> 0.5", only: :dev}
]
end

23
priv/repo/migrations/20260402170033_replace_enrichment_booleans_with_enums.exs
View file

@ -46,9 +46,24 @@ defmodule Microwaveprop.Repo.Migrations.ReplaceEnrichmentBooleansWithEnums do
end
# Partial indexes for finding contacts needing enrichment
create index(:qsos, [:hrrr_status], where: "hrrr_status != 'complete'", name: :qsos_hrrr_status_pending_index)
create index(:qsos, [:weather_status], where: "weather_status != 'complete'", name: :qsos_weather_status_pending_index)
create index(:qsos, [:terrain_status], where: "terrain_status != 'complete'", name: :qsos_terrain_status_pending_index)
create index(:qsos, [:iemre_status], where: "iemre_status != 'complete'", name: :qsos_iemre_status_pending_index)
create index(:qsos, [:hrrr_status],
where: "hrrr_status != 'complete'",
name: :qsos_hrrr_status_pending_index
)
create index(:qsos, [:weather_status],
where: "weather_status != 'complete'",
name: :qsos_weather_status_pending_index
)
create index(:qsos, [:terrain_status],
where: "terrain_status != 'complete'",
name: :qsos_terrain_status_pending_index
)
create index(:qsos, [:iemre_status],
where: "iemre_status != 'complete'",
name: :qsos_iemre_status_pending_index
)
end
end

58
test/microwaveprop/radio/csv_import_test.exs
View file

@ -315,12 +315,14 @@ defmodule Microwaveprop.Radio.CsvImportTest do
use ExUnitProperties
property "ISO 8601 format always parses successfully" do
check all year <- integer(2000..2030),
month <- integer(1..12),
day <- integer(1..28),
hour <- integer(0..23),
minute <- integer(0..59),
second <- integer(0..59) do
check all(
year <- integer(2000..2030),
month <- integer(1..12),
day <- integer(1..28),
hour <- integer(0..23),
minute <- integer(0..59),
second <- integer(0..59)
) do
y = String.pad_leading(to_string(year), 4, "0")
m = String.pad_leading(to_string(month), 2, "0")
d = String.pad_leading(to_string(day), 2, "0")
@ -334,12 +336,14 @@ defmodule Microwaveprop.Radio.CsvImportTest do
end
property "US date format with 24h time always parses" do
check all year <- integer(2000..2030),
month <- integer(1..12),
day <- integer(1..28),
hour <- integer(0..23),
minute <- integer(0..59),
sep <- member_of(["/", "-", "."]) do
check all(
year <- integer(2000..2030),
month <- integer(1..12),
day <- integer(1..28),
hour <- integer(0..23),
minute <- integer(0..59),
sep <- member_of(["/", "-", "."])
) do
input = "#{month}#{sep}#{day}#{sep}#{year} #{hour}:#{String.pad_leading(to_string(minute), 2, "0")}"
assert {:ok, result} = CsvImport.normalize_timestamp(input)
assert String.ends_with?(result, "Z")
@ -351,12 +355,14 @@ defmodule Microwaveprop.Radio.CsvImportTest do
end
property "US date with AM/PM parses correctly" do
check all year <- integer(2000..2030),
month <- integer(1..12),
day <- integer(1..28),
hour_12 <- integer(1..12),
minute <- integer(0..59),
ampm <- member_of(["AM", "PM", "am", "pm"]) do
check all(
year <- integer(2000..2030),
month <- integer(1..12),
day <- integer(1..28),
hour_12 <- integer(1..12),
minute <- integer(0..59),
ampm <- member_of(["AM", "PM", "am", "pm"])
) do
input = "#{month}/#{day}/#{year} #{hour_12}:#{String.pad_leading(to_string(minute), 2, "0")} #{ampm}"
assert {:ok, result} = CsvImport.normalize_timestamp(input)
{:ok, dt, _} = DateTime.from_iso8601(result)
@ -383,12 +389,14 @@ defmodule Microwaveprop.Radio.CsvImportTest do
fn y, m, d, h, mi -> "#{m}/#{d}/#{y} #{h}:#{mi}" end
])
check all year <- integer(2000..2030),
month <- integer(1..12),
day <- integer(1..28),
hour <- integer(0..23),
minute <- integer(0..59),
fmt <- formats do
check all(
year <- integer(2000..2030),
month <- integer(1..12),
day <- integer(1..28),
hour <- integer(0..23),
minute <- integer(0..59),
fmt <- formats
) do
y = String.pad_leading(to_string(year), 4, "0")
m = String.pad_leading(to_string(month), 2, "0")
d = String.pad_leading(to_string(day), 2, "0")
@ -409,7 +417,7 @@ defmodule Microwaveprop.Radio.CsvImportTest do
end
property "garbage input never returns ok" do
check all garbage <- string(:alphanumeric, min_length: 1, max_length: 5) do
check all(garbage <- string(:alphanumeric, min_length: 1, max_length: 5)) do
case CsvImport.normalize_timestamp(garbage) do
{:ok, _} -> flunk("Parsed garbage: #{inspect(garbage)}")
{:error, _} -> :ok

4
test/microwaveprop/workers/contact_weather_enqueue_worker_test.exs
View file

@ -240,7 +240,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
# QSO without pos1 should not be marked as queued
all_qsos = Repo.all(Contact)
refute Enum.any?(all_qsos, & &1.weather_status == :queued)
refute Enum.any?(all_qsos, &(&1.weather_status == :queued))
end
test "enqueues HRRR jobs and marks QSOs as hrrr_status" do
@ -425,7 +425,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorkerTest do
assert :ok = ContactWeatherEnqueueWorker.perform(%Oban.Job{args: %{}})
all_qsos = Repo.all(Contact)
refute Enum.any?(all_qsos, & &1.terrain_status == :queued)
refute Enum.any?(all_qsos, &(&1.terrain_status == :queued))
end
end

4
test/mix/tasks/reset_enrichment_test.exs
View file

@ -28,7 +28,9 @@ defmodule Mix.Tasks.ResetEnrichmentTest do
# Set all queued flags to true (not in changeset cast)
Contact
|> where([q], q.id == ^contact.id)
|> Repo.update_all(set: [weather_status: "queued", hrrr_status: "queued", iemre_status: "queued", terrain_status: "queued"])
|> Repo.update_all(
set: [weather_status: "queued", hrrr_status: "queued", iemre_status: "queued", terrain_status: "queued"]
)
Repo.get!(Contact, contact.id)
end