diff --git a/lib/microwaveprop/propagation/scorer.ex b/lib/microwaveprop/propagation/scorer.ex index 547783be..8c60de03 100644 --- a/lib/microwaveprop/propagation/scorer.ex +++ b/lib/microwaveprop/propagation/scorer.ex @@ -387,49 +387,56 @@ defmodule Microwaveprop.Propagation.Scorer do """ @spec path_integrated_conditions([map()], map()) :: map() | nil def path_integrated_conditions(profiles, contact) do - lon = Kernel.||(contact.pos1["lon"] || contact.pos1["lng"], -97.0) - - # Single-pass extraction of all fields to avoid 6 separate Enum traversals - {temps, dewpoints, pressures, gradients, bl_depths, pwats} = - Enum.reduce(profiles, {[], [], [], [], [], []}, fn p, {t, d, pr, g, b, pw} -> - { - if(is_nil(p.surface_temp_c), do: t, else: [p.surface_temp_c | t]), - if(is_nil(p.surface_dewpoint_c), do: d, else: [p.surface_dewpoint_c | d]), - if(is_nil(p.surface_pressure_mb), do: pr, else: [p.surface_pressure_mb | pr]), - if(is_nil(p.min_refractivity_gradient), do: g, else: [p.min_refractivity_gradient | g]), - if(is_nil(p.hpbl_m), do: b, else: [p.hpbl_m | b]), - if(is_nil(p.pwat_mm), do: pw, else: [p.pwat_mm | pw]) - } - end) - - if temps == [] or dewpoints == [] do - nil - else - avg_temp_c = Enum.sum(temps) / length(temps) - avg_dewpoint_c = Enum.sum(dewpoints) / length(dewpoints) - avg_temp_f = c_to_f(avg_temp_c) - avg_dewpoint_f = c_to_f(avg_dewpoint_c) - - %{ - abs_humidity: absolute_humidity(avg_temp_c, avg_dewpoint_c), - temp_f: avg_temp_f, - dewpoint_f: avg_dewpoint_f, - wind_speed_kts: nil, - sky_cover_pct: nil, - utc_hour: contact.qso_timestamp.hour, - utc_minute: contact.qso_timestamp.minute, - month: contact.qso_timestamp.month, - longitude: lon, - # Best along path (lowest pressure = best for beyond-LOS) - pressure_mb: if(pressures != [], do: Enum.min(pressures)), - prev_pressure_mb: nil, - rain_rate_mmhr: 0.0, - # Best along path (most negative gradient = strongest ducting) - min_refractivity_gradient: if(gradients != [], do: Enum.min(gradients)), - # Average along path - bl_depth_m: if(bl_depths != [], do: Enum.sum(bl_depths) / length(bl_depths)), - pwat_mm: if(pwats != [], do: Enum.sum(pwats) / length(pwats)) - } - end + extracted = extract_profile_fields(profiles) + build_path_conditions(extracted, contact) end + + defp extract_profile_fields(profiles) do + Enum.reduce(profiles, {[], [], [], [], [], []}, fn p, {t, d, pr, g, b, pw} -> + { + if(is_nil(p.surface_temp_c), do: t, else: [p.surface_temp_c | t]), + if(is_nil(p.surface_dewpoint_c), do: d, else: [p.surface_dewpoint_c | d]), + if(is_nil(p.surface_pressure_mb), do: pr, else: [p.surface_pressure_mb | pr]), + if(is_nil(p.min_refractivity_gradient), do: g, else: [p.min_refractivity_gradient | g]), + if(is_nil(p.hpbl_m), do: b, else: [p.hpbl_m | b]), + if(is_nil(p.pwat_mm), do: pw, else: [p.pwat_mm | pw]) + } + end) + end + + defp build_path_conditions({temps, _dewpoints, _pressures, _gradients, _bl_depths, _pwats}, _contact) when temps == [], + do: nil + + defp build_path_conditions({_temps, dewpoints, _pressures, _gradients, _bl_depths, _pwats}, _contact) + when dewpoints == [], do: nil + + defp build_path_conditions({temps, dewpoints, pressures, gradients, bl_depths, pwats}, contact) do + lon = Kernel.||(contact.pos1["lon"] || contact.pos1["lng"], -97.0) + avg_temp_c = Enum.sum(temps) / length(temps) + avg_dewpoint_c = Enum.sum(dewpoints) / length(dewpoints) + + %{ + abs_humidity: absolute_humidity(avg_temp_c, avg_dewpoint_c), + temp_f: c_to_f(avg_temp_c), + dewpoint_f: c_to_f(avg_dewpoint_c), + wind_speed_kts: nil, + sky_cover_pct: nil, + utc_hour: contact.qso_timestamp.hour, + utc_minute: contact.qso_timestamp.minute, + month: contact.qso_timestamp.month, + longitude: lon, + pressure_mb: safe_min(pressures), + prev_pressure_mb: nil, + rain_rate_mmhr: 0.0, + min_refractivity_gradient: safe_min(gradients), + bl_depth_m: safe_avg(bl_depths), + pwat_mm: safe_avg(pwats) + } + end + + defp safe_min([]), do: nil + defp safe_min(list), do: Enum.min(list) + + defp safe_avg([]), do: nil + defp safe_avg(list), do: Enum.sum(list) / length(list) end diff --git a/lib/microwaveprop/radio.ex b/lib/microwaveprop/radio.ex index 4bb1025b..e59244f6 100644 --- a/lib/microwaveprop/radio.ex +++ b/lib/microwaveprop/radio.ex @@ -59,25 +59,29 @@ defmodule Microwaveprop.Radio do def group_reciprocals(contacts) do {groups, _seen_ids} = Enum.reduce(contacts, {[], MapSet.new()}, fn contact, {groups, seen} -> - if MapSet.member?(seen, contact.id) do - {groups, seen} - else - reciprocals = - Enum.filter(contacts, fn other -> - other.id != contact.id and - not MapSet.member?(seen, other.id) and - other.band == contact.band and - reciprocal_pair?(contact, other) - end) - - new_seen = Enum.reduce(reciprocals, MapSet.put(seen, contact.id), &MapSet.put(&2, &1.id)) - {[{contact, reciprocals} | groups], new_seen} - end + group_contact(contact, contacts, groups, seen) end) Enum.reverse(groups) end + defp group_contact(contact, contacts, groups, seen) do + if MapSet.member?(seen, contact.id) do + {groups, seen} + else + reciprocals = + Enum.filter(contacts, fn other -> + other.id != contact.id and + not MapSet.member?(seen, other.id) and + other.band == contact.band and + reciprocal_pair?(contact, other) + end) + + new_seen = Enum.reduce(reciprocals, MapSet.put(seen, contact.id), &MapSet.put(&2, &1.id)) + {[{contact, reciprocals} | groups], new_seen} + end + end + defp reciprocal_pair?(a, b) do same_stations = (a.station1 == b.station2 and a.station2 == b.station1) or @@ -236,23 +240,27 @@ defmodule Microwaveprop.Radio do {contact.id, dist} end - if updates != [] do - # Build a single UPDATE ... FROM (VALUES ...) statement - # Ecto.UUID.dump!/1 converts string UUIDs to the 16-byte binary Postgrex expects - {values_sql, params} = - updates - |> Enum.with_index() - |> Enum.reduce({"", []}, fn {{id, dist}, i}, {sql, params} -> - frag = "($#{i * 2 + 1}::uuid, $#{i * 2 + 2}::numeric)" - sep = if sql == "", do: "", else: ", " - {sql <> sep <> frag, params ++ [Ecto.UUID.dump!(id), dist]} - end) + execute_distance_updates(updates) + end - Repo.query!( - "UPDATE contacts AS c SET distance_km = v.dist FROM (VALUES #{values_sql}) AS v(id, dist) WHERE c.id = v.id", - params - ) - end + defp execute_distance_updates([]), do: nil + + defp execute_distance_updates(updates) do + # Build a single UPDATE ... FROM (VALUES ...) statement + # Ecto.UUID.dump!/1 converts string UUIDs to the 16-byte binary Postgrex expects + {values_sql, params} = + updates + |> Enum.with_index() + |> Enum.reduce({"", []}, fn {{id, dist}, i}, {sql, params} -> + frag = "($#{i * 2 + 1}::uuid, $#{i * 2 + 2}::numeric)" + sep = if sql == "", do: "", else: ", " + {sql <> sep <> frag, params ++ [Ecto.UUID.dump!(id), dist]} + end) + + Repo.query!( + "UPDATE contacts AS c SET distance_km = v.dist FROM (VALUES #{values_sql}) AS v(id, dist) WHERE c.id = v.id", + params + ) end defp deg_to_rad(deg), do: deg * :math.pi() / 180 @@ -284,61 +292,74 @@ defmodule Microwaveprop.Radio do needs_pos2 = is_nil(contact.pos2) && contact.grid2 if needs_pos1 || needs_pos2 do - pos1 = contact.pos1 || latlon_from_grid(contact.grid1) - pos2 = contact.pos2 || latlon_from_grid(contact.grid2) - - distance = - if pos1 && pos2 do - lon1 = pos1["lon"] || pos1["lng"] - lon2 = pos2["lon"] || pos2["lng"] - - pos1["lat"] - |> haversine_km(lon1, pos2["lat"], lon2) - |> round() - |> Decimal.new() - else - contact.distance_km - end - - changes = - %{} - |> then(fn c -> if pos1 && is_nil(contact.pos1), do: Map.put(c, :pos1, pos1), else: c end) - |> then(fn c -> if pos2 && is_nil(contact.pos2), do: Map.put(c, :pos2, pos2), else: c end) - |> then(fn c -> if distance && is_nil(contact.distance_km), do: Map.put(c, :distance_km, distance), else: c end) - - if changes == %{} do - contact - else - # Reset unavailable statuses to pending now that positions exist - new_pos1 = Map.get(changes, :pos1) - new_pos2 = Map.get(changes, :pos2) - - changes = - if new_pos1 do - changes - |> maybe_reset_status(:hrrr_status, contact.hrrr_status) - |> maybe_reset_status(:weather_status, contact.weather_status) - |> maybe_reset_status(:iemre_status, contact.iemre_status) - else - changes - end - - changes = - if new_pos1 && (new_pos2 || contact.pos2) do - maybe_reset_status(changes, :terrain_status, contact.terrain_status) - else - changes - end - - contact - |> Ecto.Changeset.change(changes) - |> Repo.update!() - end + do_ensure_positions(contact) else contact end end + defp do_ensure_positions(contact) do + pos1 = contact.pos1 || latlon_from_grid(contact.grid1) + pos2 = contact.pos2 || latlon_from_grid(contact.grid2) + distance = compute_distance(pos1, pos2, contact.distance_km) + + changes = build_position_changes(contact, pos1, pos2, distance) + + apply_position_changes(contact, changes) + end + + defp compute_distance(pos1, pos2, _existing_distance) when not is_nil(pos1) and not is_nil(pos2) do + lon1 = pos1["lon"] || pos1["lng"] + lon2 = pos2["lon"] || pos2["lng"] + + pos1["lat"] + |> haversine_km(lon1, pos2["lat"], lon2) + |> round() + |> Decimal.new() + end + + defp compute_distance(_pos1, _pos2, existing_distance), do: existing_distance + + defp build_position_changes(contact, pos1, pos2, distance) do + %{} + |> then(fn c -> if pos1 && is_nil(contact.pos1), do: Map.put(c, :pos1, pos1), else: c end) + |> then(fn c -> if pos2 && is_nil(contact.pos2), do: Map.put(c, :pos2, pos2), else: c end) + |> then(fn c -> + if distance && is_nil(contact.distance_km), do: Map.put(c, :distance_km, distance), else: c + end) + end + + defp apply_position_changes(contact, changes) when changes == %{}, do: contact + + defp apply_position_changes(contact, changes) do + changes = reset_enrichment_statuses(contact, changes) + + contact + |> Ecto.Changeset.change(changes) + |> Repo.update!() + end + + defp reset_enrichment_statuses(contact, changes) do + new_pos1 = Map.get(changes, :pos1) + new_pos2 = Map.get(changes, :pos2) + + changes = + if new_pos1 do + changes + |> maybe_reset_status(:hrrr_status, contact.hrrr_status) + |> maybe_reset_status(:weather_status, contact.weather_status) + |> maybe_reset_status(:iemre_status, contact.iemre_status) + else + changes + end + + if new_pos1 && (new_pos2 || contact.pos2) do + maybe_reset_status(changes, :terrain_status, contact.terrain_status) + else + changes + end + end + defp maybe_reset_status(changes, field, :unavailable), do: Map.put(changes, field, :pending) defp maybe_reset_status(changes, _field, _status), do: changes @@ -368,33 +389,41 @@ defmodule Microwaveprop.Radio do changeset = Contact.submission_changeset(%Contact{}, attrs) if changeset.valid? do - grid1 = Ecto.Changeset.get_change(changeset, :grid1) - grid2 = Ecto.Changeset.get_change(changeset, :grid2) - - if existing = find_duplicate_contact(changeset) do - {:error, :duplicate, existing} - else - case {Maidenhead.to_latlon(grid1), Maidenhead.to_latlon(grid2)} do - {{:ok, {lat1, lon1}}, {:ok, {lat2, lon2}}} -> - distance = lat1 |> haversine_km(lon1, lat2, lon2) |> round() |> Decimal.new() - - changeset - |> Ecto.Changeset.put_change(:pos1, %{"lat" => lat1, "lon" => lon1}) - |> Ecto.Changeset.put_change(:pos2, %{"lat" => lat2, "lon" => lon2}) - |> Ecto.Changeset.put_change(:distance_km, distance) - |> Ecto.Changeset.put_change(:user_submitted, true) - |> Repo.insert() - - _ -> - changeset = Ecto.Changeset.add_error(changeset, :grid1, "could not resolve grid to coordinates") - {:error, %{changeset | action: :insert}} - end - end + insert_validated_contact(changeset) else {:error, %{changeset | action: :insert}} end end + defp insert_validated_contact(changeset) do + if existing = find_duplicate_contact(changeset) do + {:error, :duplicate, existing} + else + resolve_grids_and_insert(changeset) + end + end + + defp resolve_grids_and_insert(changeset) do + grid1 = Ecto.Changeset.get_change(changeset, :grid1) + grid2 = Ecto.Changeset.get_change(changeset, :grid2) + + case {Maidenhead.to_latlon(grid1), Maidenhead.to_latlon(grid2)} do + {{:ok, {lat1, lon1}}, {:ok, {lat2, lon2}}} -> + distance = lat1 |> haversine_km(lon1, lat2, lon2) |> round() |> Decimal.new() + + changeset + |> Ecto.Changeset.put_change(:pos1, %{"lat" => lat1, "lon" => lon1}) + |> Ecto.Changeset.put_change(:pos2, %{"lat" => lat2, "lon" => lon2}) + |> Ecto.Changeset.put_change(:distance_km, distance) + |> Ecto.Changeset.put_change(:user_submitted, true) + |> Repo.insert() + + _ -> + changeset = Ecto.Changeset.add_error(changeset, :grid1, "could not resolve grid to coordinates") + {:error, %{changeset | action: :insert}} + end + end + defp find_duplicate_contact(changeset) do s1 = changeset |> Ecto.Changeset.get_field(:station1) |> to_string() |> String.upcase() s2 = changeset |> Ecto.Changeset.get_field(:station2) |> to_string() |> String.upcase() diff --git a/lib/microwaveprop/radio/csv_import.ex b/lib/microwaveprop/radio/csv_import.ex index 2b01b900..eef52187 100644 --- a/lib/microwaveprop/radio/csv_import.ex +++ b/lib/microwaveprop/radio/csv_import.ex @@ -163,26 +163,34 @@ defmodule Microwaveprop.Radio.CsvImport do |> Map.new() |> Map.put("submitter_email", submitter_email) - case normalize_timestamp(attrs["qso_timestamp"]) do - {:ok, iso_timestamp} -> - attrs = Map.put(attrs, "qso_timestamp", iso_timestamp) - changeset = Contact.submission_changeset(%Contact{}, attrs) - - if changeset.valid? do - {:ok, datetime} = parse_iso_datetime(iso_timestamp) - {:parsed, %{row_num: row_num, attrs: attrs, timestamp: datetime}} - else - {:invalid, %{row_num: row_num, messages: changeset_error_strings(changeset)}} - end - - {:error, msg} -> - {:invalid, %{row_num: row_num, messages: [msg]}} - end + parse_row_with_timestamp(attrs, row_num) else {:invalid, %{row_num: row_num, messages: ["expected #{@expected_columns} columns, got #{length(fields)}"]}} end end + defp parse_row_with_timestamp(attrs, row_num) do + case normalize_timestamp(attrs["qso_timestamp"]) do + {:ok, iso_timestamp} -> + validate_parsed_row(attrs, row_num, iso_timestamp) + + {:error, msg} -> + {:invalid, %{row_num: row_num, messages: [msg]}} + end + end + + defp validate_parsed_row(attrs, row_num, iso_timestamp) do + attrs = Map.put(attrs, "qso_timestamp", iso_timestamp) + changeset = Contact.submission_changeset(%Contact{}, attrs) + + if changeset.valid? do + {:ok, datetime} = parse_iso_datetime(iso_timestamp) + {:parsed, %{row_num: row_num, attrs: attrs, timestamp: datetime}} + else + {:invalid, %{row_num: row_num, messages: changeset_error_strings(changeset)}} + end + end + defp split_parsed(rows) do rows |> Enum.reduce({[], []}, fn diff --git a/lib/microwaveprop/weather.ex b/lib/microwaveprop/weather.ex index c8c5c4f3..b5d16f52 100644 --- a/lib/microwaveprop/weather.ex +++ b/lib/microwaveprop/weather.ex @@ -21,7 +21,7 @@ defmodule Microwaveprop.Weather do # Approximate km per degree latitude @km_per_deg_lat 111.0 - @spec find_or_create_station(map()) :: {:ok, %Station{}} | {:error, Ecto.Changeset.t()} + @spec find_or_create_station(map()) :: {:ok, Station.t()} | {:error, Ecto.Changeset.t()} def find_or_create_station(attrs) do code = attrs[:station_code] || attrs["station_code"] type = attrs[:station_type] || attrs["station_type"] @@ -43,7 +43,7 @@ defmodule Microwaveprop.Weather do end end - @spec upsert_surface_observation(%Station{}, map()) :: {:ok, %SurfaceObservation{}} | {:error, Ecto.Changeset.t()} + @spec upsert_surface_observation(Station.t(), map()) :: {:ok, SurfaceObservation.t()} | {:error, Ecto.Changeset.t()} def upsert_surface_observation(%Station{} = station, attrs) do attrs = Map.put(attrs, :station_id, station.id) @@ -78,7 +78,7 @@ defmodule Microwaveprop.Weather do ) end - @spec upsert_sounding(%Station{}, map()) :: {:ok, %Sounding{}} | {:error, Ecto.Changeset.t()} + @spec upsert_sounding(Station.t(), map()) :: {:ok, Sounding.t()} | {:error, Ecto.Changeset.t()} def upsert_sounding(%Station{} = station, attrs) do attrs = Map.put(attrs, :station_id, station.id) @@ -116,7 +116,7 @@ defmodule Microwaveprop.Weather do ) end - @spec upsert_solar_index(map()) :: {:ok, %SolarIndex{}} | {:error, Ecto.Changeset.t()} + @spec upsert_solar_index(map()) :: {:ok, SolarIndex.t()} | {:error, Ecto.Changeset.t()} def upsert_solar_index(attrs) do %SolarIndex{} |> SolarIndex.changeset(attrs) @@ -229,7 +229,7 @@ defmodule Microwaveprop.Weather do end) end - @spec get_solar_index(Date.t()) :: %SolarIndex{} | nil + @spec get_solar_index(Date.t()) :: SolarIndex.t() | nil def get_solar_index(date) do Repo.get_by(SolarIndex, date: date) end @@ -274,7 +274,7 @@ defmodule Microwaveprop.Weather do :ok end - @spec nearby_stations(float(), float(), String.t(), number()) :: [%Station{}] + @spec nearby_stations(float(), float(), String.t(), number()) :: [Station.t()] def nearby_stations(lat, lon, station_type, radius_km) do dlat = radius_km / @km_per_deg_lat dlon = radius_km / (@km_per_deg_lat * :math.cos(lat * :math.pi() / 180)) @@ -310,8 +310,8 @@ defmodule Microwaveprop.Weather do end @spec weather_for_contact(map(), keyword()) :: %{ - surface_observations: [%SurfaceObservation{}], - soundings: [%Sounding{}] + surface_observations: [SurfaceObservation.t()], + soundings: [Sounding.t()] } def weather_for_contact(contact_params, opts \\ []) do lat = contact_params[:lat] || contact_params.lat @@ -441,7 +441,7 @@ defmodule Microwaveprop.Weather do |> Map.drop([:profile, :duct_characteristics, :surface_temp_c, :surface_dewpoint_c]) end - @spec upsert_hrrr_profile(map()) :: {:ok, %HrrrProfile{}} | {:error, Ecto.Changeset.t()} + @spec upsert_hrrr_profile(map()) :: {:ok, HrrrProfile.t()} | {:error, Ecto.Changeset.t()} def upsert_hrrr_profile(attrs) do %HrrrProfile{} |> HrrrProfile.changeset(attrs) @@ -497,7 +497,7 @@ defmodule Microwaveprop.Weather do |> Repo.exists?() end - @spec hrrr_for_contact(map()) :: %HrrrProfile{} | nil + @spec hrrr_for_contact(map()) :: HrrrProfile.t() | nil def hrrr_for_contact(%{pos1: nil}), do: nil def hrrr_for_contact(contact) do @@ -509,7 +509,7 @@ defmodule Microwaveprop.Weather do end end - @spec find_nearest_hrrr(float(), float(), DateTime.t()) :: %HrrrProfile{} | nil + @spec find_nearest_hrrr(float(), float(), DateTime.t()) :: HrrrProfile.t() | nil def find_nearest_hrrr(lat, lon, timestamp) do dlat = 0.07 dlon = 0.07 @@ -539,7 +539,7 @@ defmodule Microwaveprop.Weather do |> Repo.one() end - @spec hrrr_profiles_for_path(map()) :: [%HrrrProfile{}] + @spec hrrr_profiles_for_path(map()) :: [HrrrProfile.t()] def hrrr_profiles_for_path(%{pos1: nil}), do: [] def hrrr_profiles_for_path(contact) do @@ -554,13 +554,13 @@ defmodule Microwaveprop.Weather do Tries HRRR first (3 km, hourly), falls back to ERA5 (0.25°, hourly). Returns the profile struct or nil. """ - @spec best_profile_for_contact(map()) :: %HrrrProfile{} | %Era5Profile{} | nil + @spec best_profile_for_contact(map()) :: HrrrProfile.t() | Era5Profile.t() | nil def best_profile_for_contact(contact) do hrrr_for_contact(contact) || era5_for_contact(contact) end @doc "Find all atmospheric profiles along a contact's path, from any source." - @spec profiles_along_path(map()) :: [%HrrrProfile{} | %Era5Profile{}] + @spec profiles_along_path(map()) :: [HrrrProfile.t() | Era5Profile.t()] def profiles_along_path(contact) do hrrr_path = hrrr_profiles_for_path(contact) @@ -571,7 +571,7 @@ defmodule Microwaveprop.Weather do end end - @spec era5_for_contact(map()) :: %Era5Profile{} | nil + @spec era5_for_contact(map()) :: Era5Profile.t() | nil def era5_for_contact(%{pos1: nil}), do: nil def era5_for_contact(contact) do @@ -583,7 +583,7 @@ defmodule Microwaveprop.Weather do end end - @spec era5_profiles_for_path(map()) :: [%Era5Profile{}] + @spec era5_profiles_for_path(map()) :: [Era5Profile.t()] def era5_profiles_for_path(%{pos1: nil}), do: [] def era5_profiles_for_path(contact) do @@ -593,7 +593,7 @@ defmodule Microwaveprop.Weather do |> Enum.reject(&is_nil/1) end - @spec find_nearest_era5(float(), float(), DateTime.t()) :: %Era5Profile{} | nil + @spec find_nearest_era5(float(), float(), DateTime.t()) :: Era5Profile.t() | nil def find_nearest_era5(lat, lon, timestamp) do dlat = 0.15 dlon = 0.15 @@ -621,7 +621,7 @@ defmodule Microwaveprop.Weather do |> Repo.one() end - @spec find_nearest_rtma(float(), float(), DateTime.t()) :: %RtmaObservation{} | nil + @spec find_nearest_rtma(float(), float(), DateTime.t()) :: RtmaObservation.t() | nil def find_nearest_rtma(lat, lon, timestamp) do dlat = 0.05 dlon = 0.05 @@ -695,7 +695,7 @@ defmodule Microwaveprop.Weather do {Float.round(lat * 8) / 8, Float.round(lon * 8) / 8} end - @spec upsert_iemre_observation(map()) :: {:ok, %IemreObservation{}} | {:error, Ecto.Changeset.t()} + @spec upsert_iemre_observation(map()) :: {:ok, IemreObservation.t()} | {:error, Ecto.Changeset.t()} def upsert_iemre_observation(attrs) do %IemreObservation{} |> IemreObservation.changeset(attrs) @@ -712,7 +712,7 @@ defmodule Microwaveprop.Weather do |> Repo.exists?() end - @spec iemre_for_contact(map()) :: %IemreObservation{} | nil + @spec iemre_for_contact(map()) :: IemreObservation.t() | nil def iemre_for_contact(%{pos1: nil}), do: nil def iemre_for_contact(contact) do @@ -724,7 +724,7 @@ defmodule Microwaveprop.Weather do end end - @spec find_nearest_iemre(float(), float(), DateTime.t()) :: %IemreObservation{} | nil + @spec find_nearest_iemre(float(), float(), DateTime.t()) :: IemreObservation.t() | nil def find_nearest_iemre(lat, lon, timestamp) do {rlat, rlon} = round_to_iemre_grid(lat, lon) date = DateTime.to_date(timestamp) @@ -734,7 +734,7 @@ defmodule Microwaveprop.Weather do |> Repo.one() end - @spec iemre_for_path(map()) :: [%IemreObservation{}] + @spec iemre_for_path(map()) :: [IemreObservation.t()] def iemre_for_path(%{pos1: nil}), do: [] def iemre_for_path(contact) do @@ -753,7 +753,7 @@ defmodule Microwaveprop.Weather do `:observed_at`, etc. — the same shape regardless of which table the data came from. Returns `nil` if neither source has data. """ - @spec recent_surface_obs(float(), float(), DateTime.t()) :: %Metar5minObservation{} | %SurfaceObservation{} | nil + @spec recent_surface_obs(float(), float(), DateTime.t()) :: Metar5minObservation.t() | SurfaceObservation.t() | nil def recent_surface_obs(lat, lon, timestamp) do dlat = 0.5 dlon = 0.5 diff --git a/lib/microwaveprop/weather/grib2/extractor.ex b/lib/microwaveprop/weather/grib2/extractor.ex index bd0a265a..17b30e4d 100644 --- a/lib/microwaveprop/weather/grib2/extractor.ex +++ b/lib/microwaveprop/weather/grib2/extractor.ex @@ -56,32 +56,30 @@ defmodule Microwaveprop.Weather.Grib2.Extractor do if grid_indices == [] do {:ok, %{}} else - result = - Enum.reduce(messages, {:ok, init_grid(points)}, fn msg, {:ok, acc} -> - case extract_grid_with_indices(msg, grid_indices) do - {:ok, key, values} -> - merged = - Enum.reduce(values, acc, fn {point, value}, grid -> - Map.update!(grid, point, &Map.put(&1, key, value)) - end) - - {:ok, merged} - - {:error, _reason} -> - {:ok, acc} - end - end) - - case result do - {:ok, grid} -> - {:ok, Map.reject(grid, fn {_point, values} -> values == %{} end)} - - error -> - error - end + messages + |> reduce_messages(grid_indices, init_grid(points)) + |> filter_empty_grid() end end + defp reduce_messages(messages, grid_indices, initial_grid) do + Enum.reduce(messages, {:ok, initial_grid}, fn msg, {:ok, acc} -> + case extract_grid_with_indices(msg, grid_indices) do + {:ok, key, values} -> + {:ok, merge_grid_values(values, acc, key)} + + {:error, _reason} -> + {:ok, acc} + end + end) + end + + defp filter_empty_grid({:ok, grid}) do + {:ok, Map.reject(grid, fn {_point, values} -> values == %{} end)} + end + + defp filter_empty_grid(error), do: error + @doc """ Split a binary blob into individual GRIB2 messages by scanning for "GRIB" magic bytes and reading the total length from the indicator section. @@ -147,15 +145,7 @@ defmodule Microwaveprop.Weather.Grib2.Extractor do case batch_unpack(packing, data, unique_indices) do {:ok, index_values} -> - values = - Enum.flat_map(grid_indices, fn {point, index} -> - case Map.get(index_values, index) do - nil -> [] - value -> [{point, value}] - end - end) - - {:ok, key, values} + {:ok, key, resolve_grid_values(grid_indices, index_values)} {:error, reason} -> {:error, reason} @@ -165,6 +155,21 @@ defmodule Microwaveprop.Weather.Grib2.Extractor do e -> {:error, "GRIB2 grid extraction failed: #{inspect(e)}"} end + defp merge_grid_values(values, grid, key) do + Enum.reduce(values, grid, fn {point, value}, acc -> + Map.update!(acc, point, &Map.put(&1, key, value)) + end) + end + + defp resolve_grid_values(grid_indices, index_values) do + Enum.flat_map(grid_indices, fn {point, index} -> + case Map.get(index_values, index) do + nil -> [] + value -> [{point, value}] + end + end) + end + defp batch_unpack(%{template: 3} = params, data, indices) do ComplexPacking.extract_values(params, data, indices) end diff --git a/lib/microwaveprop/weather/grib2/wgrib2.ex b/lib/microwaveprop/weather/grib2/wgrib2.ex index 572d8c12..6bba3d6e 100644 --- a/lib/microwaveprop/weather/grib2/wgrib2.ex +++ b/lib/microwaveprop/weather/grib2/wgrib2.ex @@ -170,8 +170,14 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do # each cell extracts its values from every message, passes the per-cell # map to the reducer, and keeps only the reduced output. defp parse_lola_binary_mapped(bin_data, messages, grid_spec, cell_reducer) do - %{lon_count: nx, lat_count: ny, lon_start: lon_start, lon_step: lon_step, lat_start: lat_start, lat_step: lat_step} = - grid_spec + %{ + lon_count: nx, + lat_count: ny, + lon_start: lon_start, + lon_step: lon_step, + lat_start: lat_start, + lat_step: lat_step + } = grid_spec points_per_message = nx * ny bytes_per_message = points_per_message * 4 @@ -194,33 +200,35 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do Enum.reduce(0..(nx - 1), acc_outer, fn i, acc_inner -> cell_offset = (j * nx + i) * 4 - - # Extract all values for this cell across all messages - cell_data = - Enum.reduce(msg_meta, %{}, fn {key, data_offset}, cell_acc -> - offset = data_offset + cell_offset - <<_::binary-size(offset), value::float-little-32, _::binary>> = bin_data - - if value > @undefined_value / 2 do - cell_acc - else - Map.put(cell_acc, key, value) - end - end) - - if map_size(cell_data) > 0 do - lon = Float.round(denormalize_lon(lon_start + i * lon_step), 3) - reduced = cell_reducer.(cell_data) - Map.put(acc_inner, {lat, lon}, reduced) - else - acc_inner - end + cell_data = extract_cell_values(bin_data, msg_meta, cell_offset) + reduce_mapped_cell(cell_data, lat, lon_start + i * lon_step, cell_reducer, acc_inner) end) end) {:ok, result} end + defp reduce_mapped_cell(cell_data, _lat, _raw_lon, _cell_reducer, acc) when cell_data == %{}, do: acc + + defp reduce_mapped_cell(cell_data, lat, raw_lon, cell_reducer, acc) do + lon = Float.round(denormalize_lon(raw_lon), 3) + reduced = cell_reducer.(cell_data) + Map.put(acc, {lat, lon}, reduced) + end + + defp extract_cell_values(bin_data, msg_meta, cell_offset) do + Enum.reduce(msg_meta, %{}, fn {key, data_offset}, cell_acc -> + offset = data_offset + cell_offset + <<_::binary-size(offset), value::float-little-32, _::binary>> = bin_data + + if value > @undefined_value / 2 do + cell_acc + else + Map.put(cell_acc, key, value) + end + end) + end + defp extract_with_wgrib2(grib_binary, match_pattern, grid_spec) do %{ lon_start: lon_start, @@ -431,22 +439,25 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do lat = Float.round(lat_start + j * lat_step, 3) Enum.reduce(0..(nx - 1), outer, fn i, inner -> - offset = (j * nx + i) * 4 - <<_::binary-size(offset), value::float-little-32, _::binary>> = chunk - - if value > @undefined_value / 2 do - inner - else - lon = Float.round(denormalize_lon(lon_start + i * lon_step), 3) - Map.put(inner, {lat, lon}, value) - end + extract_grid_point(chunk, nx, j, i, lat, lon_start, lon_step, inner) end) end) end + defp extract_grid_point(chunk, nx, j, i, lat, lon_start, lon_step, acc) do + offset = (j * nx + i) * 4 + <<_::binary-size(offset), value::float-little-32, _::binary>> = chunk + + if value > @undefined_value / 2 do + acc + else + lon = Float.round(denormalize_lon(lon_start + i * lon_step), 3) + Map.put(acc, {lat, lon}, value) + end + end + defp parse_lola_binary(bin_data, messages, grid_spec) do - %{lon_count: nx, lat_count: ny, lon_start: lon_start, lon_step: lon_step, lat_start: lat_start, lat_step: lat_step} = - grid_spec + %{lon_count: nx, lat_count: ny} = grid_spec points_per_message = nx * ny bytes_per_message = points_per_message * 4 @@ -468,7 +479,7 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do if data_offset + bytes_per_message <= byte_size(bin_data) do chunk = binary_part(bin_data, data_offset, bytes_per_message) - merge_message_values(acc, key, chunk, nx, ny, lon_start, lon_step, lat_start, lat_step) + merge_message_values(acc, key, chunk, grid_spec) else acc end @@ -477,7 +488,16 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do {:ok, result} end - defp merge_message_values(acc, key, chunk, nx, ny, lon_start, lon_step, lat_start, lat_step) do + defp merge_message_values(acc, key, chunk, grid_spec) do + %{ + lon_count: nx, + lat_count: ny, + lon_start: lon_start, + lon_step: lon_step, + lat_start: lat_start, + lat_step: lat_step + } = grid_spec + # Binary is row-major: lat varies slowest, lon varies fastest # Each value is a 32-bit IEEE 754 little-endian float Enum.reduce(0..(ny - 1), acc, fn j, acc_outer -> @@ -486,20 +506,22 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do Enum.reduce(0..(nx - 1), acc_outer, fn i, acc_inner -> offset = (j * nx + i) * 4 <<_::binary-size(offset), value::float-little-32, _::binary>> = chunk - - if value > @undefined_value / 2 do - # Skip undefined values (ocean/outside-domain points) - acc_inner - else - lon = Float.round(denormalize_lon(lon_start + i * lon_step), 3) - point = {lat, lon} - existing = Map.get(acc_inner, point, %{}) - Map.put(acc_inner, point, Map.put(existing, key, value)) - end + merge_keyed_grid_value(value, lat, lon_start + i * lon_step, acc_inner, key) end) end) end + defp merge_keyed_grid_value(value, _lat, _raw_lon, acc, _key) when value > @undefined_value / 2 do + acc + end + + defp merge_keyed_grid_value(value, lat, raw_lon, acc, key) do + lon = Float.round(denormalize_lon(raw_lon), 3) + point = {lat, lon} + existing = Map.get(acc, point, %{}) + Map.put(acc, point, Map.put(existing, key, value)) + end + @doc """ Extract values at specific `{lat, lon}` points from a GRIB2 file on disk using wgrib2 `-lon`. One file scan, text output, no binary grid — uses @@ -543,37 +565,43 @@ defmodule Microwaveprop.Weather.Grib2.Wgrib2 do |> String.split("\n") |> Enum.reject(&(&1 == "")) |> Enum.reduce(%{}, fn line, acc -> - parts = String.split(line, ":") - - # Extract var and level from the inventory portion - case extract_var_level(parts) do - [var, level] -> - key = "#{var}:#{level}" - - # Extract lon/lat/val segments (everything after the inventory) - Enum.reduce(parts, acc, fn segment, inner_acc -> - case parse_lon_val_segment(segment) do - {lat, lon, val} -> - point = snap_to_nearest(lat, lon, points) - - if point do - existing = Map.get(inner_acc, point, %{}) - Map.put(inner_acc, point, Map.put(existing, key, val)) - else - inner_acc - end - - nil -> - inner_acc - end - end) - - _ -> - acc - end + parse_lon_line(line, acc, points) end) end + defp parse_lon_line(line, acc, points) do + parts = String.split(line, ":") + + case extract_var_level(parts) do + [var, level] -> + key = "#{var}:#{level}" + + Enum.reduce(parts, acc, fn segment, inner_acc -> + merge_lon_val_segment(segment, inner_acc, key, points) + end) + + _ -> + acc + end + end + + defp merge_lon_val_segment(segment, acc, key, points) do + case parse_lon_val_segment(segment) do + {lat, lon, val} -> + point = snap_to_nearest(lat, lon, points) + + if point do + existing = Map.get(acc, point, %{}) + Map.put(acc, point, Map.put(existing, key, val)) + else + acc + end + + nil -> + acc + end + end + defp extract_var_level(parts) when length(parts) >= 5 do [Enum.at(parts, 3), Enum.at(parts, 4)] end diff --git a/lib/microwaveprop/weather/hrrr_client.ex b/lib/microwaveprop/weather/hrrr_client.ex index d5463dd6..cbe55e22 100644 --- a/lib/microwaveprop/weather/hrrr_client.ex +++ b/lib/microwaveprop/weather/hrrr_client.ex @@ -159,21 +159,25 @@ defmodule Microwaveprop.Weather.HrrrClient do idx_entries |> Enum.with_index() |> Enum.flat_map(fn {entry, idx} -> - if entry.var == w.var && entry.level == w.level do - end_offset = - case Enum.at(idx_entries, idx + 1) do - nil -> entry.offset + 10_000_000 - next -> next.offset - 1 - end - - [{entry.offset, end_offset}] - else - [] - end + byte_range_for_entry(entry, idx, idx_entries, w) end) end) end + defp byte_range_for_entry(entry, idx, idx_entries, wanted) do + if entry.var == wanted.var && entry.level == wanted.level do + end_offset = + case Enum.at(idx_entries, idx + 1) do + nil -> entry.offset + 10_000_000 + next -> next.offset - 1 + end + + [{entry.offset, end_offset}] + else + [] + end + end + @spec merge_grid_data(map(), map()) :: map() def merge_grid_data(sfc_grid, prs_grid) do Map.merge(sfc_grid, prs_grid, fn _point, sfc_data, prs_data -> diff --git a/lib/microwaveprop/weather/hrrr_native_client.ex b/lib/microwaveprop/weather/hrrr_native_client.ex index b3efd94a..1d64b770 100644 --- a/lib/microwaveprop/weather/hrrr_native_client.ex +++ b/lib/microwaveprop/weather/hrrr_native_client.ex @@ -275,14 +275,7 @@ defmodule Microwaveprop.Weather.HrrrNativeClient do # -lon extracts only at the requested points with text output. case Wgrib2.extract_points_from_file(grib_path, match_pattern, points) do {:ok, point_data} -> - result = - Map.new(points, fn point -> - parsed = Map.get(point_data, point, %{}) - profile = if map_size(parsed) > 0, do: build_native_profile(parsed), else: %{level_count: 0} - {point, profile} - end) - - {:ok, result} + {:ok, build_profiles_from_point_data(points, point_data)} error -> error @@ -311,36 +304,52 @@ defmodule Microwaveprop.Weather.HrrrNativeClient do match_pattern = ":(#{Enum.join(@native_variables, "|")}):.*hybrid level:" if Wgrib2.available?() do - grid_spec = bounding_grid(points) - - case Wgrib2.extract_grid(grib_binary, match_pattern, grid_spec) do - {:ok, grid_data} -> - # Nearest-neighbor lookup: for each requested point, find - # the grid cell with the smallest (lat, lon) distance. - result = - Map.new(points, fn {lat, lon} -> - nearest = nearest_grid_cell(grid_data, lat, lon) - profile = if nearest, do: build_native_profile(nearest), else: %{level_count: 0} - {{lat, lon}, profile} - end) - - {:ok, result} - - error -> - error - end + extract_native_profiles_wgrib2(grib_binary, match_pattern, points) else - # Fallback: pure Elixir (slow) - alias Microwaveprop.Weather.Grib2.Extractor + extract_native_profiles_elixir(grib_binary, points) + end + end - case Extractor.extract_grid(grib_binary, points) do - {:ok, grid_data} -> - result = Map.new(grid_data, fn {pt, parsed} -> {pt, build_native_profile(parsed)} end) - {:ok, result} + defp build_profiles_from_point_data(points, point_data) do + Map.new(points, fn point -> + parsed = Map.get(point_data, point, %{}) + profile = if map_size(parsed) > 0, do: build_native_profile(parsed), else: %{level_count: 0} + {point, profile} + end) + end - error -> - error - end + defp extract_native_profiles_wgrib2(grib_binary, match_pattern, points) do + alias Microwaveprop.Weather.Grib2.Wgrib2 + + grid_spec = bounding_grid(points) + + case Wgrib2.extract_grid(grib_binary, match_pattern, grid_spec) do + {:ok, grid_data} -> + {:ok, build_nearest_profiles(points, grid_data)} + + error -> + error + end + end + + defp build_nearest_profiles(points, grid_data) do + Map.new(points, fn {lat, lon} -> + nearest = nearest_grid_cell(grid_data, lat, lon) + profile = if nearest, do: build_native_profile(nearest), else: %{level_count: 0} + {{lat, lon}, profile} + end) + end + + defp extract_native_profiles_elixir(grib_binary, points) do + alias Microwaveprop.Weather.Grib2.Extractor + + case Extractor.extract_grid(grib_binary, points) do + {:ok, grid_data} -> + result = Map.new(grid_data, fn {pt, parsed} -> {pt, build_native_profile(parsed)} end) + {:ok, result} + + error -> + error end end diff --git a/lib/microwaveprop/weather/sounding_params.ex b/lib/microwaveprop/weather/sounding_params.ex index f6242427..3f7e74b5 100644 --- a/lib/microwaveprop/weather/sounding_params.ex +++ b/lib/microwaveprop/weather/sounding_params.ex @@ -115,35 +115,13 @@ defmodule Microwaveprop.Weather.SoundingParams do sorted |> Enum.chunk_every(2, 1, :discard) |> Enum.flat_map(fn [lower, upper] -> - if upper["tmpc"] > lower["tmpc"] do - base = lower["hght"] - sfc_hght - top = upper["hght"] - sfc_hght - strength = upper["tmpc"] - lower["tmpc"] - dh_100m = (top - base) / 100.0 - rate = if dh_100m > 0, do: strength / dh_100m, else: 0.0 - [%{base: base, top: top, strength: strength, rate: rate}] - else - [] - end + inversion_from_layer(lower, upper, sfc_hght) end) # Merge adjacent layers within 200m gap merged = raw_inversions - |> Enum.reduce([], fn inv, acc -> - case acc do - [] -> - [inv] - - [last | rest] -> - if inv.base <= last.top + 200 do - merged_inv = %{last | top: inv.top, strength: last.strength + inv.strength} - [merged_inv | rest] - else - [inv | acc] - end - end - end) + |> Enum.reduce([], fn inv, acc -> merge_adjacent_inversion(inv, acc) end) |> Enum.reverse() # Keep only meaningful inversions: strength >= 0.5°C AND base below 5000m AGL @@ -155,28 +133,7 @@ defmodule Microwaveprop.Weather.SoundingParams do refract_profile |> Enum.chunk_every(2, 1, :discard) |> Enum.reduce({[], nil}, fn [prev, curr], {ducts, duct_state} -> - dm = curr.m - prev.m - - case {dm < 0, duct_state} do - {true, nil} -> - # Entering a duct - {ducts, %{base: prev.h_agl, base_m: prev.m}} - - {false, %{base: base, base_m: base_m}} -> - # Exiting a duct - top = prev.h_agl - strength = base_m - prev.m - - if strength > 2 do - duct = %{"base" => base, "top" => top, "strength" => Float.round(strength, 1)} - {[duct | ducts], nil} - else - {ducts, nil} - end - - _ -> - {ducts, duct_state} - end + classify_duct_transition(prev, curr, ducts, duct_state) end) # Handle case where profile ends while still in a duct @@ -193,6 +150,61 @@ defmodule Microwaveprop.Weather.SoundingParams do Enum.reverse(ducts) end + defp inversion_from_layer(lower, upper, sfc_hght) do + if upper["tmpc"] > lower["tmpc"] do + base = lower["hght"] - sfc_hght + top = upper["hght"] - sfc_hght + strength = upper["tmpc"] - lower["tmpc"] + dh_100m = (top - base) / 100.0 + rate = if dh_100m > 0, do: strength / dh_100m, else: 0.0 + [%{base: base, top: top, strength: strength, rate: rate}] + else + [] + end + end + + defp merge_adjacent_inversion(inv, []) do + [inv] + end + + defp merge_adjacent_inversion(inv, [last | rest] = acc) do + if inv.base <= last.top + 200 do + merged_inv = %{last | top: inv.top, strength: last.strength + inv.strength} + [merged_inv | rest] + else + [inv | acc] + end + end + + defp classify_duct_transition(prev, curr, ducts, duct_state) do + dm = curr.m - prev.m + + case {dm < 0, duct_state} do + {true, nil} -> + # Entering a duct + {ducts, %{base: prev.h_agl, base_m: prev.m}} + + {false, %{base: base, base_m: base_m}} -> + # Exiting a duct + finalize_duct(ducts, base, base_m, prev) + + _ -> + {ducts, duct_state} + end + end + + defp finalize_duct(ducts, base, base_m, prev) do + top = prev.h_agl + strength = base_m - prev.m + + if strength > 2 do + duct = %{"base" => base, "top" => top, "strength" => Float.round(strength, 1)} + {[duct | ducts], nil} + else + {ducts, nil} + end + end + defp compute_k_index(sorted) do p850 = nearest_level(sorted, 850) p700 = nearest_level(sorted, 700) diff --git a/lib/microwaveprop/workers/asos_adjustment_worker.ex b/lib/microwaveprop/workers/asos_adjustment_worker.ex index b81b40ab..aa9b2244 100644 --- a/lib/microwaveprop/workers/asos_adjustment_worker.ex +++ b/lib/microwaveprop/workers/asos_adjustment_worker.ex @@ -56,30 +56,34 @@ defmodule Microwaveprop.Workers.AsosAdjustmentWorker do :ok else valid_time = DateTime.truncate(now, :second) - scores = score_grid_from_asos(fresh, valid_time) - - Logger.info("AsosAdjustment: computed #{length(scores)} adjusted scores") - - case Propagation.upsert_scores(scores) do - {:ok, count} -> - Logger.info("AsosAdjustment: upserted #{count} scores") - - Phoenix.PubSub.broadcast( - Microwaveprop.PubSub, - "propagation:updated", - {:propagation_updated, valid_time} - ) - - :ok - - error -> - Logger.error("AsosAdjustment: upsert failed: #{inspect(error)}") - error - end + apply_asos_scores(fresh, valid_time) end end end + defp apply_asos_scores(fresh, valid_time) do + scores = score_grid_from_asos(fresh, valid_time) + + Logger.info("AsosAdjustment: computed #{length(scores)} adjusted scores") + + case Propagation.upsert_scores(scores) do + {:ok, count} -> + Logger.info("AsosAdjustment: upserted #{count} scores") + + Phoenix.PubSub.broadcast( + Microwaveprop.PubSub, + "propagation:updated", + {:propagation_updated, valid_time} + ) + + :ok + + error -> + Logger.error("AsosAdjustment: upsert failed: #{inspect(error)}") + error + end + end + defp score_grid_from_asos(observations, valid_time) do # Build a spatial index of observations obs_list = diff --git a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex index ac6148b2..09304439 100644 --- a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex +++ b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex @@ -23,13 +23,10 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do """ def enqueue_for_contact(contact, types \\ [:hrrr, :weather, :terrain, :iemre]) do contact = Radio.ensure_positions!(contact) - weather_jobs = if :weather in types, do: build_weather_jobs([contact]), else: [] - hrrr_jobs = if :hrrr in types, do: build_hrrr_jobs([contact]), else: [] - terrain_jobs = if :terrain in types, do: build_terrain_jobs([contact]), else: [] - iemre_jobs = if :iemre in types, do: build_iemre_jobs([contact]), else: [] - era5_jobs = if :era5 in types, do: build_era5_jobs([contact]), else: [] + jobs_by_type = build_jobs_by_type(contact, types) - bulk_jobs = weather_jobs ++ hrrr_jobs ++ terrain_jobs ++ iemre_jobs + bulk_jobs = + jobs_by_type[:weather] ++ jobs_by_type[:hrrr] ++ jobs_by_type[:terrain] ++ jobs_by_type[:iemre] if bulk_jobs != [] do insert_all_chunked(bulk_jobs) @@ -37,21 +34,35 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do # ERA5 jobs must go through Oban.insert/1 so the worker's unique # constraint is honored; Oban OSS insert_all does not check uniqueness. - insert_unique(era5_jobs) + insert_unique(jobs_by_type[:era5]) + mark_enrichment_statuses(contact, types, jobs_by_type) + + :ok + end + + defp build_jobs_by_type(contact, types) do + %{ + weather: if(:weather in types, do: build_weather_jobs([contact]), else: []), + hrrr: if(:hrrr in types, do: build_hrrr_jobs([contact]), else: []), + terrain: if(:terrain in types, do: build_terrain_jobs([contact]), else: []), + iemre: if(:iemre in types, do: build_iemre_jobs([contact]), else: []), + era5: if(:era5 in types, do: build_era5_jobs([contact]), else: []) + } + end + + defp mark_enrichment_statuses(contact, types, jobs_by_type) do ids = [contact.id] if contact.pos1 do - if :weather in types, do: mark_status!(ids, :weather_status, weather_jobs) - if :hrrr in types, do: mark_status!(ids, :hrrr_status, hrrr_jobs) - if :iemre in types, do: mark_status!(ids, :iemre_status, iemre_jobs) + if :weather in types, do: mark_status!(ids, :weather_status, jobs_by_type[:weather]) + if :hrrr in types, do: mark_status!(ids, :hrrr_status, jobs_by_type[:hrrr]) + if :iemre in types, do: mark_status!(ids, :iemre_status, jobs_by_type[:iemre]) end if contact.pos1 && contact.pos2 do - if :terrain in types, do: mark_status!(ids, :terrain_status, terrain_jobs) + if :terrain in types, do: mark_status!(ids, :terrain_status, jobs_by_type[:terrain]) end - - :ok end @impl Oban.Worker diff --git a/lib/microwaveprop/workers/hrrr_fetch_worker.ex b/lib/microwaveprop/workers/hrrr_fetch_worker.ex index fd1b1ae1..86fa1dbf 100644 --- a/lib/microwaveprop/workers/hrrr_fetch_worker.ex +++ b/lib/microwaveprop/workers/hrrr_fetch_worker.ex @@ -32,26 +32,7 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do :ok else Logger.info("HRRR batch: fetching #{length(point_tuples)} points for #{valid_time_str}") - - case HrrrClient.fetch_grid(point_tuples, valid_time) do - {:ok, grid_data} -> - profiles = - Enum.map(grid_data, fn {{lat, lon}, data} -> - build_profile_attrs(lat, lon, valid_time, data) - end) - - Weather.upsert_hrrr_profiles_batch(profiles, skip_existing: true) - - Enum.each(grid_data, fn {{lat, lon}, _data} -> - broadcast_enrichment(lat, lon, valid_time) - end) - - Logger.info("HRRR batch: saved #{map_size(grid_data)} profiles for #{valid_time_str}") - :ok - - {:error, reason} -> - handle_error(reason, "batch @ #{valid_time_str}") - end + fetch_and_store_batch(point_tuples, valid_time, valid_time_str) end end @@ -78,6 +59,28 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do end end + defp fetch_and_store_batch(point_tuples, valid_time, valid_time_str) do + case HrrrClient.fetch_grid(point_tuples, valid_time) do + {:ok, grid_data} -> + profiles = + Enum.map(grid_data, fn {{lat, lon}, data} -> + build_profile_attrs(lat, lon, valid_time, data) + end) + + Weather.upsert_hrrr_profiles_batch(profiles, skip_existing: true) + + Enum.each(grid_data, fn {{lat, lon}, _data} -> + broadcast_enrichment(lat, lon, valid_time) + end) + + Logger.info("HRRR batch: saved #{map_size(grid_data)} profiles for #{valid_time_str}") + :ok + + {:error, reason} -> + handle_error(reason, "batch @ #{valid_time_str}") + end + end + defp store_profile(lat, lon, valid_time, data) do attrs = build_profile_attrs(lat, lon, valid_time, data) Weather.upsert_hrrr_profile(attrs) diff --git a/lib/microwaveprop/workers/iemre_fetch_worker.ex b/lib/microwaveprop/workers/iemre_fetch_worker.ex index 75630e65..107a90b0 100644 --- a/lib/microwaveprop/workers/iemre_fetch_worker.ex +++ b/lib/microwaveprop/workers/iemre_fetch_worker.ex @@ -22,34 +22,41 @@ defmodule Microwaveprop.Workers.IemreFetchWorker do :ok else Logger.info("Fetching IEMRE data for #{lat},#{lon} @ #{date_str}") + fetch_and_store_iemre(lat, lon, date, date_str) + end + end - case IemClient.fetch_iemre(lat, lon, date) do - {:ok, []} -> - # Store stub so this lat/lon/date isn't retried on future backfills - Weather.upsert_iemre_observation(%{lat: lat, lon: lon, date: date, hourly: []}) - Logger.info("IEMRE: no data available for #{lat},#{lon} @ #{date_str}, stored stub") - :ok + defp fetch_and_store_iemre(lat, lon, date, date_str) do + case IemClient.fetch_iemre(lat, lon, date) do + {:ok, []} -> + # Store stub so this lat/lon/date isn't retried on future backfills + Weather.upsert_iemre_observation(%{lat: lat, lon: lon, date: date, hourly: []}) + Logger.info("IEMRE: no data available for #{lat},#{lon} @ #{date_str}, stored stub") + :ok - {:ok, data} -> - Weather.upsert_iemre_observation(%{ - lat: lat, - lon: lon, - date: date, - hourly: data - }) + {:ok, data} -> + Weather.upsert_iemre_observation(%{ + lat: lat, + lon: lon, + date: date, + hourly: data + }) - Logger.info("IEMRE observation saved for #{lat},#{lon} @ #{date_str} (#{length(data)} hours)") - :ok + Logger.info("IEMRE observation saved for #{lat},#{lon} @ #{date_str} (#{length(data)} hours)") + :ok - {:error, reason} -> - if transient_failure?(reason) do - Logger.error("IEMRE transient error for #{lat},#{lon} @ #{date_str}: #{inspect(reason)}") - {:error, reason} - else - Logger.warning("IEMRE permanent failure for #{lat},#{lon} @ #{date_str}: #{inspect(reason)}") - {:cancel, reason} - end - end + {:error, reason} -> + handle_error(reason, lat, lon, date_str) + end + end + + defp handle_error(reason, lat, lon, date_str) do + if transient_failure?(reason) do + Logger.error("IEMRE transient error for #{lat},#{lon} @ #{date_str}: #{inspect(reason)}") + {:error, reason} + else + Logger.warning("IEMRE permanent failure for #{lat},#{lon} @ #{date_str}: #{inspect(reason)}") + {:cancel, reason} end end diff --git a/lib/microwaveprop/workers/propagation_grid_worker.ex b/lib/microwaveprop/workers/propagation_grid_worker.ex index 06f20663..a7c9d283 100644 --- a/lib/microwaveprop/workers/propagation_grid_worker.ex +++ b/lib/microwaveprop/workers/propagation_grid_worker.ex @@ -122,42 +122,7 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do count = grid_data |> Stream.flat_map(fn {{lat, lon}, profile} -> - temp = profile.surface_temp_c - - if temp && temp > -80 && temp < 60 do - params = - if is_list(profile.profile) and length(profile.profile) >= 3, - do: SoundingParams.derive(profile.profile) - - attrs = %{ - valid_time: valid_time, - lat: lat, - lon: lon, - run_time: profile.run_time, - profile: profile.profile || [], - hpbl_m: profile.hpbl_m, - pwat_mm: profile.pwat_mm, - surface_temp_c: profile.surface_temp_c, - surface_dewpoint_c: profile.surface_dewpoint_c, - surface_pressure_mb: profile.surface_pressure_mb - } - - attrs = - if params do - Map.merge(attrs, %{ - surface_refractivity: params.surface_refractivity, - min_refractivity_gradient: params.min_refractivity_gradient, - ducting_detected: params.ducting_detected, - duct_characteristics: params.duct_characteristics - }) - else - attrs - end - - [attrs] - else - [] - end + build_profile_attrs_for_grid(lat, lon, valid_time, profile) end) |> Stream.chunk_every(500) |> Enum.reduce(0, fn chunk, acc -> @@ -168,6 +133,45 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do Logger.info("PropagationGrid: stored #{count} HRRR profiles") end + defp build_profile_attrs_for_grid(lat, lon, valid_time, profile) do + temp = profile.surface_temp_c + + if temp && temp > -80 && temp < 60 do + params = + if is_list(profile.profile) and length(profile.profile) >= 3, + do: SoundingParams.derive(profile.profile) + + attrs = %{ + valid_time: valid_time, + lat: lat, + lon: lon, + run_time: profile.run_time, + profile: profile.profile || [], + hpbl_m: profile.hpbl_m, + pwat_mm: profile.pwat_mm, + surface_temp_c: profile.surface_temp_c, + surface_dewpoint_c: profile.surface_dewpoint_c, + surface_pressure_mb: profile.surface_pressure_mb + } + + attrs = + if params do + Map.merge(attrs, %{ + surface_refractivity: params.surface_refractivity, + min_refractivity_gradient: params.min_refractivity_gradient, + ducting_detected: params.ducting_detected, + duct_characteristics: params.duct_characteristics + }) + else + attrs + end + + [attrs] + else + [] + end + end + defp merge_native_duct_data(grid_data, run_time, forecast_hour) do hour_dt = HrrrClient.nearest_hrrr_hour(run_time) date = DateTime.to_date(hour_dt) @@ -179,13 +183,7 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do end) do {:ok, duct_grid} -> Logger.info("PropagationGrid: merged #{map_size(duct_grid)} native duct cells") - - Map.new(grid_data, fn {point, profile} -> - case Map.get(duct_grid, point) do - nil -> {point, profile} - duct -> {point, Map.merge(profile, duct)} - end - end) + apply_duct_grid(grid_data, duct_grid) {:error, reason} -> Logger.warning("PropagationGrid: native duct fetch failed (continuing without): #{inspect(reason)}") @@ -193,6 +191,15 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do end end + defp apply_duct_grid(grid_data, duct_grid) do + Map.new(grid_data, fn {point, profile} -> + case Map.get(duct_grid, point) do + nil -> {point, profile} + duct -> {point, Map.merge(profile, duct)} + end + end) + end + defp compute_scores(grid_data, valid_time) do # Algorithm is the primary scorer. ML score stored in factors for comparison. compute_scores_algorithm(grid_data, valid_time) diff --git a/lib/microwaveprop_web/live/about_live.ex b/lib/microwaveprop_web/live/about_live.ex index 8de0288e..ce14e575 100644 --- a/lib/microwaveprop_web/live/about_live.ex +++ b/lib/microwaveprop_web/live/about_live.ex @@ -203,7 +203,11 @@ defmodule MicrowavepropWeb.AboutLive do