From 6489f081383660bddc502a6c1c0f8d343ecda303 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sat, 28 Mar 2026 15:57:19 -0500 Subject: [PATCH] Add weather data schema, context, sounding params, and IEM ingestion Store surface observations (ASOS) and upper-air soundings (RAOB) alongside QSOs for atmospheric propagation correlation. Three new tables: weather_stations, surface_observations, and soundings with JSONB profiles and pre-computed derived parameters (refractivity, gradients, duct detection, stability indices). Includes IEM API client for historical data import and import script seeded with 95 ASOS + 9 sounding stations from PropCast coverage area. --- lib/microwaveprop/weather.ex | 97 ++++++ lib/microwaveprop/weather/iem_client.ex | 159 +++++++++ lib/microwaveprop/weather/sounding.ex | 40 +++ lib/microwaveprop/weather/sounding_params.ex | 266 +++++++++++++++ lib/microwaveprop/weather/station.ex | 33 ++ .../weather/surface_observation.ex | 35 ++ priv/repo/import_weather.exs | 320 ++++++++++++++++++ .../20260328204648_create_weather_tables.exs | 66 ++++ .../microwaveprop/weather/iem_client_test.exs | 119 +++++++ .../weather/sounding_params_test.exs | 161 +++++++++ test/microwaveprop/weather/sounding_test.exs | 115 +++++++ test/microwaveprop/weather/station_test.exs | 71 ++++ .../weather/surface_observation_test.exs | 79 +++++ test/microwaveprop/weather_test.exs | 199 +++++++++++ 14 files changed, 1760 insertions(+) create mode 100644 lib/microwaveprop/weather.ex create mode 100644 lib/microwaveprop/weather/iem_client.ex create mode 100644 lib/microwaveprop/weather/sounding.ex create mode 100644 lib/microwaveprop/weather/sounding_params.ex create mode 100644 lib/microwaveprop/weather/station.ex create mode 100644 lib/microwaveprop/weather/surface_observation.ex create mode 100644 priv/repo/import_weather.exs create mode 100644 priv/repo/migrations/20260328204648_create_weather_tables.exs create mode 100644 test/microwaveprop/weather/iem_client_test.exs create mode 100644 test/microwaveprop/weather/sounding_params_test.exs create mode 100644 test/microwaveprop/weather/sounding_test.exs create mode 100644 test/microwaveprop/weather/station_test.exs create mode 100644 test/microwaveprop/weather/surface_observation_test.exs create mode 100644 test/microwaveprop/weather_test.exs diff --git a/lib/microwaveprop/weather.ex b/lib/microwaveprop/weather.ex new file mode 100644 index 00000000..15adb630 --- /dev/null +++ b/lib/microwaveprop/weather.ex @@ -0,0 +1,97 @@ +defmodule Microwaveprop.Weather do + @moduledoc false + + import Ecto.Query + + alias Microwaveprop.Repo + alias Microwaveprop.Weather.Sounding + alias Microwaveprop.Weather.Station + alias Microwaveprop.Weather.SurfaceObservation + + # Approximate km per degree latitude + @km_per_deg_lat 111.0 + + def find_or_create_station(attrs) do + code = attrs[:station_code] || attrs["station_code"] + type = attrs[:station_type] || attrs["station_type"] + + if code && type do + case Repo.get_by(Station, station_code: code, station_type: type) do + nil -> + %Station{} + |> Station.changeset(attrs) + |> Repo.insert() + + station -> + {:ok, station} + end + else + %Station{} + |> Station.changeset(attrs) + |> Repo.insert() + end + end + + def upsert_surface_observation(%Station{} = station, attrs) do + attrs = Map.put(attrs, :station_id, station.id) + + %SurfaceObservation{} + |> SurfaceObservation.changeset(attrs) + |> Repo.insert( + on_conflict: {:replace_all_except, [:id, :station_id, :observed_at, :inserted_at]}, + conflict_target: [:station_id, :observed_at], + returning: true + ) + end + + def upsert_sounding(%Station{} = station, attrs) do + attrs = Map.put(attrs, :station_id, station.id) + + %Sounding{} + |> Sounding.changeset(attrs) + |> Repo.insert( + on_conflict: {:replace_all_except, [:id, :station_id, :observed_at, :inserted_at]}, + conflict_target: [:station_id, :observed_at], + returning: true + ) + end + + def weather_for_qso(qso_params, opts \\ []) do + lat = qso_params[:lat] || qso_params.lat + lon = qso_params[:lon] || qso_params.lon + timestamp = qso_params[:timestamp] || qso_params.timestamp + + radius_km = Keyword.get(opts, :radius_km, 150) + time_window_hours = Keyword.get(opts, :time_window_hours, 6) + + # Bounding box in degrees + dlat = radius_km / @km_per_deg_lat + dlon = radius_km / (@km_per_deg_lat * :math.cos(lat * :math.pi() / 180)) + + time_start = DateTime.add(timestamp, -time_window_hours * 3600, :second) + time_end = DateTime.add(timestamp, time_window_hours * 3600, :second) + + station_ids = + Station + |> where( + [s], + s.lat >= ^(lat - dlat) and s.lat <= ^(lat + dlat) and + s.lon >= ^(lon - dlon) and s.lon <= ^(lon + dlon) + ) + |> select([s], s.id) + + surface_observations = + SurfaceObservation + |> where([o], o.station_id in subquery(station_ids)) + |> where([o], o.observed_at >= ^time_start and o.observed_at <= ^time_end) + |> Repo.all() + + soundings = + Sounding + |> where([s], s.station_id in subquery(station_ids)) + |> where([s], s.observed_at >= ^time_start and s.observed_at <= ^time_end) + |> Repo.all() + + %{surface_observations: surface_observations, soundings: soundings} + end +end diff --git a/lib/microwaveprop/weather/iem_client.ex b/lib/microwaveprop/weather/iem_client.ex new file mode 100644 index 00000000..49e88a20 --- /dev/null +++ b/lib/microwaveprop/weather/iem_client.ex @@ -0,0 +1,159 @@ +defmodule Microwaveprop.Weather.IemClient do + @moduledoc false + + @iem_base "https://mesonet.agron.iastate.edu" + + # --- URL builders --- + + def asos_url(station_id, start_dt, end_dt) do + "#{@iem_base}/cgi-bin/request/asos.py" <> + "?station=#{station_id}" <> + "&data=tmpf&data=dwpf&data=relh&data=sknt&data=drct" <> + "&data=mslp&data=alti&data=skyc1" <> + "&year1=#{start_dt.year}&month1=#{start_dt.month}&day1=#{start_dt.day}" <> + "&hour1=#{start_dt.hour}&minute1=0" <> + "&year2=#{end_dt.year}&month2=#{end_dt.month}&day2=#{end_dt.day}" <> + "&hour2=#{end_dt.hour}&minute2=59" <> + "&format=onlycomma&latlon=no&elev=no&missing=null&trace=null&direct=no&report_type=3" + end + + def raob_url(station_id, dt) do + ts = format_iem_ts(dt) + "#{@iem_base}/json/raob.py?ts=#{ts}&station=#{station_id}" + end + + # --- HTTP fetchers --- + + def fetch_asos(station_id, start_dt, end_dt) do + url = asos_url(station_id, start_dt, end_dt) + + case Req.get(url) do + {:ok, %{status: 200, body: body}} -> + {:ok, parse_asos_csv(body)} + + {:ok, %{status: status}} -> + {:error, "IEM ASOS HTTP #{status}"} + + {:error, reason} -> + {:error, reason} + end + end + + def fetch_raob(station_id, dt) do + url = raob_url(station_id, dt) + + case Req.get(url) do + {:ok, %{status: 200, body: body}} -> + {:ok, parse_raob_json(body)} + + {:ok, %{status: status}} -> + {:error, "IEM RAOB HTTP #{status}"} + + {:error, reason} -> + {:error, reason} + end + end + + # --- Parsers --- + + def parse_asos_csv(csv_text) do + csv_text + |> String.split("\n") + |> Enum.reject(fn line -> + line == "" or String.starts_with?(line, "#") or String.starts_with?(line, "station") + end) + |> Enum.map(&parse_asos_row/1) + end + + def parse_raob_json(json) when is_map(json) do + json + |> Map.get("profiles", []) + |> Enum.map(fn entry -> + %{ + observed_at: parse_raob_timestamp(entry["valid"]), + profile: entry["profile"] || [] + } + end) + end + + # --- Private --- + + defp parse_asos_row(line) do + parts = String.split(line, ",") + + %{ + observed_at: parse_asos_timestamp(Enum.at(parts, 1)), + temp_f: parse_float(Enum.at(parts, 2)), + dewpoint_f: parse_float(Enum.at(parts, 3)), + relative_humidity: parse_float(Enum.at(parts, 4)), + wind_speed_kts: parse_float(Enum.at(parts, 5)), + wind_direction_deg: parse_int(Enum.at(parts, 6)), + sea_level_pressure_mb: parse_float(Enum.at(parts, 7)), + altimeter_setting: parse_float(Enum.at(parts, 8)), + sky_condition: parse_nullable_string(Enum.at(parts, 9)) + } + end + + defp parse_asos_timestamp(str) when is_binary(str) do + # Format: "2026-03-28 18:53" + case DateTime.from_iso8601(String.trim(str) <> ":00Z") do + {:ok, dt, _} -> DateTime.truncate(dt, :second) + _ -> nil + end + end + + defp parse_asos_timestamp(_), do: nil + + defp parse_raob_timestamp(str) when is_binary(str) do + # Format: "2026-03-28 12:00:00+00:00" + cleaned = + str + |> String.trim() + |> String.replace(~r/\+00:00$/, "Z") + |> String.replace(" ", "T") + + case DateTime.from_iso8601(cleaned) do + {:ok, dt, _} -> DateTime.truncate(dt, :second) + _ -> nil + end + end + + defp parse_raob_timestamp(_), do: nil + + defp parse_float(nil), do: nil + defp parse_float("null"), do: nil + + defp parse_float(str) do + case Float.parse(String.trim(str)) do + {val, _} -> val + :error -> nil + end + end + + defp parse_int(nil), do: nil + defp parse_int("null"), do: nil + + defp parse_int(str) do + case Integer.parse(String.trim(str)) do + {val, _} -> val + :error -> nil + end + end + + defp parse_nullable_string(nil), do: nil + defp parse_nullable_string("null"), do: nil + + defp parse_nullable_string(str) do + trimmed = String.trim(str) + if trimmed == "", do: nil, else: trimmed + end + + defp format_iem_ts(dt) do + y = Integer.to_string(dt.year) + mo = dt.month |> Integer.to_string() |> String.pad_leading(2, "0") + d = dt.day |> Integer.to_string() |> String.pad_leading(2, "0") + h = dt.hour |> Integer.to_string() |> String.pad_leading(2, "0") + mi = dt.minute |> Integer.to_string() |> String.pad_leading(2, "0") + "#{y}#{mo}#{d}#{h}#{mi}" + end +end diff --git a/lib/microwaveprop/weather/sounding.ex b/lib/microwaveprop/weather/sounding.ex new file mode 100644 index 00000000..82899c58 --- /dev/null +++ b/lib/microwaveprop/weather/sounding.ex @@ -0,0 +1,40 @@ +defmodule Microwaveprop.Weather.Sounding do + @moduledoc false + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + schema "soundings" do + belongs_to :station, Microwaveprop.Weather.Station + field :observed_at, :utc_datetime + field :profile, {:array, :map} + field :level_count, :integer + field :surface_pressure_mb, :float + field :surface_temp_c, :float + field :surface_dewpoint_c, :float + field :surface_refractivity, :float + field :min_refractivity_gradient, :float + field :boundary_layer_depth_m, :float + field :precipitable_water_mm, :float + field :k_index, :float + field :lifted_index, :float + field :ducting_detected, :boolean, default: false + field :duct_characteristics, {:array, :map} + + timestamps(type: :utc_datetime) + end + + @required_fields ~w(station_id observed_at profile level_count)a + @optional_fields ~w(surface_pressure_mb surface_temp_c surface_dewpoint_c surface_refractivity min_refractivity_gradient boundary_layer_depth_m precipitable_water_mm k_index lifted_index ducting_detected duct_characteristics)a + + def changeset(sounding, attrs) do + sounding + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) + |> foreign_key_constraint(:station_id) + |> unique_constraint([:station_id, :observed_at]) + end +end diff --git a/lib/microwaveprop/weather/sounding_params.ex b/lib/microwaveprop/weather/sounding_params.ex new file mode 100644 index 00000000..7fd1e01b --- /dev/null +++ b/lib/microwaveprop/weather/sounding_params.ex @@ -0,0 +1,266 @@ +defmodule Microwaveprop.Weather.SoundingParams do + @moduledoc false + + @doc "Buck equation for saturation vapor pressure (hPa) given temperature in °C." + def sat_vap_pres(t_c) do + 6.1121 * :math.exp((18.678 - t_c / 234.5) * (t_c / (257.14 + t_c))) + end + + @doc "Mixing ratio (g/kg) from dewpoint (°C) and pressure (hPa)." + def mixing_ratio(t_c, p_mb) do + e = sat_vap_pres(t_c) + 622.0 * e / (p_mb - e) + end + + @doc """ + Derive atmospheric propagation parameters from a sounding profile. + + Profile is a list of maps with keys: "pres", "hght", "tmpc", "dwpc", "drct", "sknt". + Returns nil if fewer than 3 valid levels. + """ + def derive(profile) when is_list(profile) do + sorted = + profile + |> Enum.filter(fn p -> p["pres"] != nil and p["tmpc"] != nil and p["hght"] != nil end) + |> Enum.sort_by(fn p -> p["pres"] end, :desc) + + if length(sorted) < 3 do + nil + else + do_derive(sorted) + end + end + + defp do_derive(sorted) do + sfc = hd(sorted) + sfc_hght = sfc["hght"] + + refract_profile = compute_refractivity_profile(sorted, sfc_hght) + dn_dh = compute_gradients(refract_profile) + inversions = detect_inversions(sorted, sfc_hght) + ducts = detect_ducts(refract_profile) + k_index = compute_k_index(sorted) + li = compute_lifted_index(sorted, sfc) + pw = compute_precipitable_water(sorted) + bl_depth = compute_boundary_layer_depth(sorted, sfc) + min_grad = find_min_gradient(dn_dh) + + sfc_n = + case refract_profile do + [first | _] -> first.n + _ -> nil + end + + %{ + level_count: length(sorted), + surface_pressure_mb: sfc["pres"], + surface_temp_c: sfc["tmpc"], + surface_dewpoint_c: sfc["dwpc"] || sfc["tmpc"] - 10, + surface_refractivity: sfc_n, + min_refractivity_gradient: min_grad, + boundary_layer_depth_m: bl_depth, + precipitable_water_mm: pw, + k_index: k_index, + lifted_index: li, + ducting_detected: ducts != [], + ducts: ducts, + duct_characteristics: if(ducts == [], do: nil, else: ducts), + inversions: inversions, + profile: sorted + } + end + + defp compute_refractivity_profile(sorted, sfc_hght) do + sorted + |> Enum.filter(fn p -> p["dwpc"] != nil end) + |> Enum.map(fn p -> + t_k = p["tmpc"] + 273.15 + e = sat_vap_pres(p["dwpc"]) + n = 77.6 * p["pres"] / t_k + 3.73e5 * e / (t_k * t_k) + h_agl = p["hght"] - sfc_hght + m = n + 0.157 * h_agl + + %{ + pres: p["pres"], + h_agl: h_agl, + h_km: h_agl / 1000.0, + tmpc: p["tmpc"], + dwpc: p["dwpc"], + n: n, + m: m + } + end) + end + + defp compute_gradients(refract_profile) do + refract_profile + |> Enum.chunk_every(2, 1, :discard) + |> Enum.flat_map(fn [prev, curr] -> + dh_km = (curr.h_agl - prev.h_agl) / 1000.0 + + if abs(dh_km) < 0.01 do + [] + else + dn = curr.n - prev.n + [%{h_agl: (curr.h_agl + prev.h_agl) / 2.0, gradient: dn / dh_km}] + end + end) + end + + defp detect_inversions(sorted, sfc_hght) do + raw_inversions = + sorted + |> Enum.chunk_every(2, 1, :discard) + |> Enum.flat_map(fn [lower, upper] -> + if upper["tmpc"] > lower["tmpc"] do + base = lower["hght"] - sfc_hght + top = upper["hght"] - sfc_hght + strength = upper["tmpc"] - lower["tmpc"] + dh_100m = (top - base) / 100.0 + rate = if dh_100m > 0, do: strength / dh_100m, else: 0.0 + [%{base: base, top: top, strength: strength, rate: rate}] + else + [] + end + end) + + # Merge adjacent layers within 200m gap + merged = + raw_inversions + |> Enum.reduce([], fn inv, acc -> + case acc do + [] -> + [inv] + + [last | rest] -> + if inv.base <= last.top + 200 do + merged_inv = %{last | top: inv.top, strength: last.strength + inv.strength} + [merged_inv | rest] + else + [inv | acc] + end + end + end) + |> Enum.reverse() + + # Keep only meaningful inversions: strength >= 0.5°C AND base below 5000m AGL + Enum.filter(merged, fn inv -> inv.strength >= 0.5 and inv.base < 5000 end) + end + + defp detect_ducts(refract_profile) do + {ducts, in_duct_state} = + refract_profile + |> Enum.chunk_every(2, 1, :discard) + |> Enum.reduce({[], nil}, fn [prev, curr], {ducts, duct_state} -> + dm = curr.m - prev.m + + case {dm < 0, duct_state} do + {true, nil} -> + # Entering a duct + {ducts, %{base: prev.h_agl, base_m: prev.m}} + + {false, %{base: base, base_m: base_m}} -> + # Exiting a duct + top = prev.h_agl + strength = base_m - prev.m + + if strength > 2 do + duct = %{"base" => base, "top" => top, "strength" => Float.round(strength, 1)} + {[duct | ducts], nil} + else + {ducts, nil} + end + + _ -> + {ducts, duct_state} + end + end) + + # Handle case where profile ends while still in a duct + ducts = + case in_duct_state do + %{base: _base, base_m: _base_m} -> + # Duct didn't close — discard (no clear top) + ducts + + nil -> + ducts + end + + Enum.reverse(ducts) + end + + defp compute_k_index(sorted) do + p850 = nearest_level(sorted, 850) + p700 = nearest_level(sorted, 700) + p500 = nearest_level(sorted, 500) + + with %{"tmpc" => t850} when t850 != nil <- p850, + %{"tmpc" => t700} when t700 != nil <- p700, + %{"tmpc" => t500} when t500 != nil <- p500 do + td850 = p850["dwpc"] || -30.0 + td700 = p700["dwpc"] || -30.0 + t850 - t500 + td850 - (t700 - td700) + else + _ -> nil + end + end + + defp compute_lifted_index(sorted, sfc) do + p500 = nearest_level(sorted, 500) + + case p500 do + %{"tmpc" => t500, "hght" => h500} when t500 != nil and h500 != nil -> + sfc_tmpc = sfc["tmpc"] + sfc_hght = sfc["hght"] + # Simplified LI: T500 - (Tsfc - lapse_rate * height_diff) + t500 - (sfc_tmpc - (h500 - sfc_hght) * 0.00976) + + _ -> + nil + end + end + + defp compute_precipitable_water(sorted) do + sorted + |> Enum.chunk_every(2, 1, :discard) + |> Enum.reduce(0.0, fn [lower, upper], pw -> + if lower["dwpc"] != nil and upper["dwpc"] != nil do + mr1 = mixing_ratio(lower["dwpc"], lower["pres"]) + mr2 = mixing_ratio(upper["dwpc"], upper["pres"]) + dp = lower["pres"] - upper["pres"] + pw + (mr1 + mr2) / 2.0 * dp / 9.81 / 10.0 + else + pw + end + end) + end + + defp compute_boundary_layer_depth(sorted, sfc) do + sfc_hght = sfc["hght"] + sfc_tmpc = sfc["tmpc"] + theta_sfc = sfc_tmpc + 273.15 + 9.8 * (sfc_hght / 1000.0) + + sorted + |> tl() + |> Enum.find_value(fn p -> + theta = p["tmpc"] + 273.15 + 9.8 * (p["hght"] / 1000.0) + + if theta > theta_sfc + 2 do + p["hght"] - sfc_hght + end + end) + end + + defp find_min_gradient([]), do: nil + + defp find_min_gradient(dn_dh) do + dn_dh + |> Enum.min_by(fn %{gradient: g} -> g end) + |> Map.get(:gradient) + end + + defp nearest_level(sorted, target_pres) do + Enum.min_by(sorted, fn p -> abs(p["pres"] - target_pres) end) + end +end diff --git a/lib/microwaveprop/weather/station.ex b/lib/microwaveprop/weather/station.ex new file mode 100644 index 00000000..9491bdb4 --- /dev/null +++ b/lib/microwaveprop/weather/station.ex @@ -0,0 +1,33 @@ +defmodule Microwaveprop.Weather.Station do + @moduledoc false + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + schema "weather_stations" do + field :station_code, :string + field :station_type, :string + field :wmo_number, :integer + field :name, :string + field :lat, :float + field :lon, :float + field :elevation_m, :float + field :state, :string + + timestamps(type: :utc_datetime) + end + + @required_fields ~w(station_code station_type name lat lon)a + @optional_fields ~w(wmo_number elevation_m state)a + + def changeset(station, attrs) do + station + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) + |> validate_inclusion(:station_type, ~w(asos sounding)) + |> unique_constraint([:station_code, :station_type]) + end +end diff --git a/lib/microwaveprop/weather/surface_observation.ex b/lib/microwaveprop/weather/surface_observation.ex new file mode 100644 index 00000000..bcb48e31 --- /dev/null +++ b/lib/microwaveprop/weather/surface_observation.ex @@ -0,0 +1,35 @@ +defmodule Microwaveprop.Weather.SurfaceObservation do + @moduledoc false + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + schema "surface_observations" do + belongs_to :station, Microwaveprop.Weather.Station + field :observed_at, :utc_datetime + field :temp_f, :float + field :dewpoint_f, :float + field :relative_humidity, :float + field :wind_speed_kts, :float + field :wind_direction_deg, :integer + field :sea_level_pressure_mb, :float + field :altimeter_setting, :float + field :sky_condition, :string + + timestamps(type: :utc_datetime) + end + + @required_fields ~w(station_id observed_at)a + @optional_fields ~w(temp_f dewpoint_f relative_humidity wind_speed_kts wind_direction_deg sea_level_pressure_mb altimeter_setting sky_condition)a + + def changeset(observation, attrs) do + observation + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) + |> foreign_key_constraint(:station_id) + |> unique_constraint([:station_id, :observed_at]) + end +end diff --git a/priv/repo/import_weather.exs b/priv/repo/import_weather.exs new file mode 100644 index 00000000..452c665b --- /dev/null +++ b/priv/repo/import_weather.exs @@ -0,0 +1,320 @@ +alias Microwaveprop.{Repo, Weather} +alias Microwaveprop.Weather.{IemClient, SoundingParams} + +import Ecto.Query + +# --- Station definitions (from PropCast) --- + +asos_stations = [ + %{station_code: "KDFW", name: "Dallas/Fort Worth Intl", lat: 32.897, lon: -97.038, state: "TX"}, + %{station_code: "KDAL", name: "Dallas Love Field", lat: 32.847, lon: -96.851, state: "TX"}, + %{station_code: "KADS", name: "Addison", lat: 32.969, lon: -96.836, state: "TX"}, + %{station_code: "KTKI", name: "McKinney National", lat: 33.178, lon: -96.590, state: "TX"}, + %{station_code: "KGPM", name: "Grand Prairie Muni", lat: 32.698, lon: -97.043, state: "TX"}, + %{station_code: "KFWS", name: "Fort Worth Spinks", lat: 32.565, lon: -97.308, state: "TX"}, + %{station_code: "KFTW", name: "Fort Worth Meacham", lat: 32.820, lon: -97.362, state: "TX"}, + %{station_code: "KAFW", name: "Fort Worth Alliance", lat: 32.988, lon: -97.319, state: "TX"}, + %{station_code: "KRBD", name: "Dallas Executive", lat: 32.681, lon: -96.868, state: "TX"}, + %{station_code: "KGKY", name: "Arlington Muni", lat: 32.664, lon: -97.094, state: "TX"}, + %{station_code: "KCPT", name: "Cleburne Regional", lat: 32.354, lon: -97.434, state: "TX"}, + %{station_code: "KHQZ", name: "Mesquite Metro", lat: 32.747, lon: -96.530, state: "TX"}, + %{station_code: "KGYI", name: "Sherman/Denison", lat: 33.729, lon: -96.669, state: "TX"}, + %{station_code: "KGLE", name: "Gainesville Muni", lat: 33.651, lon: -97.197, state: "TX"}, + %{station_code: "KGVT", name: "Greenville/Hunt Co", lat: 33.068, lon: -96.065, state: "TX"}, + %{station_code: "KJDD", name: "Sulphur Springs", lat: 33.152, lon: -95.623, state: "TX"}, + %{station_code: "KGDJ", name: "Granbury Regional", lat: 32.445, lon: -97.817, state: "TX"}, + %{station_code: "KMWL", name: "Mineral Wells", lat: 32.782, lon: -98.060, state: "TX"}, + %{station_code: "KBKD", name: "Breckenridge/Stephens Co", lat: 32.719, lon: -98.890, state: "TX"}, + %{station_code: "KWEA", name: "Weatherford/Parker Co", lat: 32.746, lon: -97.682, state: "TX"}, + %{station_code: "KGGG", name: "Longview East Texas", lat: 32.384, lon: -94.711, state: "TX"}, + %{station_code: "KTYR", name: "Tyler Pounds", lat: 32.354, lon: -95.402, state: "TX"}, + %{station_code: "KCRS", name: "Corsicana/Navarro Co", lat: 32.028, lon: -96.400, state: "TX"}, + %{station_code: "KWXD", name: "Waxahachie/Ellis Co", lat: 32.392, lon: -96.849, state: "TX"}, + %{station_code: "KPNX", name: "Henderson/Rusk Co", lat: 32.152, lon: -94.913, state: "TX"}, + %{station_code: "KPWG", name: "Waco Regional", lat: 31.611, lon: -97.095, state: "TX"}, + %{station_code: "KACT", name: "Waco TSTC", lat: 31.611, lon: -97.226, state: "TX"}, + %{station_code: "KTPL", name: "Temple/Draughon-Miller", lat: 31.152, lon: -97.408, state: "TX"}, + %{station_code: "KILE", name: "Killeen/Fort Cavazos", lat: 31.086, lon: -97.683, state: "TX"}, + %{station_code: "KGRK", name: "Killeen-Fort Hood", lat: 31.067, lon: -97.829, state: "TX"}, + %{station_code: "KAUS", name: "Austin-Bergstrom", lat: 30.198, lon: -97.670, state: "TX"}, + %{station_code: "KSAT", name: "San Antonio Intl", lat: 29.534, lon: -98.470, state: "TX"}, + %{station_code: "KBMQ", name: "Burnet Muni", lat: 30.739, lon: -98.239, state: "TX"}, + %{station_code: "KSSF", name: "San Antonio Stinson", lat: 29.337, lon: -98.471, state: "TX"}, + %{station_code: "KGTU", name: "Georgetown Muni", lat: 30.678, lon: -97.679, state: "TX"}, + %{station_code: "KHYI", name: "San Marcos Rgnl", lat: 29.893, lon: -97.863, state: "TX"}, + %{station_code: "KHOU", name: "Houston Hobby", lat: 29.645, lon: -95.279, state: "TX"}, + %{station_code: "KIAH", name: "Houston Bush Intcntl", lat: 29.980, lon: -95.342, state: "TX"}, + %{station_code: "KCXO", name: "Conroe North Houston", lat: 30.352, lon: -95.415, state: "TX"}, + %{station_code: "KBPT", name: "Beaumont/Port Arthur", lat: 29.951, lon: -94.021, state: "TX"}, + %{station_code: "KLCH", name: "Lake Charles Regional", lat: 30.126, lon: -93.223, state: "LA"}, + %{station_code: "KVCT", name: "Victoria Regional", lat: 28.852, lon: -96.918, state: "TX"}, + %{station_code: "KCRP", name: "Corpus Christi Intl", lat: 27.770, lon: -97.502, state: "TX"}, + %{station_code: "KOKC", name: "Oklahoma City Will Rogers", lat: 35.393, lon: -97.601, state: "OK"}, + %{station_code: "KOUN", name: "Norman/OU", lat: 35.246, lon: -97.472, state: "OK"}, + %{station_code: "KPWA", name: "Wiley Post", lat: 35.534, lon: -97.647, state: "OK"}, + %{station_code: "KSWO", name: "Stillwater Regional", lat: 36.161, lon: -97.086, state: "OK"}, + %{station_code: "KTIK", name: "Tinker AFB", lat: 35.415, lon: -97.387, state: "OK"}, + %{station_code: "KLAW", name: "Lawton/Fort Sill", lat: 34.568, lon: -98.416, state: "OK"}, + %{station_code: "KADU", name: "Ardmore Downtown", lat: 34.303, lon: -97.019, state: "OK"}, + %{station_code: "KSNL", name: "Shawnee Regional", lat: 35.358, lon: -96.943, state: "OK"}, + %{station_code: "KCSM", name: "Clinton/Sherman", lat: 35.339, lon: -99.201, state: "OK"}, + %{station_code: "KGAG", name: "Gage/Ellis Co", lat: 36.296, lon: -99.777, state: "OK"}, + %{station_code: "KWDG", name: "Enid Woodring", lat: 36.379, lon: -97.791, state: "OK"}, + %{station_code: "KBVO", name: "Bartlesville", lat: 36.765, lon: -96.012, state: "OK"}, + %{station_code: "KTUL", name: "Tulsa Intl", lat: 36.198, lon: -95.888, state: "OK"}, + %{station_code: "KMKO", name: "Muskogee/Davis Field", lat: 35.657, lon: -95.367, state: "OK"}, + %{station_code: "KPOC", name: "Pontotoc Co/Ada", lat: 34.802, lon: -96.671, state: "OK"}, + %{station_code: "KGMJ", name: "Grove Muni", lat: 36.607, lon: -94.733, state: "OK"}, + %{station_code: "KABI", name: "Abilene Regional", lat: 32.411, lon: -99.682, state: "TX"}, + %{station_code: "KMAF", name: "Midland Intl", lat: 31.943, lon: -102.201, state: "TX"}, + %{station_code: "KBPG", name: "Big Spring McMahon", lat: 32.213, lon: -101.522, state: "TX"}, + %{station_code: "KSNS", name: "San Angelo Mathis", lat: 31.358, lon: -100.497, state: "TX"}, + %{station_code: "KLBB", name: "Lubbock Preston Smith", lat: 33.664, lon: -101.823, state: "TX"}, + %{station_code: "KAMA", name: "Amarillo Rick Husband", lat: 35.220, lon: -101.706, state: "TX"}, + %{station_code: "KPPA", name: "Pampa/Perry Lefors", lat: 35.613, lon: -100.996, state: "TX"}, + %{station_code: "KPVW", name: "Plainview Hale Co", lat: 34.183, lon: -101.722, state: "TX"}, + %{station_code: "KINK", name: "Wink/Winkler Co", lat: 31.779, lon: -103.201, state: "TX"}, + %{station_code: "KELP", name: "El Paso Intl", lat: 31.807, lon: -106.378, state: "TX"}, + %{station_code: "KCNM", name: "Carlsbad Cavern City", lat: 32.337, lon: -104.263, state: "NM"}, + %{station_code: "KROW", name: "Roswell Intl Air Ctr", lat: 33.302, lon: -104.531, state: "NM"}, + %{station_code: "KCLO", name: "Clovis Muni", lat: 34.434, lon: -103.079, state: "NM"}, + %{station_code: "KHOB", name: "Hobbs Lea Co", lat: 32.688, lon: -103.217, state: "NM"}, + %{station_code: "KTCC", name: "Tucumcari Muni", lat: 35.183, lon: -103.603, state: "NM"}, + %{station_code: "KICT", name: "Wichita Dwight D. Eisenhower", lat: 37.650, lon: -97.433, state: "KS"}, + %{station_code: "KHUT", name: "Hutchinson Regional", lat: 38.065, lon: -97.861, state: "KS"}, + %{station_code: "KDDC", name: "Dodge City Regional", lat: 37.763, lon: -99.965, state: "KS"}, + %{station_code: "KGLD", name: "Goodland Municipal", lat: 39.370, lon: -101.700, state: "KS"}, + %{station_code: "KGBD", name: "Great Bend Municipal", lat: 38.344, lon: -98.859, state: "KS"}, + %{station_code: "KCNU", name: "Chanute Martin Johnson", lat: 37.669, lon: -95.485, state: "KS"}, + %{station_code: "KPTT", name: "Pratt Regional", lat: 37.700, lon: -98.747, state: "KS"}, + %{station_code: "KSGF", name: "Springfield Branson Natl", lat: 37.246, lon: -93.389, state: "MO"}, + %{station_code: "KJLN", name: "Joplin Regional", lat: 37.152, lon: -94.498, state: "MO"}, + %{station_code: "KFSM", name: "Fort Smith Regional", lat: 35.337, lon: -94.368, state: "AR"}, + %{station_code: "KTXK", name: "Texarkana Regional", lat: 33.454, lon: -93.991, state: "TX"}, + %{station_code: "KLIT", name: "Little Rock Clinton", lat: 34.729, lon: -92.224, state: "AR"}, + %{station_code: "KHRO", name: "Harrison Boone Co", lat: 36.261, lon: -93.155, state: "AR"}, + %{station_code: "KFYV", name: "Fayetteville Drake Field", lat: 36.005, lon: -94.170, state: "AR"}, + %{station_code: "KXNA", name: "NW Arkansas Natl", lat: 36.282, lon: -94.307, state: "AR"}, + %{station_code: "KSHV", name: "Shreveport Regional", lat: 32.447, lon: -93.826, state: "LA"}, + %{station_code: "KMLU", name: "Monroe Regional", lat: 32.511, lon: -92.038, state: "LA"}, + %{station_code: "KJAN", name: "Jackson Medgar Wiley Evers", lat: 32.311, lon: -90.076, state: "MS"}, + %{station_code: "KMEI", name: "Meridian Regional", lat: 32.334, lon: -88.752, state: "MS"}, + %{station_code: "KNAC", name: "Nacogdoches/Mangham", lat: 31.578, lon: -94.710, state: "TX"}, + %{station_code: "KNEW", name: "New Orleans Lakefront", lat: 30.042, lon: -90.028, state: "LA"}, + %{station_code: "KMSY", name: "New Orleans Armstrong", lat: 29.993, lon: -90.258, state: "LA"} +] + +sounding_stations = [ + %{station_code: "FWD", wmo_number: 72_249, name: "Fort Worth TX", lat: 32.835, lon: -97.298, state: "TX"}, + %{station_code: "OUN", wmo_number: 72_357, name: "Oklahoma City OK", lat: 35.181, lon: -97.436, state: "OK"}, + %{station_code: "SHV", wmo_number: 72_248, name: "Shreveport LA", lat: 32.450, lon: -93.841, state: "LA"}, + %{station_code: "AMA", wmo_number: 72_363, name: "Amarillo TX", lat: 35.233, lon: -101.706, state: "TX"}, + %{station_code: "LCH", wmo_number: 72_240, name: "Lake Charles LA", lat: 30.125, lon: -93.217, state: "LA"}, + %{station_code: "MAF", wmo_number: 72_265, name: "Midland TX", lat: 31.943, lon: -102.189, state: "TX"}, + %{station_code: "LZK", wmo_number: 72_340, name: "Little Rock AR", lat: 34.836, lon: -92.263, state: "AR"}, + %{station_code: "BRO", wmo_number: 72_250, name: "Brownsville TX", lat: 25.910, lon: -97.419, state: "TX"}, + %{station_code: "SPS", wmo_number: 72_451, name: "Wichita Falls TX", lat: 33.989, lon: -98.492, state: "TX"} +] + +# --- Helpers --- + +defmodule ImportHelpers do + @moduledoc false + + @km_per_deg 111.0 + + def nearest_stations(lat, lon, stations, radius_km) do + Enum.filter(stations, fn s -> + dlat = (s.lat - lat) * @km_per_deg + dlon = (s.lon - lon) * @km_per_deg * :math.cos(lat * :math.pi() / 180) + :math.sqrt(dlat * dlat + dlon * dlon) <= radius_km + end) + end + + def sounding_times_around(dt) do + # Return the 00Z and 12Z sounding times that bracket the QSO time + date = DateTime.to_date(dt) + hour = dt.hour + + times = + if hour < 12 do + [ + DateTime.new!(Date.add(date, -1), ~T[12:00:00], "Etc/UTC"), + DateTime.new!(date, ~T[00:00:00], "Etc/UTC") + ] + else + [ + DateTime.new!(date, ~T[00:00:00], "Etc/UTC"), + DateTime.new!(date, ~T[12:00:00], "Etc/UTC") + ] + end + + Enum.uniq(times) + end + + def rate_limit(delay_ms \\ 200) do + Process.sleep(delay_ms) + end +end + +# --- Seed stations --- + +IO.puts("Seeding #{length(asos_stations)} ASOS stations...") + +asos_station_map = + Enum.reduce(asos_stations, %{}, fn attrs, acc -> + {:ok, station} = + Weather.find_or_create_station(Map.put(attrs, :station_type, "asos")) + + Map.put(acc, station.station_code, station) + end) + +IO.puts("Seeding #{length(sounding_stations)} sounding stations...") + +sounding_station_map = + Enum.reduce(sounding_stations, %{}, fn attrs, acc -> + {:ok, station} = + Weather.find_or_create_station(Map.put(attrs, :station_type, "sounding")) + + Map.put(acc, station.station_code, station) + end) + +IO.puts("Stations seeded: #{map_size(asos_station_map)} ASOS, #{map_size(sounding_station_map)} sounding") + +# --- Fetch weather for each QSO --- + +qsos = + Microwaveprop.Radio.Qso + |> where([q], not is_nil(q.pos1)) + |> order_by([q], q.qso_timestamp) + |> Repo.all() + +IO.puts("Processing #{length(qsos)} QSOs with positions...") + +# Track which station+time combos we've already fetched to avoid duplicates +asos_fetched = :ets.new(:asos_fetched, [:set, :public]) +raob_fetched = :ets.new(:raob_fetched, [:set, :public]) + +asos_station_list = Enum.map(asos_stations, fn s -> Map.put(s, :station_type, "asos") end) + +total = length(qsos) + +qsos +|> Enum.with_index(1) +|> Enum.each(fn {qso, idx} -> + lat = qso.pos1["lat"] || qso.pos1["latitude"] + lon = qso.pos1["lng"] || qso.pos1["lon"] || qso.pos1["longitude"] + + if lat && lon do + if rem(idx, 100) == 0, do: IO.puts(" QSO #{idx}/#{total}") + + # Find nearby ASOS stations (within 150km) + nearby_asos = ImportHelpers.nearest_stations(lat, lon, asos_station_list, 150) + + Enum.each(nearby_asos, fn s -> + # Fetch a 4-hour window around QSO time + start_dt = DateTime.add(qso.qso_timestamp, -2 * 3600, :second) + end_dt = DateTime.add(qso.qso_timestamp, 2 * 3600, :second) + key = {s.station_code, DateTime.to_date(start_dt), div(start_dt.hour, 4)} + + unless :ets.member(asos_fetched, key) do + :ets.insert(asos_fetched, {key, true}) + + station = Map.get(asos_station_map, s.station_code) + + if station do + case IemClient.fetch_asos(s.station_code, start_dt, end_dt) do + {:ok, rows} -> + Enum.each(rows, fn row -> + if row.observed_at do + Weather.upsert_surface_observation(station, row) + end + end) + + {:error, reason} -> + IO.puts(" ASOS error #{s.station_code}: #{inspect(reason)}") + end + + ImportHelpers.rate_limit() + end + end + end) + + # Find nearby sounding stations (within 300km) + nearby_soundings = ImportHelpers.nearest_stations(lat, lon, sounding_stations, 300) + + Enum.each(nearby_soundings, fn s -> + sounding_times = ImportHelpers.sounding_times_around(qso.qso_timestamp) + + Enum.each(sounding_times, fn sounding_time -> + key = {s.station_code, sounding_time} + + unless :ets.member(raob_fetched, key) do + :ets.insert(raob_fetched, {key, true}) + + station = Map.get(sounding_station_map, s.station_code) + + if station do + case IemClient.fetch_raob(s.station_code, sounding_time) do + {:ok, [parsed | _]} -> + params = SoundingParams.derive(parsed.profile) + + sounding_attrs = + if params do + %{ + observed_at: parsed.observed_at, + profile: parsed.profile, + level_count: params.level_count, + surface_pressure_mb: params.surface_pressure_mb, + surface_temp_c: params.surface_temp_c, + surface_dewpoint_c: params.surface_dewpoint_c, + surface_refractivity: params.surface_refractivity, + min_refractivity_gradient: params.min_refractivity_gradient, + boundary_layer_depth_m: params.boundary_layer_depth_m, + precipitable_water_mm: params.precipitable_water_mm, + k_index: params.k_index, + lifted_index: params.lifted_index, + ducting_detected: params.ducting_detected, + duct_characteristics: params.duct_characteristics + } + else + %{ + observed_at: parsed.observed_at, + profile: parsed.profile, + level_count: length(parsed.profile) + } + end + + Weather.upsert_sounding(station, sounding_attrs) + + {:ok, []} -> + :ok + + {:error, reason} -> + IO.puts(" RAOB error #{s.station_code}: #{inspect(reason)}") + end + + ImportHelpers.rate_limit() + end + end + end) + end) + end +end) + +:ets.delete(asos_fetched) +:ets.delete(raob_fetched) + +# --- Summary --- + +asos_count = Repo.aggregate(Microwaveprop.Weather.SurfaceObservation, :count) +sounding_count = Repo.aggregate(Microwaveprop.Weather.Sounding, :count) +station_count = Repo.aggregate(Microwaveprop.Weather.Station, :count) + +IO.puts(""" + +Import complete: + Stations: #{station_count} + Surface observations: #{asos_count} + Soundings: #{sounding_count} +""") diff --git a/priv/repo/migrations/20260328204648_create_weather_tables.exs b/priv/repo/migrations/20260328204648_create_weather_tables.exs new file mode 100644 index 00000000..a7f6cc12 --- /dev/null +++ b/priv/repo/migrations/20260328204648_create_weather_tables.exs @@ -0,0 +1,66 @@ +defmodule Microwaveprop.Repo.Migrations.CreateWeatherTables do + use Ecto.Migration + + def change do + create table(:weather_stations, primary_key: false) do + add :id, :binary_id, primary_key: true + add :station_code, :string, null: false + add :station_type, :string, null: false + add :wmo_number, :integer + add :name, :string, null: false + add :lat, :float, null: false + add :lon, :float, null: false + add :elevation_m, :float + add :state, :string + + timestamps(type: :utc_datetime) + end + + create unique_index(:weather_stations, [:station_code, :station_type]) + create index(:weather_stations, [:lat, :lon]) + + create table(:surface_observations, primary_key: false) do + add :id, :binary_id, primary_key: true + add :station_id, references(:weather_stations, type: :binary_id), null: false + add :observed_at, :utc_datetime, null: false + add :temp_f, :float + add :dewpoint_f, :float + add :relative_humidity, :float + add :wind_speed_kts, :float + add :wind_direction_deg, :integer + add :sea_level_pressure_mb, :float + add :altimeter_setting, :float + add :sky_condition, :string + + timestamps(type: :utc_datetime) + end + + create unique_index(:surface_observations, [:station_id, :observed_at]) + create index(:surface_observations, [:observed_at]) + + create table(:soundings, primary_key: false) do + add :id, :binary_id, primary_key: true + add :station_id, references(:weather_stations, type: :binary_id), null: false + add :observed_at, :utc_datetime, null: false + add :profile, :map, null: false + add :level_count, :integer, null: false + add :surface_pressure_mb, :float + add :surface_temp_c, :float + add :surface_dewpoint_c, :float + add :surface_refractivity, :float + add :min_refractivity_gradient, :float + add :boundary_layer_depth_m, :float + add :precipitable_water_mm, :float + add :k_index, :float + add :lifted_index, :float + add :ducting_detected, :boolean, null: false, default: false + add :duct_characteristics, :map + + timestamps(type: :utc_datetime) + end + + create unique_index(:soundings, [:station_id, :observed_at]) + create index(:soundings, [:observed_at]) + create index(:soundings, [:ducting_detected]) + end +end diff --git a/test/microwaveprop/weather/iem_client_test.exs b/test/microwaveprop/weather/iem_client_test.exs new file mode 100644 index 00000000..e491c13f --- /dev/null +++ b/test/microwaveprop/weather/iem_client_test.exs @@ -0,0 +1,119 @@ +defmodule Microwaveprop.Weather.IemClientTest do + use ExUnit.Case, async: true + + alias Microwaveprop.Weather.IemClient + + describe "parse_asos_csv/1" do + test "parses valid CSV rows" do + csv = """ + station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1 + KDFW,2026-03-28 18:53,75.0,55.0,49.12,12,180,1013.2,29.92,SCT + KDFW,2026-03-28 19:53,76.0,54.0,46.00,10,190,1013.0,29.91,FEW + """ + + rows = IemClient.parse_asos_csv(csv) + assert length(rows) == 2 + + first = hd(rows) + assert first.temp_f == 75.0 + assert first.dewpoint_f == 55.0 + assert first.relative_humidity == 49.12 + assert first.wind_speed_kts == 12.0 + assert first.wind_direction_deg == 180 + assert first.sea_level_pressure_mb == 1013.2 + assert first.altimeter_setting == 29.92 + assert first.sky_condition == "SCT" + assert first.observed_at == ~U[2026-03-28 18:53:00Z] + end + + test "handles null values in CSV" do + csv = """ + station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1 + KDFW,2026-03-28 18:53,null,null,null,null,null,null,null,null + """ + + rows = IemClient.parse_asos_csv(csv) + assert length(rows) == 1 + + first = hd(rows) + assert first.temp_f == nil + assert first.dewpoint_f == nil + assert first.sky_condition == nil + end + + test "skips comment lines and empty lines" do + csv = """ + #DEBUG something + station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1 + KDFW,2026-03-28 18:53,75.0,55.0,49.0,12,180,1013.2,29.92,SCT + + """ + + rows = IemClient.parse_asos_csv(csv) + assert length(rows) == 1 + end + end + + describe "parse_raob_json/1" do + test "parses valid RAOB JSON response" do + json = %{ + "profiles" => [ + %{ + "station" => "KFWD", + "valid" => "2026-03-28 12:00:00+00:00", + "profile" => [ + %{"pres" => 1013.0, "hght" => 171.0, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180.0, "sknt" => 10.0}, + %{"pres" => 925.0, "hght" => 800.0, "tmpc" => 20.0, "dwpc" => 12.0, "drct" => 200.0, "sknt" => 15.0} + ] + } + ] + } + + result = IemClient.parse_raob_json(json) + assert length(result) == 1 + + sounding = hd(result) + assert sounding.observed_at == ~U[2026-03-28 12:00:00Z] + assert length(sounding.profile) == 2 + assert hd(sounding.profile)["pres"] == 1013.0 + end + + test "returns empty list for empty profiles" do + json = %{"profiles" => []} + assert IemClient.parse_raob_json(json) == [] + end + + test "handles missing profile key gracefully" do + json = %{} + assert IemClient.parse_raob_json(json) == [] + end + end + + describe "asos_url/3" do + test "builds correct URL for date range" do + url = + IemClient.asos_url( + "KDFW", + ~U[2026-03-28 12:00:00Z], + ~U[2026-03-28 18:00:00Z] + ) + + assert url =~ "station=KDFW" + assert url =~ "year1=2026" + assert url =~ "month1=3" + assert url =~ "day1=28" + assert url =~ "hour1=12" + assert url =~ "year2=2026" + assert url =~ "hour2=18" + assert url =~ "format=onlycomma" + end + end + + describe "raob_url/2" do + test "builds correct URL for sounding fetch" do + url = IemClient.raob_url("FWD", ~U[2026-03-28 12:00:00Z]) + assert url =~ "station=FWD" + assert url =~ "ts=202603281200" + end + end +end diff --git a/test/microwaveprop/weather/sounding_params_test.exs b/test/microwaveprop/weather/sounding_params_test.exs new file mode 100644 index 00000000..a282fb08 --- /dev/null +++ b/test/microwaveprop/weather/sounding_params_test.exs @@ -0,0 +1,161 @@ +defmodule Microwaveprop.Weather.SoundingParamsTest do + use ExUnit.Case, async: true + + alias Microwaveprop.Weather.SoundingParams + + # Realistic 3-level profile (surface, 925, 850 hPa) + @simple_profile [ + %{"pres" => 1013.0, "hght" => 171, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180, "sknt" => 10}, + %{"pres" => 925.0, "hght" => 800, "tmpc" => 20.0, "dwpc" => 12.0, "drct" => 200, "sknt" => 15}, + %{"pres" => 850.0, "hght" => 1500, "tmpc" => 15.0, "dwpc" => 8.0, "drct" => 220, "sknt" => 20} + ] + + # Profile with a temperature inversion (temp increases with height in one layer) + @inversion_profile [ + %{"pres" => 1013.0, "hght" => 100, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180, "sknt" => 5}, + %{"pres" => 980.0, "hght" => 400, "tmpc" => 22.0, "dwpc" => 14.0, "drct" => 190, "sknt" => 8}, + %{"pres" => 950.0, "hght" => 700, "tmpc" => 26.0, "dwpc" => 10.0, "drct" => 200, "sknt" => 12}, + %{"pres" => 925.0, "hght" => 900, "tmpc" => 20.0, "dwpc" => 8.0, "drct" => 210, "sknt" => 15}, + %{"pres" => 850.0, "hght" => 1500, "tmpc" => 14.0, "dwpc" => 5.0, "drct" => 230, "sknt" => 20}, + %{"pres" => 700.0, "hght" => 3100, "tmpc" => 2.0, "dwpc" => -5.0, "drct" => 250, "sknt" => 30}, + %{"pres" => 500.0, "hght" => 5600, "tmpc" => -15.0, "dwpc" => -25.0, "drct" => 270, "sknt" => 40} + ] + + # Profile designed to create a duct (M decreases with height) + # Strong surface-based inversion with very dry air aloft → M decreases + @ducting_profile [ + %{"pres" => 1013.0, "hght" => 0, "tmpc" => 30.0, "dwpc" => 25.0, "drct" => 180, "sknt" => 5}, + %{"pres" => 1000.0, "hght" => 100, "tmpc" => 35.0, "dwpc" => 5.0, "drct" => 180, "sknt" => 5}, + %{"pres" => 980.0, "hght" => 300, "tmpc" => 28.0, "dwpc" => 10.0, "drct" => 190, "sknt" => 8}, + %{"pres" => 950.0, "hght" => 550, "tmpc" => 24.0, "dwpc" => 12.0, "drct" => 200, "sknt" => 10}, + %{"pres" => 925.0, "hght" => 800, "tmpc" => 20.0, "dwpc" => 10.0, "drct" => 210, "sknt" => 12}, + %{"pres" => 850.0, "hght" => 1500, "tmpc" => 14.0, "dwpc" => 5.0, "drct" => 230, "sknt" => 18}, + %{"pres" => 700.0, "hght" => 3100, "tmpc" => 2.0, "dwpc" => -10.0, "drct" => 250, "sknt" => 25}, + %{"pres" => 500.0, "hght" => 5600, "tmpc" => -18.0, "dwpc" => -30.0, "drct" => 270, "sknt" => 35} + ] + + describe "sat_vap_pres/1" do + test "returns known value at 20°C (~23.4 hPa)" do + result = SoundingParams.sat_vap_pres(20.0) + assert_in_delta result, 23.4, 0.5 + end + + test "returns known value at 0°C (~6.1 hPa)" do + result = SoundingParams.sat_vap_pres(0.0) + assert_in_delta result, 6.1, 0.2 + end + end + + describe "mixing_ratio/2" do + test "returns reasonable value for typical surface conditions" do + # At 15°C, 1013 hPa, mixing ratio ~10.7 g/kg + result = SoundingParams.mixing_ratio(15.0, 1013.0) + assert_in_delta result, 10.7, 1.0 + end + end + + describe "derive/1" do + test "returns nil for profiles with fewer than 3 valid levels" do + short = [ + %{"pres" => 1013.0, "hght" => 171, "tmpc" => 25.0, "dwpc" => 15.0}, + %{"pres" => 925.0, "hght" => 800, "tmpc" => 20.0, "dwpc" => 12.0} + ] + + assert SoundingParams.derive(short) == nil + end + + test "returns nil for empty profile" do + assert SoundingParams.derive([]) == nil + end + + test "computes surface parameters" do + result = SoundingParams.derive(@simple_profile) + assert result + assert_in_delta result.surface_temp_c, 25.0, 0.01 + assert_in_delta result.surface_dewpoint_c, 15.0, 0.01 + assert_in_delta result.surface_pressure_mb, 1013.0, 0.01 + end + + test "computes surface refractivity (N-units)" do + result = SoundingParams.derive(@simple_profile) + # At 25°C, 15°C dewpoint, 1013 hPa — N should be ~330-350 + assert result.surface_refractivity > 300 + assert result.surface_refractivity < 400 + end + + test "computes dN/dh gradients" do + result = SoundingParams.derive(@simple_profile) + # Standard atmosphere dN/dh is about -40/km + assert result.min_refractivity_gradient < 0 + end + + test "detects temperature inversions" do + result = SoundingParams.derive(@inversion_profile) + assert length(result.inversions) > 0 + + inv = hd(result.inversions) + assert inv.strength > 0 + assert inv.base >= 0 + end + + test "no inversions in standard lapse profile" do + # Simple profile has monotonic temperature decrease + result = SoundingParams.derive(@simple_profile) + assert result.inversions == [] + end + + test "computes precipitable water" do + result = SoundingParams.derive(@inversion_profile) + # PW should be positive and reasonable (5-80mm) + assert result.precipitable_water_mm > 0 + assert result.precipitable_water_mm < 100 + end + + test "computes K-Index when standard levels available" do + result = SoundingParams.derive(@inversion_profile) + # K-Index should be computed (profile has 850, 700, 500 levels) + assert result.k_index + end + + test "computes Lifted Index when 500 hPa available" do + result = SoundingParams.derive(@inversion_profile) + assert result.lifted_index + end + + test "detects ducting in strong inversion profile" do + result = SoundingParams.derive(@ducting_profile) + assert result.ducting_detected == true + assert length(result.ducts) > 0 + end + + test "no ducting in standard profile" do + result = SoundingParams.derive(@simple_profile) + assert result.ducting_detected == false + assert result.ducts == [] + end + + test "computes boundary layer depth" do + result = SoundingParams.derive(@inversion_profile) + # BL depth should be positive and reasonable + assert result.boundary_layer_depth_m > 0 + assert result.boundary_layer_depth_m < 5000 + end + + test "returns level_count matching filtered profile" do + result = SoundingParams.derive(@simple_profile) + assert result.level_count == 3 + end + + test "filters out levels with nil pressure, temp, or height" do + profile_with_nils = [ + %{"pres" => 1013.0, "hght" => 171, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180, "sknt" => 10}, + %{"pres" => nil, "hght" => 500, "tmpc" => 22.0, "dwpc" => 13.0, "drct" => 190, "sknt" => 12}, + %{"pres" => 925.0, "hght" => 800, "tmpc" => 20.0, "dwpc" => 12.0, "drct" => 200, "sknt" => 15}, + %{"pres" => 850.0, "hght" => 1500, "tmpc" => 15.0, "dwpc" => 8.0, "drct" => 220, "sknt" => 20} + ] + + result = SoundingParams.derive(profile_with_nils) + assert result.level_count == 3 + end + end +end diff --git a/test/microwaveprop/weather/sounding_test.exs b/test/microwaveprop/weather/sounding_test.exs new file mode 100644 index 00000000..e3a65859 --- /dev/null +++ b/test/microwaveprop/weather/sounding_test.exs @@ -0,0 +1,115 @@ +defmodule Microwaveprop.Weather.SoundingTest do + use Microwaveprop.DataCase, async: true + + alias Microwaveprop.Weather.Sounding + alias Microwaveprop.Weather.Station + + defp create_sounding_station do + %Station{} + |> Station.changeset(%{ + station_code: "FWD", + station_type: "sounding", + wmo_number: 72_249, + name: "Fort Worth", + lat: 32.83, + lon: -97.30 + }) + |> Repo.insert!() + end + + @sample_profile [ + %{"pres" => 1013.0, "hght" => 171, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180, "sknt" => 10}, + %{"pres" => 925.0, "hght" => 800, "tmpc" => 20.0, "dwpc" => 12.0, "drct" => 200, "sknt" => 15}, + %{"pres" => 850.0, "hght" => 1500, "tmpc" => 15.0, "dwpc" => 8.0, "drct" => 220, "sknt" => 20} + ] + + @valid_sounding_attrs %{ + observed_at: ~U[2026-03-28 12:00:00Z], + profile: @sample_profile, + level_count: 3, + surface_pressure_mb: 1013.0, + surface_temp_c: 25.0, + surface_dewpoint_c: 15.0, + surface_refractivity: 320.5, + min_refractivity_gradient: -45.0, + boundary_layer_depth_m: 1200.0, + precipitable_water_mm: 25.0, + k_index: 28.0, + lifted_index: -2.0, + ducting_detected: false, + duct_characteristics: nil + } + + describe "changeset/2" do + test "valid attributes with station_id" do + station = create_sounding_station() + attrs = Map.put(@valid_sounding_attrs, :station_id, station.id) + changeset = Sounding.changeset(%Sounding{}, attrs) + assert changeset.valid? + end + + test "requires station_id, observed_at, profile, level_count" do + changeset = Sounding.changeset(%Sounding{}, %{}) + + assert %{ + station_id: ["can't be blank"], + observed_at: ["can't be blank"], + profile: ["can't be blank"], + level_count: ["can't be blank"] + } = errors_on(changeset) + end + + test "derived params are optional" do + station = create_sounding_station() + + attrs = %{ + station_id: station.id, + observed_at: ~U[2026-03-28 12:00:00Z], + profile: @sample_profile, + level_count: 3 + } + + changeset = Sounding.changeset(%Sounding{}, attrs) + assert changeset.valid? + end + + test "persists to the database with JSONB profile" do + station = create_sounding_station() + attrs = Map.put(@valid_sounding_attrs, :station_id, station.id) + changeset = Sounding.changeset(%Sounding{}, attrs) + assert {:ok, sounding} = Repo.insert(changeset) + assert length(sounding.profile) == 3 + assert hd(sounding.profile)["pres"] == 1013.0 + assert sounding.ducting_detected == false + assert sounding.k_index == 28.0 + end + + test "persists duct_characteristics as JSONB" do + station = create_sounding_station() + + duct_chars = [ + %{"base" => 200, "top" => 500, "strength" => "5.2"} + ] + + attrs = + @valid_sounding_attrs + |> Map.put(:station_id, station.id) + |> Map.put(:ducting_detected, true) + |> Map.put(:duct_characteristics, duct_chars) + + changeset = Sounding.changeset(%Sounding{}, attrs) + assert {:ok, sounding} = Repo.insert(changeset) + assert sounding.ducting_detected == true + assert [%{"base" => 200, "top" => 500}] = sounding.duct_characteristics + end + + test "enforces unique station_id + observed_at" do + station = create_sounding_station() + attrs = Map.put(@valid_sounding_attrs, :station_id, station.id) + + assert {:ok, _} = %Sounding{} |> Sounding.changeset(attrs) |> Repo.insert() + assert {:error, changeset} = %Sounding{} |> Sounding.changeset(attrs) |> Repo.insert() + assert %{station_id: ["has already been taken"]} = errors_on(changeset) + end + end +end diff --git a/test/microwaveprop/weather/station_test.exs b/test/microwaveprop/weather/station_test.exs new file mode 100644 index 00000000..22663d69 --- /dev/null +++ b/test/microwaveprop/weather/station_test.exs @@ -0,0 +1,71 @@ +defmodule Microwaveprop.Weather.StationTest do + use Microwaveprop.DataCase, async: true + + alias Microwaveprop.Weather.Station + + @valid_attrs %{ + station_code: "KDFW", + station_type: "asos", + name: "Dallas/Fort Worth International", + lat: 32.8998, + lon: -97.0403, + elevation_m: 171.0, + state: "TX" + } + + describe "changeset/2" do + test "valid attributes" do + changeset = Station.changeset(%Station{}, @valid_attrs) + assert changeset.valid? + end + + test "valid sounding station with wmo_number" do + attrs = %{ + station_code: "FWD", + station_type: "sounding", + wmo_number: 72_249, + name: "Fort Worth", + lat: 32.83, + lon: -97.30 + } + + changeset = Station.changeset(%Station{}, attrs) + assert changeset.valid? + end + + test "requires station_code, station_type, name, lat, lon" do + changeset = Station.changeset(%Station{}, %{}) + + assert %{ + station_code: ["can't be blank"], + station_type: ["can't be blank"], + name: ["can't be blank"], + lat: ["can't be blank"], + lon: ["can't be blank"] + } = errors_on(changeset) + end + + test "validates station_type inclusion" do + changeset = Station.changeset(%Station{}, %{@valid_attrs | station_type: "radar"}) + assert %{station_type: ["is invalid"]} = errors_on(changeset) + end + + test "persists to the database" do + changeset = Station.changeset(%Station{}, @valid_attrs) + assert {:ok, station} = Repo.insert(changeset) + assert station.station_code == "KDFW" + assert station.station_type == "asos" + assert station.lat == 32.8998 + assert station.lon == -97.0403 + end + + test "enforces unique station_code + station_type" do + changeset = Station.changeset(%Station{}, @valid_attrs) + assert {:ok, _} = Repo.insert(changeset) + + duplicate = Station.changeset(%Station{}, @valid_attrs) + assert {:error, changeset} = Repo.insert(duplicate) + assert %{station_code: ["has already been taken"]} = errors_on(changeset) + end + end +end diff --git a/test/microwaveprop/weather/surface_observation_test.exs b/test/microwaveprop/weather/surface_observation_test.exs new file mode 100644 index 00000000..6e449fc9 --- /dev/null +++ b/test/microwaveprop/weather/surface_observation_test.exs @@ -0,0 +1,79 @@ +defmodule Microwaveprop.Weather.SurfaceObservationTest do + use Microwaveprop.DataCase, async: true + + alias Microwaveprop.Weather.Station + alias Microwaveprop.Weather.SurfaceObservation + + defp create_station do + %Station{} + |> Station.changeset(%{ + station_code: "KDFW", + station_type: "asos", + name: "Dallas/Fort Worth International", + lat: 32.8998, + lon: -97.0403 + }) + |> Repo.insert!() + end + + @valid_obs_attrs %{ + observed_at: ~U[2026-03-28 18:00:00Z], + temp_f: 75.0, + dewpoint_f: 55.0, + relative_humidity: 49.0, + wind_speed_kts: 12.0, + wind_direction_deg: 180, + sea_level_pressure_mb: 1013.25, + altimeter_setting: 29.92, + sky_condition: "SCT" + } + + describe "changeset/2" do + test "valid attributes with station_id" do + station = create_station() + attrs = Map.put(@valid_obs_attrs, :station_id, station.id) + changeset = SurfaceObservation.changeset(%SurfaceObservation{}, attrs) + assert changeset.valid? + end + + test "requires station_id and observed_at" do + changeset = SurfaceObservation.changeset(%SurfaceObservation{}, %{}) + + assert %{ + station_id: ["can't be blank"], + observed_at: ["can't be blank"] + } = errors_on(changeset) + end + + test "all weather fields are optional" do + station = create_station() + + attrs = %{ + station_id: station.id, + observed_at: ~U[2026-03-28 18:00:00Z] + } + + changeset = SurfaceObservation.changeset(%SurfaceObservation{}, attrs) + assert changeset.valid? + end + + test "persists to the database" do + station = create_station() + attrs = Map.put(@valid_obs_attrs, :station_id, station.id) + changeset = SurfaceObservation.changeset(%SurfaceObservation{}, attrs) + assert {:ok, obs} = Repo.insert(changeset) + assert obs.temp_f == 75.0 + assert obs.sky_condition == "SCT" + assert obs.station_id == station.id + end + + test "enforces unique station_id + observed_at" do + station = create_station() + attrs = Map.put(@valid_obs_attrs, :station_id, station.id) + + assert {:ok, _} = %SurfaceObservation{} |> SurfaceObservation.changeset(attrs) |> Repo.insert() + assert {:error, changeset} = %SurfaceObservation{} |> SurfaceObservation.changeset(attrs) |> Repo.insert() + assert %{station_id: ["has already been taken"]} = errors_on(changeset) + end + end +end diff --git a/test/microwaveprop/weather_test.exs b/test/microwaveprop/weather_test.exs new file mode 100644 index 00000000..e8bd17f3 --- /dev/null +++ b/test/microwaveprop/weather_test.exs @@ -0,0 +1,199 @@ +defmodule Microwaveprop.WeatherTest do + use Microwaveprop.DataCase, async: true + + alias Microwaveprop.Weather + + @station_attrs %{ + station_code: "KDFW", + station_type: "asos", + name: "Dallas/Fort Worth International", + lat: 32.8998, + lon: -97.0403, + elevation_m: 171.0, + state: "TX" + } + + @sounding_station_attrs %{ + station_code: "FWD", + station_type: "sounding", + wmo_number: 72_249, + name: "Fort Worth", + lat: 32.83, + lon: -97.30 + } + + describe "find_or_create_station/1" do + test "creates a new station" do + assert {:ok, station} = Weather.find_or_create_station(@station_attrs) + assert station.station_code == "KDFW" + assert station.station_type == "asos" + end + + test "returns existing station if already created" do + {:ok, first} = Weather.find_or_create_station(@station_attrs) + {:ok, second} = Weather.find_or_create_station(@station_attrs) + assert first.id == second.id + end + + test "returns error for invalid attributes" do + assert {:error, _changeset} = Weather.find_or_create_station(%{station_code: "X"}) + end + end + + describe "upsert_surface_observation/2" do + test "inserts a new observation" do + {:ok, station} = Weather.find_or_create_station(@station_attrs) + + obs_attrs = %{ + observed_at: ~U[2026-03-28 18:00:00Z], + temp_f: 75.0, + dewpoint_f: 55.0, + sky_condition: "SCT" + } + + assert {:ok, obs} = Weather.upsert_surface_observation(station, obs_attrs) + assert obs.temp_f == 75.0 + assert obs.station_id == station.id + end + + test "updates existing observation on conflict" do + {:ok, station} = Weather.find_or_create_station(@station_attrs) + + obs_attrs = %{ + observed_at: ~U[2026-03-28 18:00:00Z], + temp_f: 75.0, + sky_condition: "SCT" + } + + {:ok, first} = Weather.upsert_surface_observation(station, obs_attrs) + {:ok, second} = Weather.upsert_surface_observation(station, %{obs_attrs | temp_f: 78.0}) + + assert first.id == second.id + assert second.temp_f == 78.0 + end + end + + describe "upsert_sounding/2" do + @sample_profile [ + %{"pres" => 1013.0, "hght" => 171, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180, "sknt" => 10}, + %{"pres" => 925.0, "hght" => 800, "tmpc" => 20.0, "dwpc" => 12.0, "drct" => 200, "sknt" => 15}, + %{"pres" => 850.0, "hght" => 1500, "tmpc" => 15.0, "dwpc" => 8.0, "drct" => 220, "sknt" => 20} + ] + + test "inserts a new sounding" do + {:ok, station} = Weather.find_or_create_station(@sounding_station_attrs) + + sounding_attrs = %{ + observed_at: ~U[2026-03-28 12:00:00Z], + profile: @sample_profile, + level_count: 3, + surface_temp_c: 25.0 + } + + assert {:ok, sounding} = Weather.upsert_sounding(station, sounding_attrs) + assert sounding.level_count == 3 + assert sounding.station_id == station.id + end + + test "updates existing sounding on conflict" do + {:ok, station} = Weather.find_or_create_station(@sounding_station_attrs) + + sounding_attrs = %{ + observed_at: ~U[2026-03-28 12:00:00Z], + profile: @sample_profile, + level_count: 3, + surface_temp_c: 25.0 + } + + {:ok, first} = Weather.upsert_sounding(station, sounding_attrs) + {:ok, second} = Weather.upsert_sounding(station, %{sounding_attrs | surface_temp_c: 26.0}) + + assert first.id == second.id + assert second.surface_temp_c == 26.0 + end + end + + describe "weather_for_qso/2" do + test "returns nearby surface observations within time window" do + {:ok, station} = Weather.find_or_create_station(@station_attrs) + + {:ok, _obs} = + Weather.upsert_surface_observation(station, %{ + observed_at: ~U[2026-03-28 18:00:00Z], + temp_f: 75.0 + }) + + # QSO near the station in space and time + result = + Weather.weather_for_qso( + %{lat: 32.90, lon: -97.05, timestamp: ~U[2026-03-28 18:30:00Z]}, + radius_km: 100, + time_window_hours: 3 + ) + + assert length(result.surface_observations) == 1 + assert hd(result.surface_observations).temp_f == 75.0 + end + + test "returns nearby soundings within time window" do + {:ok, station} = Weather.find_or_create_station(@sounding_station_attrs) + + {:ok, _sounding} = + Weather.upsert_sounding(station, %{ + observed_at: ~U[2026-03-28 12:00:00Z], + profile: [%{"pres" => 1013.0, "hght" => 171, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180, "sknt" => 10}], + level_count: 1, + surface_temp_c: 25.0 + }) + + result = + Weather.weather_for_qso( + %{lat: 32.85, lon: -97.28, timestamp: ~U[2026-03-28 14:00:00Z]}, + radius_km: 100, + time_window_hours: 6 + ) + + assert length(result.soundings) == 1 + end + + test "excludes observations outside radius" do + {:ok, station} = Weather.find_or_create_station(@station_attrs) + + {:ok, _obs} = + Weather.upsert_surface_observation(station, %{ + observed_at: ~U[2026-03-28 18:00:00Z], + temp_f: 75.0 + }) + + # QSO far away + result = + Weather.weather_for_qso( + %{lat: 40.0, lon: -80.0, timestamp: ~U[2026-03-28 18:00:00Z]}, + radius_km: 100, + time_window_hours: 3 + ) + + assert result.surface_observations == [] + end + + test "excludes observations outside time window" do + {:ok, station} = Weather.find_or_create_station(@station_attrs) + + {:ok, _obs} = + Weather.upsert_surface_observation(station, %{ + observed_at: ~U[2026-03-28 06:00:00Z], + temp_f: 75.0 + }) + + # QSO much later + result = + Weather.weather_for_qso( + %{lat: 32.90, lon: -97.05, timestamp: ~U[2026-03-28 22:00:00Z]}, + radius_km: 100, + time_window_hours: 3 + ) + + assert result.surface_observations == [] + end + end +end