Add weather data schema, context, sounding params, and IEM ingestion
Store surface observations (ASOS) and upper-air soundings (RAOB) alongside QSOs for atmospheric propagation correlation. Three new tables: weather_stations, surface_observations, and soundings with JSONB profiles and pre-computed derived parameters (refractivity, gradients, duct detection, stability indices). Includes IEM API client for historical data import and import script seeded with 95 ASOS + 9 sounding stations from PropCast coverage area.
This commit is contained in:
parent
e0ab6560d0
commit
6489f08138
14 changed files with 1760 additions and 0 deletions
97
lib/microwaveprop/weather.ex
Normal file
97
lib/microwaveprop/weather.ex
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
defmodule Microwaveprop.Weather do
|
||||
@moduledoc false
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Weather.Sounding
|
||||
alias Microwaveprop.Weather.Station
|
||||
alias Microwaveprop.Weather.SurfaceObservation
|
||||
|
||||
# Approximate km per degree latitude
|
||||
@km_per_deg_lat 111.0
|
||||
|
||||
def find_or_create_station(attrs) do
|
||||
code = attrs[:station_code] || attrs["station_code"]
|
||||
type = attrs[:station_type] || attrs["station_type"]
|
||||
|
||||
if code && type do
|
||||
case Repo.get_by(Station, station_code: code, station_type: type) do
|
||||
nil ->
|
||||
%Station{}
|
||||
|> Station.changeset(attrs)
|
||||
|> Repo.insert()
|
||||
|
||||
station ->
|
||||
{:ok, station}
|
||||
end
|
||||
else
|
||||
%Station{}
|
||||
|> Station.changeset(attrs)
|
||||
|> Repo.insert()
|
||||
end
|
||||
end
|
||||
|
||||
def upsert_surface_observation(%Station{} = station, attrs) do
|
||||
attrs = Map.put(attrs, :station_id, station.id)
|
||||
|
||||
%SurfaceObservation{}
|
||||
|> SurfaceObservation.changeset(attrs)
|
||||
|> Repo.insert(
|
||||
on_conflict: {:replace_all_except, [:id, :station_id, :observed_at, :inserted_at]},
|
||||
conflict_target: [:station_id, :observed_at],
|
||||
returning: true
|
||||
)
|
||||
end
|
||||
|
||||
def upsert_sounding(%Station{} = station, attrs) do
|
||||
attrs = Map.put(attrs, :station_id, station.id)
|
||||
|
||||
%Sounding{}
|
||||
|> Sounding.changeset(attrs)
|
||||
|> Repo.insert(
|
||||
on_conflict: {:replace_all_except, [:id, :station_id, :observed_at, :inserted_at]},
|
||||
conflict_target: [:station_id, :observed_at],
|
||||
returning: true
|
||||
)
|
||||
end
|
||||
|
||||
def weather_for_qso(qso_params, opts \\ []) do
|
||||
lat = qso_params[:lat] || qso_params.lat
|
||||
lon = qso_params[:lon] || qso_params.lon
|
||||
timestamp = qso_params[:timestamp] || qso_params.timestamp
|
||||
|
||||
radius_km = Keyword.get(opts, :radius_km, 150)
|
||||
time_window_hours = Keyword.get(opts, :time_window_hours, 6)
|
||||
|
||||
# Bounding box in degrees
|
||||
dlat = radius_km / @km_per_deg_lat
|
||||
dlon = radius_km / (@km_per_deg_lat * :math.cos(lat * :math.pi() / 180))
|
||||
|
||||
time_start = DateTime.add(timestamp, -time_window_hours * 3600, :second)
|
||||
time_end = DateTime.add(timestamp, time_window_hours * 3600, :second)
|
||||
|
||||
station_ids =
|
||||
Station
|
||||
|> where(
|
||||
[s],
|
||||
s.lat >= ^(lat - dlat) and s.lat <= ^(lat + dlat) and
|
||||
s.lon >= ^(lon - dlon) and s.lon <= ^(lon + dlon)
|
||||
)
|
||||
|> select([s], s.id)
|
||||
|
||||
surface_observations =
|
||||
SurfaceObservation
|
||||
|> where([o], o.station_id in subquery(station_ids))
|
||||
|> where([o], o.observed_at >= ^time_start and o.observed_at <= ^time_end)
|
||||
|> Repo.all()
|
||||
|
||||
soundings =
|
||||
Sounding
|
||||
|> where([s], s.station_id in subquery(station_ids))
|
||||
|> where([s], s.observed_at >= ^time_start and s.observed_at <= ^time_end)
|
||||
|> Repo.all()
|
||||
|
||||
%{surface_observations: surface_observations, soundings: soundings}
|
||||
end
|
||||
end
|
||||
159
lib/microwaveprop/weather/iem_client.ex
Normal file
159
lib/microwaveprop/weather/iem_client.ex
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
defmodule Microwaveprop.Weather.IemClient do
|
||||
@moduledoc false
|
||||
|
||||
@iem_base "https://mesonet.agron.iastate.edu"
|
||||
|
||||
# --- URL builders ---
|
||||
|
||||
def asos_url(station_id, start_dt, end_dt) do
|
||||
"#{@iem_base}/cgi-bin/request/asos.py" <>
|
||||
"?station=#{station_id}" <>
|
||||
"&data=tmpf&data=dwpf&data=relh&data=sknt&data=drct" <>
|
||||
"&data=mslp&data=alti&data=skyc1" <>
|
||||
"&year1=#{start_dt.year}&month1=#{start_dt.month}&day1=#{start_dt.day}" <>
|
||||
"&hour1=#{start_dt.hour}&minute1=0" <>
|
||||
"&year2=#{end_dt.year}&month2=#{end_dt.month}&day2=#{end_dt.day}" <>
|
||||
"&hour2=#{end_dt.hour}&minute2=59" <>
|
||||
"&format=onlycomma&latlon=no&elev=no&missing=null&trace=null&direct=no&report_type=3"
|
||||
end
|
||||
|
||||
def raob_url(station_id, dt) do
|
||||
ts = format_iem_ts(dt)
|
||||
"#{@iem_base}/json/raob.py?ts=#{ts}&station=#{station_id}"
|
||||
end
|
||||
|
||||
# --- HTTP fetchers ---
|
||||
|
||||
def fetch_asos(station_id, start_dt, end_dt) do
|
||||
url = asos_url(station_id, start_dt, end_dt)
|
||||
|
||||
case Req.get(url) do
|
||||
{:ok, %{status: 200, body: body}} ->
|
||||
{:ok, parse_asos_csv(body)}
|
||||
|
||||
{:ok, %{status: status}} ->
|
||||
{:error, "IEM ASOS HTTP #{status}"}
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_raob(station_id, dt) do
|
||||
url = raob_url(station_id, dt)
|
||||
|
||||
case Req.get(url) do
|
||||
{:ok, %{status: 200, body: body}} ->
|
||||
{:ok, parse_raob_json(body)}
|
||||
|
||||
{:ok, %{status: status}} ->
|
||||
{:error, "IEM RAOB HTTP #{status}"}
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
# --- Parsers ---
|
||||
|
||||
def parse_asos_csv(csv_text) do
|
||||
csv_text
|
||||
|> String.split("\n")
|
||||
|> Enum.reject(fn line ->
|
||||
line == "" or String.starts_with?(line, "#") or String.starts_with?(line, "station")
|
||||
end)
|
||||
|> Enum.map(&parse_asos_row/1)
|
||||
end
|
||||
|
||||
def parse_raob_json(json) when is_map(json) do
|
||||
json
|
||||
|> Map.get("profiles", [])
|
||||
|> Enum.map(fn entry ->
|
||||
%{
|
||||
observed_at: parse_raob_timestamp(entry["valid"]),
|
||||
profile: entry["profile"] || []
|
||||
}
|
||||
end)
|
||||
end
|
||||
|
||||
# --- Private ---
|
||||
|
||||
defp parse_asos_row(line) do
|
||||
parts = String.split(line, ",")
|
||||
|
||||
%{
|
||||
observed_at: parse_asos_timestamp(Enum.at(parts, 1)),
|
||||
temp_f: parse_float(Enum.at(parts, 2)),
|
||||
dewpoint_f: parse_float(Enum.at(parts, 3)),
|
||||
relative_humidity: parse_float(Enum.at(parts, 4)),
|
||||
wind_speed_kts: parse_float(Enum.at(parts, 5)),
|
||||
wind_direction_deg: parse_int(Enum.at(parts, 6)),
|
||||
sea_level_pressure_mb: parse_float(Enum.at(parts, 7)),
|
||||
altimeter_setting: parse_float(Enum.at(parts, 8)),
|
||||
sky_condition: parse_nullable_string(Enum.at(parts, 9))
|
||||
}
|
||||
end
|
||||
|
||||
defp parse_asos_timestamp(str) when is_binary(str) do
|
||||
# Format: "2026-03-28 18:53"
|
||||
case DateTime.from_iso8601(String.trim(str) <> ":00Z") do
|
||||
{:ok, dt, _} -> DateTime.truncate(dt, :second)
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_asos_timestamp(_), do: nil
|
||||
|
||||
defp parse_raob_timestamp(str) when is_binary(str) do
|
||||
# Format: "2026-03-28 12:00:00+00:00"
|
||||
cleaned =
|
||||
str
|
||||
|> String.trim()
|
||||
|> String.replace(~r/\+00:00$/, "Z")
|
||||
|> String.replace(" ", "T")
|
||||
|
||||
case DateTime.from_iso8601(cleaned) do
|
||||
{:ok, dt, _} -> DateTime.truncate(dt, :second)
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_raob_timestamp(_), do: nil
|
||||
|
||||
defp parse_float(nil), do: nil
|
||||
defp parse_float("null"), do: nil
|
||||
|
||||
defp parse_float(str) do
|
||||
case Float.parse(String.trim(str)) do
|
||||
{val, _} -> val
|
||||
:error -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_int(nil), do: nil
|
||||
defp parse_int("null"), do: nil
|
||||
|
||||
defp parse_int(str) do
|
||||
case Integer.parse(String.trim(str)) do
|
||||
{val, _} -> val
|
||||
:error -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_nullable_string(nil), do: nil
|
||||
defp parse_nullable_string("null"), do: nil
|
||||
|
||||
defp parse_nullable_string(str) do
|
||||
trimmed = String.trim(str)
|
||||
if trimmed == "", do: nil, else: trimmed
|
||||
end
|
||||
|
||||
defp format_iem_ts(dt) do
|
||||
y = Integer.to_string(dt.year)
|
||||
mo = dt.month |> Integer.to_string() |> String.pad_leading(2, "0")
|
||||
d = dt.day |> Integer.to_string() |> String.pad_leading(2, "0")
|
||||
h = dt.hour |> Integer.to_string() |> String.pad_leading(2, "0")
|
||||
mi = dt.minute |> Integer.to_string() |> String.pad_leading(2, "0")
|
||||
"#{y}#{mo}#{d}#{h}#{mi}"
|
||||
end
|
||||
end
|
||||
40
lib/microwaveprop/weather/sounding.ex
Normal file
40
lib/microwaveprop/weather/sounding.ex
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
defmodule Microwaveprop.Weather.Sounding do
|
||||
@moduledoc false
|
||||
use Ecto.Schema
|
||||
|
||||
import Ecto.Changeset
|
||||
|
||||
@primary_key {:id, :binary_id, autogenerate: true}
|
||||
@foreign_key_type :binary_id
|
||||
|
||||
schema "soundings" do
|
||||
belongs_to :station, Microwaveprop.Weather.Station
|
||||
field :observed_at, :utc_datetime
|
||||
field :profile, {:array, :map}
|
||||
field :level_count, :integer
|
||||
field :surface_pressure_mb, :float
|
||||
field :surface_temp_c, :float
|
||||
field :surface_dewpoint_c, :float
|
||||
field :surface_refractivity, :float
|
||||
field :min_refractivity_gradient, :float
|
||||
field :boundary_layer_depth_m, :float
|
||||
field :precipitable_water_mm, :float
|
||||
field :k_index, :float
|
||||
field :lifted_index, :float
|
||||
field :ducting_detected, :boolean, default: false
|
||||
field :duct_characteristics, {:array, :map}
|
||||
|
||||
timestamps(type: :utc_datetime)
|
||||
end
|
||||
|
||||
@required_fields ~w(station_id observed_at profile level_count)a
|
||||
@optional_fields ~w(surface_pressure_mb surface_temp_c surface_dewpoint_c surface_refractivity min_refractivity_gradient boundary_layer_depth_m precipitable_water_mm k_index lifted_index ducting_detected duct_characteristics)a
|
||||
|
||||
def changeset(sounding, attrs) do
|
||||
sounding
|
||||
|> cast(attrs, @required_fields ++ @optional_fields)
|
||||
|> validate_required(@required_fields)
|
||||
|> foreign_key_constraint(:station_id)
|
||||
|> unique_constraint([:station_id, :observed_at])
|
||||
end
|
||||
end
|
||||
266
lib/microwaveprop/weather/sounding_params.ex
Normal file
266
lib/microwaveprop/weather/sounding_params.ex
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
defmodule Microwaveprop.Weather.SoundingParams do
|
||||
@moduledoc false
|
||||
|
||||
@doc "Buck equation for saturation vapor pressure (hPa) given temperature in °C."
|
||||
def sat_vap_pres(t_c) do
|
||||
6.1121 * :math.exp((18.678 - t_c / 234.5) * (t_c / (257.14 + t_c)))
|
||||
end
|
||||
|
||||
@doc "Mixing ratio (g/kg) from dewpoint (°C) and pressure (hPa)."
|
||||
def mixing_ratio(t_c, p_mb) do
|
||||
e = sat_vap_pres(t_c)
|
||||
622.0 * e / (p_mb - e)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Derive atmospheric propagation parameters from a sounding profile.
|
||||
|
||||
Profile is a list of maps with keys: "pres", "hght", "tmpc", "dwpc", "drct", "sknt".
|
||||
Returns nil if fewer than 3 valid levels.
|
||||
"""
|
||||
def derive(profile) when is_list(profile) do
|
||||
sorted =
|
||||
profile
|
||||
|> Enum.filter(fn p -> p["pres"] != nil and p["tmpc"] != nil and p["hght"] != nil end)
|
||||
|> Enum.sort_by(fn p -> p["pres"] end, :desc)
|
||||
|
||||
if length(sorted) < 3 do
|
||||
nil
|
||||
else
|
||||
do_derive(sorted)
|
||||
end
|
||||
end
|
||||
|
||||
defp do_derive(sorted) do
|
||||
sfc = hd(sorted)
|
||||
sfc_hght = sfc["hght"]
|
||||
|
||||
refract_profile = compute_refractivity_profile(sorted, sfc_hght)
|
||||
dn_dh = compute_gradients(refract_profile)
|
||||
inversions = detect_inversions(sorted, sfc_hght)
|
||||
ducts = detect_ducts(refract_profile)
|
||||
k_index = compute_k_index(sorted)
|
||||
li = compute_lifted_index(sorted, sfc)
|
||||
pw = compute_precipitable_water(sorted)
|
||||
bl_depth = compute_boundary_layer_depth(sorted, sfc)
|
||||
min_grad = find_min_gradient(dn_dh)
|
||||
|
||||
sfc_n =
|
||||
case refract_profile do
|
||||
[first | _] -> first.n
|
||||
_ -> nil
|
||||
end
|
||||
|
||||
%{
|
||||
level_count: length(sorted),
|
||||
surface_pressure_mb: sfc["pres"],
|
||||
surface_temp_c: sfc["tmpc"],
|
||||
surface_dewpoint_c: sfc["dwpc"] || sfc["tmpc"] - 10,
|
||||
surface_refractivity: sfc_n,
|
||||
min_refractivity_gradient: min_grad,
|
||||
boundary_layer_depth_m: bl_depth,
|
||||
precipitable_water_mm: pw,
|
||||
k_index: k_index,
|
||||
lifted_index: li,
|
||||
ducting_detected: ducts != [],
|
||||
ducts: ducts,
|
||||
duct_characteristics: if(ducts == [], do: nil, else: ducts),
|
||||
inversions: inversions,
|
||||
profile: sorted
|
||||
}
|
||||
end
|
||||
|
||||
defp compute_refractivity_profile(sorted, sfc_hght) do
|
||||
sorted
|
||||
|> Enum.filter(fn p -> p["dwpc"] != nil end)
|
||||
|> Enum.map(fn p ->
|
||||
t_k = p["tmpc"] + 273.15
|
||||
e = sat_vap_pres(p["dwpc"])
|
||||
n = 77.6 * p["pres"] / t_k + 3.73e5 * e / (t_k * t_k)
|
||||
h_agl = p["hght"] - sfc_hght
|
||||
m = n + 0.157 * h_agl
|
||||
|
||||
%{
|
||||
pres: p["pres"],
|
||||
h_agl: h_agl,
|
||||
h_km: h_agl / 1000.0,
|
||||
tmpc: p["tmpc"],
|
||||
dwpc: p["dwpc"],
|
||||
n: n,
|
||||
m: m
|
||||
}
|
||||
end)
|
||||
end
|
||||
|
||||
defp compute_gradients(refract_profile) do
|
||||
refract_profile
|
||||
|> Enum.chunk_every(2, 1, :discard)
|
||||
|> Enum.flat_map(fn [prev, curr] ->
|
||||
dh_km = (curr.h_agl - prev.h_agl) / 1000.0
|
||||
|
||||
if abs(dh_km) < 0.01 do
|
||||
[]
|
||||
else
|
||||
dn = curr.n - prev.n
|
||||
[%{h_agl: (curr.h_agl + prev.h_agl) / 2.0, gradient: dn / dh_km}]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp detect_inversions(sorted, sfc_hght) do
|
||||
raw_inversions =
|
||||
sorted
|
||||
|> Enum.chunk_every(2, 1, :discard)
|
||||
|> Enum.flat_map(fn [lower, upper] ->
|
||||
if upper["tmpc"] > lower["tmpc"] do
|
||||
base = lower["hght"] - sfc_hght
|
||||
top = upper["hght"] - sfc_hght
|
||||
strength = upper["tmpc"] - lower["tmpc"]
|
||||
dh_100m = (top - base) / 100.0
|
||||
rate = if dh_100m > 0, do: strength / dh_100m, else: 0.0
|
||||
[%{base: base, top: top, strength: strength, rate: rate}]
|
||||
else
|
||||
[]
|
||||
end
|
||||
end)
|
||||
|
||||
# Merge adjacent layers within 200m gap
|
||||
merged =
|
||||
raw_inversions
|
||||
|> Enum.reduce([], fn inv, acc ->
|
||||
case acc do
|
||||
[] ->
|
||||
[inv]
|
||||
|
||||
[last | rest] ->
|
||||
if inv.base <= last.top + 200 do
|
||||
merged_inv = %{last | top: inv.top, strength: last.strength + inv.strength}
|
||||
[merged_inv | rest]
|
||||
else
|
||||
[inv | acc]
|
||||
end
|
||||
end
|
||||
end)
|
||||
|> Enum.reverse()
|
||||
|
||||
# Keep only meaningful inversions: strength >= 0.5°C AND base below 5000m AGL
|
||||
Enum.filter(merged, fn inv -> inv.strength >= 0.5 and inv.base < 5000 end)
|
||||
end
|
||||
|
||||
defp detect_ducts(refract_profile) do
|
||||
{ducts, in_duct_state} =
|
||||
refract_profile
|
||||
|> Enum.chunk_every(2, 1, :discard)
|
||||
|> Enum.reduce({[], nil}, fn [prev, curr], {ducts, duct_state} ->
|
||||
dm = curr.m - prev.m
|
||||
|
||||
case {dm < 0, duct_state} do
|
||||
{true, nil} ->
|
||||
# Entering a duct
|
||||
{ducts, %{base: prev.h_agl, base_m: prev.m}}
|
||||
|
||||
{false, %{base: base, base_m: base_m}} ->
|
||||
# Exiting a duct
|
||||
top = prev.h_agl
|
||||
strength = base_m - prev.m
|
||||
|
||||
if strength > 2 do
|
||||
duct = %{"base" => base, "top" => top, "strength" => Float.round(strength, 1)}
|
||||
{[duct | ducts], nil}
|
||||
else
|
||||
{ducts, nil}
|
||||
end
|
||||
|
||||
_ ->
|
||||
{ducts, duct_state}
|
||||
end
|
||||
end)
|
||||
|
||||
# Handle case where profile ends while still in a duct
|
||||
ducts =
|
||||
case in_duct_state do
|
||||
%{base: _base, base_m: _base_m} ->
|
||||
# Duct didn't close — discard (no clear top)
|
||||
ducts
|
||||
|
||||
nil ->
|
||||
ducts
|
||||
end
|
||||
|
||||
Enum.reverse(ducts)
|
||||
end
|
||||
|
||||
defp compute_k_index(sorted) do
|
||||
p850 = nearest_level(sorted, 850)
|
||||
p700 = nearest_level(sorted, 700)
|
||||
p500 = nearest_level(sorted, 500)
|
||||
|
||||
with %{"tmpc" => t850} when t850 != nil <- p850,
|
||||
%{"tmpc" => t700} when t700 != nil <- p700,
|
||||
%{"tmpc" => t500} when t500 != nil <- p500 do
|
||||
td850 = p850["dwpc"] || -30.0
|
||||
td700 = p700["dwpc"] || -30.0
|
||||
t850 - t500 + td850 - (t700 - td700)
|
||||
else
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp compute_lifted_index(sorted, sfc) do
|
||||
p500 = nearest_level(sorted, 500)
|
||||
|
||||
case p500 do
|
||||
%{"tmpc" => t500, "hght" => h500} when t500 != nil and h500 != nil ->
|
||||
sfc_tmpc = sfc["tmpc"]
|
||||
sfc_hght = sfc["hght"]
|
||||
# Simplified LI: T500 - (Tsfc - lapse_rate * height_diff)
|
||||
t500 - (sfc_tmpc - (h500 - sfc_hght) * 0.00976)
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
defp compute_precipitable_water(sorted) do
|
||||
sorted
|
||||
|> Enum.chunk_every(2, 1, :discard)
|
||||
|> Enum.reduce(0.0, fn [lower, upper], pw ->
|
||||
if lower["dwpc"] != nil and upper["dwpc"] != nil do
|
||||
mr1 = mixing_ratio(lower["dwpc"], lower["pres"])
|
||||
mr2 = mixing_ratio(upper["dwpc"], upper["pres"])
|
||||
dp = lower["pres"] - upper["pres"]
|
||||
pw + (mr1 + mr2) / 2.0 * dp / 9.81 / 10.0
|
||||
else
|
||||
pw
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp compute_boundary_layer_depth(sorted, sfc) do
|
||||
sfc_hght = sfc["hght"]
|
||||
sfc_tmpc = sfc["tmpc"]
|
||||
theta_sfc = sfc_tmpc + 273.15 + 9.8 * (sfc_hght / 1000.0)
|
||||
|
||||
sorted
|
||||
|> tl()
|
||||
|> Enum.find_value(fn p ->
|
||||
theta = p["tmpc"] + 273.15 + 9.8 * (p["hght"] / 1000.0)
|
||||
|
||||
if theta > theta_sfc + 2 do
|
||||
p["hght"] - sfc_hght
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp find_min_gradient([]), do: nil
|
||||
|
||||
defp find_min_gradient(dn_dh) do
|
||||
dn_dh
|
||||
|> Enum.min_by(fn %{gradient: g} -> g end)
|
||||
|> Map.get(:gradient)
|
||||
end
|
||||
|
||||
defp nearest_level(sorted, target_pres) do
|
||||
Enum.min_by(sorted, fn p -> abs(p["pres"] - target_pres) end)
|
||||
end
|
||||
end
|
||||
33
lib/microwaveprop/weather/station.ex
Normal file
33
lib/microwaveprop/weather/station.ex
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
defmodule Microwaveprop.Weather.Station do
|
||||
@moduledoc false
|
||||
use Ecto.Schema
|
||||
|
||||
import Ecto.Changeset
|
||||
|
||||
@primary_key {:id, :binary_id, autogenerate: true}
|
||||
@foreign_key_type :binary_id
|
||||
|
||||
schema "weather_stations" do
|
||||
field :station_code, :string
|
||||
field :station_type, :string
|
||||
field :wmo_number, :integer
|
||||
field :name, :string
|
||||
field :lat, :float
|
||||
field :lon, :float
|
||||
field :elevation_m, :float
|
||||
field :state, :string
|
||||
|
||||
timestamps(type: :utc_datetime)
|
||||
end
|
||||
|
||||
@required_fields ~w(station_code station_type name lat lon)a
|
||||
@optional_fields ~w(wmo_number elevation_m state)a
|
||||
|
||||
def changeset(station, attrs) do
|
||||
station
|
||||
|> cast(attrs, @required_fields ++ @optional_fields)
|
||||
|> validate_required(@required_fields)
|
||||
|> validate_inclusion(:station_type, ~w(asos sounding))
|
||||
|> unique_constraint([:station_code, :station_type])
|
||||
end
|
||||
end
|
||||
35
lib/microwaveprop/weather/surface_observation.ex
Normal file
35
lib/microwaveprop/weather/surface_observation.ex
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
defmodule Microwaveprop.Weather.SurfaceObservation do
|
||||
@moduledoc false
|
||||
use Ecto.Schema
|
||||
|
||||
import Ecto.Changeset
|
||||
|
||||
@primary_key {:id, :binary_id, autogenerate: true}
|
||||
@foreign_key_type :binary_id
|
||||
|
||||
schema "surface_observations" do
|
||||
belongs_to :station, Microwaveprop.Weather.Station
|
||||
field :observed_at, :utc_datetime
|
||||
field :temp_f, :float
|
||||
field :dewpoint_f, :float
|
||||
field :relative_humidity, :float
|
||||
field :wind_speed_kts, :float
|
||||
field :wind_direction_deg, :integer
|
||||
field :sea_level_pressure_mb, :float
|
||||
field :altimeter_setting, :float
|
||||
field :sky_condition, :string
|
||||
|
||||
timestamps(type: :utc_datetime)
|
||||
end
|
||||
|
||||
@required_fields ~w(station_id observed_at)a
|
||||
@optional_fields ~w(temp_f dewpoint_f relative_humidity wind_speed_kts wind_direction_deg sea_level_pressure_mb altimeter_setting sky_condition)a
|
||||
|
||||
def changeset(observation, attrs) do
|
||||
observation
|
||||
|> cast(attrs, @required_fields ++ @optional_fields)
|
||||
|> validate_required(@required_fields)
|
||||
|> foreign_key_constraint(:station_id)
|
||||
|> unique_constraint([:station_id, :observed_at])
|
||||
end
|
||||
end
|
||||
320
priv/repo/import_weather.exs
Normal file
320
priv/repo/import_weather.exs
Normal file
|
|
@ -0,0 +1,320 @@
|
|||
alias Microwaveprop.{Repo, Weather}
|
||||
alias Microwaveprop.Weather.{IemClient, SoundingParams}
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
# --- Station definitions (from PropCast) ---
|
||||
|
||||
asos_stations = [
|
||||
%{station_code: "KDFW", name: "Dallas/Fort Worth Intl", lat: 32.897, lon: -97.038, state: "TX"},
|
||||
%{station_code: "KDAL", name: "Dallas Love Field", lat: 32.847, lon: -96.851, state: "TX"},
|
||||
%{station_code: "KADS", name: "Addison", lat: 32.969, lon: -96.836, state: "TX"},
|
||||
%{station_code: "KTKI", name: "McKinney National", lat: 33.178, lon: -96.590, state: "TX"},
|
||||
%{station_code: "KGPM", name: "Grand Prairie Muni", lat: 32.698, lon: -97.043, state: "TX"},
|
||||
%{station_code: "KFWS", name: "Fort Worth Spinks", lat: 32.565, lon: -97.308, state: "TX"},
|
||||
%{station_code: "KFTW", name: "Fort Worth Meacham", lat: 32.820, lon: -97.362, state: "TX"},
|
||||
%{station_code: "KAFW", name: "Fort Worth Alliance", lat: 32.988, lon: -97.319, state: "TX"},
|
||||
%{station_code: "KRBD", name: "Dallas Executive", lat: 32.681, lon: -96.868, state: "TX"},
|
||||
%{station_code: "KGKY", name: "Arlington Muni", lat: 32.664, lon: -97.094, state: "TX"},
|
||||
%{station_code: "KCPT", name: "Cleburne Regional", lat: 32.354, lon: -97.434, state: "TX"},
|
||||
%{station_code: "KHQZ", name: "Mesquite Metro", lat: 32.747, lon: -96.530, state: "TX"},
|
||||
%{station_code: "KGYI", name: "Sherman/Denison", lat: 33.729, lon: -96.669, state: "TX"},
|
||||
%{station_code: "KGLE", name: "Gainesville Muni", lat: 33.651, lon: -97.197, state: "TX"},
|
||||
%{station_code: "KGVT", name: "Greenville/Hunt Co", lat: 33.068, lon: -96.065, state: "TX"},
|
||||
%{station_code: "KJDD", name: "Sulphur Springs", lat: 33.152, lon: -95.623, state: "TX"},
|
||||
%{station_code: "KGDJ", name: "Granbury Regional", lat: 32.445, lon: -97.817, state: "TX"},
|
||||
%{station_code: "KMWL", name: "Mineral Wells", lat: 32.782, lon: -98.060, state: "TX"},
|
||||
%{station_code: "KBKD", name: "Breckenridge/Stephens Co", lat: 32.719, lon: -98.890, state: "TX"},
|
||||
%{station_code: "KWEA", name: "Weatherford/Parker Co", lat: 32.746, lon: -97.682, state: "TX"},
|
||||
%{station_code: "KGGG", name: "Longview East Texas", lat: 32.384, lon: -94.711, state: "TX"},
|
||||
%{station_code: "KTYR", name: "Tyler Pounds", lat: 32.354, lon: -95.402, state: "TX"},
|
||||
%{station_code: "KCRS", name: "Corsicana/Navarro Co", lat: 32.028, lon: -96.400, state: "TX"},
|
||||
%{station_code: "KWXD", name: "Waxahachie/Ellis Co", lat: 32.392, lon: -96.849, state: "TX"},
|
||||
%{station_code: "KPNX", name: "Henderson/Rusk Co", lat: 32.152, lon: -94.913, state: "TX"},
|
||||
%{station_code: "KPWG", name: "Waco Regional", lat: 31.611, lon: -97.095, state: "TX"},
|
||||
%{station_code: "KACT", name: "Waco TSTC", lat: 31.611, lon: -97.226, state: "TX"},
|
||||
%{station_code: "KTPL", name: "Temple/Draughon-Miller", lat: 31.152, lon: -97.408, state: "TX"},
|
||||
%{station_code: "KILE", name: "Killeen/Fort Cavazos", lat: 31.086, lon: -97.683, state: "TX"},
|
||||
%{station_code: "KGRK", name: "Killeen-Fort Hood", lat: 31.067, lon: -97.829, state: "TX"},
|
||||
%{station_code: "KAUS", name: "Austin-Bergstrom", lat: 30.198, lon: -97.670, state: "TX"},
|
||||
%{station_code: "KSAT", name: "San Antonio Intl", lat: 29.534, lon: -98.470, state: "TX"},
|
||||
%{station_code: "KBMQ", name: "Burnet Muni", lat: 30.739, lon: -98.239, state: "TX"},
|
||||
%{station_code: "KSSF", name: "San Antonio Stinson", lat: 29.337, lon: -98.471, state: "TX"},
|
||||
%{station_code: "KGTU", name: "Georgetown Muni", lat: 30.678, lon: -97.679, state: "TX"},
|
||||
%{station_code: "KHYI", name: "San Marcos Rgnl", lat: 29.893, lon: -97.863, state: "TX"},
|
||||
%{station_code: "KHOU", name: "Houston Hobby", lat: 29.645, lon: -95.279, state: "TX"},
|
||||
%{station_code: "KIAH", name: "Houston Bush Intcntl", lat: 29.980, lon: -95.342, state: "TX"},
|
||||
%{station_code: "KCXO", name: "Conroe North Houston", lat: 30.352, lon: -95.415, state: "TX"},
|
||||
%{station_code: "KBPT", name: "Beaumont/Port Arthur", lat: 29.951, lon: -94.021, state: "TX"},
|
||||
%{station_code: "KLCH", name: "Lake Charles Regional", lat: 30.126, lon: -93.223, state: "LA"},
|
||||
%{station_code: "KVCT", name: "Victoria Regional", lat: 28.852, lon: -96.918, state: "TX"},
|
||||
%{station_code: "KCRP", name: "Corpus Christi Intl", lat: 27.770, lon: -97.502, state: "TX"},
|
||||
%{station_code: "KOKC", name: "Oklahoma City Will Rogers", lat: 35.393, lon: -97.601, state: "OK"},
|
||||
%{station_code: "KOUN", name: "Norman/OU", lat: 35.246, lon: -97.472, state: "OK"},
|
||||
%{station_code: "KPWA", name: "Wiley Post", lat: 35.534, lon: -97.647, state: "OK"},
|
||||
%{station_code: "KSWO", name: "Stillwater Regional", lat: 36.161, lon: -97.086, state: "OK"},
|
||||
%{station_code: "KTIK", name: "Tinker AFB", lat: 35.415, lon: -97.387, state: "OK"},
|
||||
%{station_code: "KLAW", name: "Lawton/Fort Sill", lat: 34.568, lon: -98.416, state: "OK"},
|
||||
%{station_code: "KADU", name: "Ardmore Downtown", lat: 34.303, lon: -97.019, state: "OK"},
|
||||
%{station_code: "KSNL", name: "Shawnee Regional", lat: 35.358, lon: -96.943, state: "OK"},
|
||||
%{station_code: "KCSM", name: "Clinton/Sherman", lat: 35.339, lon: -99.201, state: "OK"},
|
||||
%{station_code: "KGAG", name: "Gage/Ellis Co", lat: 36.296, lon: -99.777, state: "OK"},
|
||||
%{station_code: "KWDG", name: "Enid Woodring", lat: 36.379, lon: -97.791, state: "OK"},
|
||||
%{station_code: "KBVO", name: "Bartlesville", lat: 36.765, lon: -96.012, state: "OK"},
|
||||
%{station_code: "KTUL", name: "Tulsa Intl", lat: 36.198, lon: -95.888, state: "OK"},
|
||||
%{station_code: "KMKO", name: "Muskogee/Davis Field", lat: 35.657, lon: -95.367, state: "OK"},
|
||||
%{station_code: "KPOC", name: "Pontotoc Co/Ada", lat: 34.802, lon: -96.671, state: "OK"},
|
||||
%{station_code: "KGMJ", name: "Grove Muni", lat: 36.607, lon: -94.733, state: "OK"},
|
||||
%{station_code: "KABI", name: "Abilene Regional", lat: 32.411, lon: -99.682, state: "TX"},
|
||||
%{station_code: "KMAF", name: "Midland Intl", lat: 31.943, lon: -102.201, state: "TX"},
|
||||
%{station_code: "KBPG", name: "Big Spring McMahon", lat: 32.213, lon: -101.522, state: "TX"},
|
||||
%{station_code: "KSNS", name: "San Angelo Mathis", lat: 31.358, lon: -100.497, state: "TX"},
|
||||
%{station_code: "KLBB", name: "Lubbock Preston Smith", lat: 33.664, lon: -101.823, state: "TX"},
|
||||
%{station_code: "KAMA", name: "Amarillo Rick Husband", lat: 35.220, lon: -101.706, state: "TX"},
|
||||
%{station_code: "KPPA", name: "Pampa/Perry Lefors", lat: 35.613, lon: -100.996, state: "TX"},
|
||||
%{station_code: "KPVW", name: "Plainview Hale Co", lat: 34.183, lon: -101.722, state: "TX"},
|
||||
%{station_code: "KINK", name: "Wink/Winkler Co", lat: 31.779, lon: -103.201, state: "TX"},
|
||||
%{station_code: "KELP", name: "El Paso Intl", lat: 31.807, lon: -106.378, state: "TX"},
|
||||
%{station_code: "KCNM", name: "Carlsbad Cavern City", lat: 32.337, lon: -104.263, state: "NM"},
|
||||
%{station_code: "KROW", name: "Roswell Intl Air Ctr", lat: 33.302, lon: -104.531, state: "NM"},
|
||||
%{station_code: "KCLO", name: "Clovis Muni", lat: 34.434, lon: -103.079, state: "NM"},
|
||||
%{station_code: "KHOB", name: "Hobbs Lea Co", lat: 32.688, lon: -103.217, state: "NM"},
|
||||
%{station_code: "KTCC", name: "Tucumcari Muni", lat: 35.183, lon: -103.603, state: "NM"},
|
||||
%{station_code: "KICT", name: "Wichita Dwight D. Eisenhower", lat: 37.650, lon: -97.433, state: "KS"},
|
||||
%{station_code: "KHUT", name: "Hutchinson Regional", lat: 38.065, lon: -97.861, state: "KS"},
|
||||
%{station_code: "KDDC", name: "Dodge City Regional", lat: 37.763, lon: -99.965, state: "KS"},
|
||||
%{station_code: "KGLD", name: "Goodland Municipal", lat: 39.370, lon: -101.700, state: "KS"},
|
||||
%{station_code: "KGBD", name: "Great Bend Municipal", lat: 38.344, lon: -98.859, state: "KS"},
|
||||
%{station_code: "KCNU", name: "Chanute Martin Johnson", lat: 37.669, lon: -95.485, state: "KS"},
|
||||
%{station_code: "KPTT", name: "Pratt Regional", lat: 37.700, lon: -98.747, state: "KS"},
|
||||
%{station_code: "KSGF", name: "Springfield Branson Natl", lat: 37.246, lon: -93.389, state: "MO"},
|
||||
%{station_code: "KJLN", name: "Joplin Regional", lat: 37.152, lon: -94.498, state: "MO"},
|
||||
%{station_code: "KFSM", name: "Fort Smith Regional", lat: 35.337, lon: -94.368, state: "AR"},
|
||||
%{station_code: "KTXK", name: "Texarkana Regional", lat: 33.454, lon: -93.991, state: "TX"},
|
||||
%{station_code: "KLIT", name: "Little Rock Clinton", lat: 34.729, lon: -92.224, state: "AR"},
|
||||
%{station_code: "KHRO", name: "Harrison Boone Co", lat: 36.261, lon: -93.155, state: "AR"},
|
||||
%{station_code: "KFYV", name: "Fayetteville Drake Field", lat: 36.005, lon: -94.170, state: "AR"},
|
||||
%{station_code: "KXNA", name: "NW Arkansas Natl", lat: 36.282, lon: -94.307, state: "AR"},
|
||||
%{station_code: "KSHV", name: "Shreveport Regional", lat: 32.447, lon: -93.826, state: "LA"},
|
||||
%{station_code: "KMLU", name: "Monroe Regional", lat: 32.511, lon: -92.038, state: "LA"},
|
||||
%{station_code: "KJAN", name: "Jackson Medgar Wiley Evers", lat: 32.311, lon: -90.076, state: "MS"},
|
||||
%{station_code: "KMEI", name: "Meridian Regional", lat: 32.334, lon: -88.752, state: "MS"},
|
||||
%{station_code: "KNAC", name: "Nacogdoches/Mangham", lat: 31.578, lon: -94.710, state: "TX"},
|
||||
%{station_code: "KNEW", name: "New Orleans Lakefront", lat: 30.042, lon: -90.028, state: "LA"},
|
||||
%{station_code: "KMSY", name: "New Orleans Armstrong", lat: 29.993, lon: -90.258, state: "LA"}
|
||||
]
|
||||
|
||||
sounding_stations = [
|
||||
%{station_code: "FWD", wmo_number: 72_249, name: "Fort Worth TX", lat: 32.835, lon: -97.298, state: "TX"},
|
||||
%{station_code: "OUN", wmo_number: 72_357, name: "Oklahoma City OK", lat: 35.181, lon: -97.436, state: "OK"},
|
||||
%{station_code: "SHV", wmo_number: 72_248, name: "Shreveport LA", lat: 32.450, lon: -93.841, state: "LA"},
|
||||
%{station_code: "AMA", wmo_number: 72_363, name: "Amarillo TX", lat: 35.233, lon: -101.706, state: "TX"},
|
||||
%{station_code: "LCH", wmo_number: 72_240, name: "Lake Charles LA", lat: 30.125, lon: -93.217, state: "LA"},
|
||||
%{station_code: "MAF", wmo_number: 72_265, name: "Midland TX", lat: 31.943, lon: -102.189, state: "TX"},
|
||||
%{station_code: "LZK", wmo_number: 72_340, name: "Little Rock AR", lat: 34.836, lon: -92.263, state: "AR"},
|
||||
%{station_code: "BRO", wmo_number: 72_250, name: "Brownsville TX", lat: 25.910, lon: -97.419, state: "TX"},
|
||||
%{station_code: "SPS", wmo_number: 72_451, name: "Wichita Falls TX", lat: 33.989, lon: -98.492, state: "TX"}
|
||||
]
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
defmodule ImportHelpers do
|
||||
@moduledoc false
|
||||
|
||||
@km_per_deg 111.0
|
||||
|
||||
def nearest_stations(lat, lon, stations, radius_km) do
|
||||
Enum.filter(stations, fn s ->
|
||||
dlat = (s.lat - lat) * @km_per_deg
|
||||
dlon = (s.lon - lon) * @km_per_deg * :math.cos(lat * :math.pi() / 180)
|
||||
:math.sqrt(dlat * dlat + dlon * dlon) <= radius_km
|
||||
end)
|
||||
end
|
||||
|
||||
def sounding_times_around(dt) do
|
||||
# Return the 00Z and 12Z sounding times that bracket the QSO time
|
||||
date = DateTime.to_date(dt)
|
||||
hour = dt.hour
|
||||
|
||||
times =
|
||||
if hour < 12 do
|
||||
[
|
||||
DateTime.new!(Date.add(date, -1), ~T[12:00:00], "Etc/UTC"),
|
||||
DateTime.new!(date, ~T[00:00:00], "Etc/UTC")
|
||||
]
|
||||
else
|
||||
[
|
||||
DateTime.new!(date, ~T[00:00:00], "Etc/UTC"),
|
||||
DateTime.new!(date, ~T[12:00:00], "Etc/UTC")
|
||||
]
|
||||
end
|
||||
|
||||
Enum.uniq(times)
|
||||
end
|
||||
|
||||
def rate_limit(delay_ms \\ 200) do
|
||||
Process.sleep(delay_ms)
|
||||
end
|
||||
end
|
||||
|
||||
# --- Seed stations ---
|
||||
|
||||
IO.puts("Seeding #{length(asos_stations)} ASOS stations...")
|
||||
|
||||
asos_station_map =
|
||||
Enum.reduce(asos_stations, %{}, fn attrs, acc ->
|
||||
{:ok, station} =
|
||||
Weather.find_or_create_station(Map.put(attrs, :station_type, "asos"))
|
||||
|
||||
Map.put(acc, station.station_code, station)
|
||||
end)
|
||||
|
||||
IO.puts("Seeding #{length(sounding_stations)} sounding stations...")
|
||||
|
||||
sounding_station_map =
|
||||
Enum.reduce(sounding_stations, %{}, fn attrs, acc ->
|
||||
{:ok, station} =
|
||||
Weather.find_or_create_station(Map.put(attrs, :station_type, "sounding"))
|
||||
|
||||
Map.put(acc, station.station_code, station)
|
||||
end)
|
||||
|
||||
IO.puts("Stations seeded: #{map_size(asos_station_map)} ASOS, #{map_size(sounding_station_map)} sounding")
|
||||
|
||||
# --- Fetch weather for each QSO ---
|
||||
|
||||
qsos =
|
||||
Microwaveprop.Radio.Qso
|
||||
|> where([q], not is_nil(q.pos1))
|
||||
|> order_by([q], q.qso_timestamp)
|
||||
|> Repo.all()
|
||||
|
||||
IO.puts("Processing #{length(qsos)} QSOs with positions...")
|
||||
|
||||
# Track which station+time combos we've already fetched to avoid duplicates
|
||||
asos_fetched = :ets.new(:asos_fetched, [:set, :public])
|
||||
raob_fetched = :ets.new(:raob_fetched, [:set, :public])
|
||||
|
||||
asos_station_list = Enum.map(asos_stations, fn s -> Map.put(s, :station_type, "asos") end)
|
||||
|
||||
total = length(qsos)
|
||||
|
||||
qsos
|
||||
|> Enum.with_index(1)
|
||||
|> Enum.each(fn {qso, idx} ->
|
||||
lat = qso.pos1["lat"] || qso.pos1["latitude"]
|
||||
lon = qso.pos1["lng"] || qso.pos1["lon"] || qso.pos1["longitude"]
|
||||
|
||||
if lat && lon do
|
||||
if rem(idx, 100) == 0, do: IO.puts(" QSO #{idx}/#{total}")
|
||||
|
||||
# Find nearby ASOS stations (within 150km)
|
||||
nearby_asos = ImportHelpers.nearest_stations(lat, lon, asos_station_list, 150)
|
||||
|
||||
Enum.each(nearby_asos, fn s ->
|
||||
# Fetch a 4-hour window around QSO time
|
||||
start_dt = DateTime.add(qso.qso_timestamp, -2 * 3600, :second)
|
||||
end_dt = DateTime.add(qso.qso_timestamp, 2 * 3600, :second)
|
||||
key = {s.station_code, DateTime.to_date(start_dt), div(start_dt.hour, 4)}
|
||||
|
||||
unless :ets.member(asos_fetched, key) do
|
||||
:ets.insert(asos_fetched, {key, true})
|
||||
|
||||
station = Map.get(asos_station_map, s.station_code)
|
||||
|
||||
if station do
|
||||
case IemClient.fetch_asos(s.station_code, start_dt, end_dt) do
|
||||
{:ok, rows} ->
|
||||
Enum.each(rows, fn row ->
|
||||
if row.observed_at do
|
||||
Weather.upsert_surface_observation(station, row)
|
||||
end
|
||||
end)
|
||||
|
||||
{:error, reason} ->
|
||||
IO.puts(" ASOS error #{s.station_code}: #{inspect(reason)}")
|
||||
end
|
||||
|
||||
ImportHelpers.rate_limit()
|
||||
end
|
||||
end
|
||||
end)
|
||||
|
||||
# Find nearby sounding stations (within 300km)
|
||||
nearby_soundings = ImportHelpers.nearest_stations(lat, lon, sounding_stations, 300)
|
||||
|
||||
Enum.each(nearby_soundings, fn s ->
|
||||
sounding_times = ImportHelpers.sounding_times_around(qso.qso_timestamp)
|
||||
|
||||
Enum.each(sounding_times, fn sounding_time ->
|
||||
key = {s.station_code, sounding_time}
|
||||
|
||||
unless :ets.member(raob_fetched, key) do
|
||||
:ets.insert(raob_fetched, {key, true})
|
||||
|
||||
station = Map.get(sounding_station_map, s.station_code)
|
||||
|
||||
if station do
|
||||
case IemClient.fetch_raob(s.station_code, sounding_time) do
|
||||
{:ok, [parsed | _]} ->
|
||||
params = SoundingParams.derive(parsed.profile)
|
||||
|
||||
sounding_attrs =
|
||||
if params do
|
||||
%{
|
||||
observed_at: parsed.observed_at,
|
||||
profile: parsed.profile,
|
||||
level_count: params.level_count,
|
||||
surface_pressure_mb: params.surface_pressure_mb,
|
||||
surface_temp_c: params.surface_temp_c,
|
||||
surface_dewpoint_c: params.surface_dewpoint_c,
|
||||
surface_refractivity: params.surface_refractivity,
|
||||
min_refractivity_gradient: params.min_refractivity_gradient,
|
||||
boundary_layer_depth_m: params.boundary_layer_depth_m,
|
||||
precipitable_water_mm: params.precipitable_water_mm,
|
||||
k_index: params.k_index,
|
||||
lifted_index: params.lifted_index,
|
||||
ducting_detected: params.ducting_detected,
|
||||
duct_characteristics: params.duct_characteristics
|
||||
}
|
||||
else
|
||||
%{
|
||||
observed_at: parsed.observed_at,
|
||||
profile: parsed.profile,
|
||||
level_count: length(parsed.profile)
|
||||
}
|
||||
end
|
||||
|
||||
Weather.upsert_sounding(station, sounding_attrs)
|
||||
|
||||
{:ok, []} ->
|
||||
:ok
|
||||
|
||||
{:error, reason} ->
|
||||
IO.puts(" RAOB error #{s.station_code}: #{inspect(reason)}")
|
||||
end
|
||||
|
||||
ImportHelpers.rate_limit()
|
||||
end
|
||||
end
|
||||
end)
|
||||
end)
|
||||
end
|
||||
end)
|
||||
|
||||
:ets.delete(asos_fetched)
|
||||
:ets.delete(raob_fetched)
|
||||
|
||||
# --- Summary ---
|
||||
|
||||
asos_count = Repo.aggregate(Microwaveprop.Weather.SurfaceObservation, :count)
|
||||
sounding_count = Repo.aggregate(Microwaveprop.Weather.Sounding, :count)
|
||||
station_count = Repo.aggregate(Microwaveprop.Weather.Station, :count)
|
||||
|
||||
IO.puts("""
|
||||
|
||||
Import complete:
|
||||
Stations: #{station_count}
|
||||
Surface observations: #{asos_count}
|
||||
Soundings: #{sounding_count}
|
||||
""")
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
defmodule Microwaveprop.Repo.Migrations.CreateWeatherTables do
|
||||
use Ecto.Migration
|
||||
|
||||
def change do
|
||||
create table(:weather_stations, primary_key: false) do
|
||||
add :id, :binary_id, primary_key: true
|
||||
add :station_code, :string, null: false
|
||||
add :station_type, :string, null: false
|
||||
add :wmo_number, :integer
|
||||
add :name, :string, null: false
|
||||
add :lat, :float, null: false
|
||||
add :lon, :float, null: false
|
||||
add :elevation_m, :float
|
||||
add :state, :string
|
||||
|
||||
timestamps(type: :utc_datetime)
|
||||
end
|
||||
|
||||
create unique_index(:weather_stations, [:station_code, :station_type])
|
||||
create index(:weather_stations, [:lat, :lon])
|
||||
|
||||
create table(:surface_observations, primary_key: false) do
|
||||
add :id, :binary_id, primary_key: true
|
||||
add :station_id, references(:weather_stations, type: :binary_id), null: false
|
||||
add :observed_at, :utc_datetime, null: false
|
||||
add :temp_f, :float
|
||||
add :dewpoint_f, :float
|
||||
add :relative_humidity, :float
|
||||
add :wind_speed_kts, :float
|
||||
add :wind_direction_deg, :integer
|
||||
add :sea_level_pressure_mb, :float
|
||||
add :altimeter_setting, :float
|
||||
add :sky_condition, :string
|
||||
|
||||
timestamps(type: :utc_datetime)
|
||||
end
|
||||
|
||||
create unique_index(:surface_observations, [:station_id, :observed_at])
|
||||
create index(:surface_observations, [:observed_at])
|
||||
|
||||
create table(:soundings, primary_key: false) do
|
||||
add :id, :binary_id, primary_key: true
|
||||
add :station_id, references(:weather_stations, type: :binary_id), null: false
|
||||
add :observed_at, :utc_datetime, null: false
|
||||
add :profile, :map, null: false
|
||||
add :level_count, :integer, null: false
|
||||
add :surface_pressure_mb, :float
|
||||
add :surface_temp_c, :float
|
||||
add :surface_dewpoint_c, :float
|
||||
add :surface_refractivity, :float
|
||||
add :min_refractivity_gradient, :float
|
||||
add :boundary_layer_depth_m, :float
|
||||
add :precipitable_water_mm, :float
|
||||
add :k_index, :float
|
||||
add :lifted_index, :float
|
||||
add :ducting_detected, :boolean, null: false, default: false
|
||||
add :duct_characteristics, :map
|
||||
|
||||
timestamps(type: :utc_datetime)
|
||||
end
|
||||
|
||||
create unique_index(:soundings, [:station_id, :observed_at])
|
||||
create index(:soundings, [:observed_at])
|
||||
create index(:soundings, [:ducting_detected])
|
||||
end
|
||||
end
|
||||
119
test/microwaveprop/weather/iem_client_test.exs
Normal file
119
test/microwaveprop/weather/iem_client_test.exs
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
defmodule Microwaveprop.Weather.IemClientTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Microwaveprop.Weather.IemClient
|
||||
|
||||
describe "parse_asos_csv/1" do
|
||||
test "parses valid CSV rows" do
|
||||
csv = """
|
||||
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1
|
||||
KDFW,2026-03-28 18:53,75.0,55.0,49.12,12,180,1013.2,29.92,SCT
|
||||
KDFW,2026-03-28 19:53,76.0,54.0,46.00,10,190,1013.0,29.91,FEW
|
||||
"""
|
||||
|
||||
rows = IemClient.parse_asos_csv(csv)
|
||||
assert length(rows) == 2
|
||||
|
||||
first = hd(rows)
|
||||
assert first.temp_f == 75.0
|
||||
assert first.dewpoint_f == 55.0
|
||||
assert first.relative_humidity == 49.12
|
||||
assert first.wind_speed_kts == 12.0
|
||||
assert first.wind_direction_deg == 180
|
||||
assert first.sea_level_pressure_mb == 1013.2
|
||||
assert first.altimeter_setting == 29.92
|
||||
assert first.sky_condition == "SCT"
|
||||
assert first.observed_at == ~U[2026-03-28 18:53:00Z]
|
||||
end
|
||||
|
||||
test "handles null values in CSV" do
|
||||
csv = """
|
||||
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1
|
||||
KDFW,2026-03-28 18:53,null,null,null,null,null,null,null,null
|
||||
"""
|
||||
|
||||
rows = IemClient.parse_asos_csv(csv)
|
||||
assert length(rows) == 1
|
||||
|
||||
first = hd(rows)
|
||||
assert first.temp_f == nil
|
||||
assert first.dewpoint_f == nil
|
||||
assert first.sky_condition == nil
|
||||
end
|
||||
|
||||
test "skips comment lines and empty lines" do
|
||||
csv = """
|
||||
#DEBUG something
|
||||
station,valid,tmpf,dwpf,relh,sknt,drct,mslp,alti,skyc1
|
||||
KDFW,2026-03-28 18:53,75.0,55.0,49.0,12,180,1013.2,29.92,SCT
|
||||
|
||||
"""
|
||||
|
||||
rows = IemClient.parse_asos_csv(csv)
|
||||
assert length(rows) == 1
|
||||
end
|
||||
end
|
||||
|
||||
describe "parse_raob_json/1" do
|
||||
test "parses valid RAOB JSON response" do
|
||||
json = %{
|
||||
"profiles" => [
|
||||
%{
|
||||
"station" => "KFWD",
|
||||
"valid" => "2026-03-28 12:00:00+00:00",
|
||||
"profile" => [
|
||||
%{"pres" => 1013.0, "hght" => 171.0, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180.0, "sknt" => 10.0},
|
||||
%{"pres" => 925.0, "hght" => 800.0, "tmpc" => 20.0, "dwpc" => 12.0, "drct" => 200.0, "sknt" => 15.0}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
result = IemClient.parse_raob_json(json)
|
||||
assert length(result) == 1
|
||||
|
||||
sounding = hd(result)
|
||||
assert sounding.observed_at == ~U[2026-03-28 12:00:00Z]
|
||||
assert length(sounding.profile) == 2
|
||||
assert hd(sounding.profile)["pres"] == 1013.0
|
||||
end
|
||||
|
||||
test "returns empty list for empty profiles" do
|
||||
json = %{"profiles" => []}
|
||||
assert IemClient.parse_raob_json(json) == []
|
||||
end
|
||||
|
||||
test "handles missing profile key gracefully" do
|
||||
json = %{}
|
||||
assert IemClient.parse_raob_json(json) == []
|
||||
end
|
||||
end
|
||||
|
||||
describe "asos_url/3" do
|
||||
test "builds correct URL for date range" do
|
||||
url =
|
||||
IemClient.asos_url(
|
||||
"KDFW",
|
||||
~U[2026-03-28 12:00:00Z],
|
||||
~U[2026-03-28 18:00:00Z]
|
||||
)
|
||||
|
||||
assert url =~ "station=KDFW"
|
||||
assert url =~ "year1=2026"
|
||||
assert url =~ "month1=3"
|
||||
assert url =~ "day1=28"
|
||||
assert url =~ "hour1=12"
|
||||
assert url =~ "year2=2026"
|
||||
assert url =~ "hour2=18"
|
||||
assert url =~ "format=onlycomma"
|
||||
end
|
||||
end
|
||||
|
||||
describe "raob_url/2" do
|
||||
test "builds correct URL for sounding fetch" do
|
||||
url = IemClient.raob_url("FWD", ~U[2026-03-28 12:00:00Z])
|
||||
assert url =~ "station=FWD"
|
||||
assert url =~ "ts=202603281200"
|
||||
end
|
||||
end
|
||||
end
|
||||
161
test/microwaveprop/weather/sounding_params_test.exs
Normal file
161
test/microwaveprop/weather/sounding_params_test.exs
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
defmodule Microwaveprop.Weather.SoundingParamsTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Microwaveprop.Weather.SoundingParams
|
||||
|
||||
# Realistic 3-level profile (surface, 925, 850 hPa)
|
||||
@simple_profile [
|
||||
%{"pres" => 1013.0, "hght" => 171, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180, "sknt" => 10},
|
||||
%{"pres" => 925.0, "hght" => 800, "tmpc" => 20.0, "dwpc" => 12.0, "drct" => 200, "sknt" => 15},
|
||||
%{"pres" => 850.0, "hght" => 1500, "tmpc" => 15.0, "dwpc" => 8.0, "drct" => 220, "sknt" => 20}
|
||||
]
|
||||
|
||||
# Profile with a temperature inversion (temp increases with height in one layer)
|
||||
@inversion_profile [
|
||||
%{"pres" => 1013.0, "hght" => 100, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180, "sknt" => 5},
|
||||
%{"pres" => 980.0, "hght" => 400, "tmpc" => 22.0, "dwpc" => 14.0, "drct" => 190, "sknt" => 8},
|
||||
%{"pres" => 950.0, "hght" => 700, "tmpc" => 26.0, "dwpc" => 10.0, "drct" => 200, "sknt" => 12},
|
||||
%{"pres" => 925.0, "hght" => 900, "tmpc" => 20.0, "dwpc" => 8.0, "drct" => 210, "sknt" => 15},
|
||||
%{"pres" => 850.0, "hght" => 1500, "tmpc" => 14.0, "dwpc" => 5.0, "drct" => 230, "sknt" => 20},
|
||||
%{"pres" => 700.0, "hght" => 3100, "tmpc" => 2.0, "dwpc" => -5.0, "drct" => 250, "sknt" => 30},
|
||||
%{"pres" => 500.0, "hght" => 5600, "tmpc" => -15.0, "dwpc" => -25.0, "drct" => 270, "sknt" => 40}
|
||||
]
|
||||
|
||||
# Profile designed to create a duct (M decreases with height)
|
||||
# Strong surface-based inversion with very dry air aloft → M decreases
|
||||
@ducting_profile [
|
||||
%{"pres" => 1013.0, "hght" => 0, "tmpc" => 30.0, "dwpc" => 25.0, "drct" => 180, "sknt" => 5},
|
||||
%{"pres" => 1000.0, "hght" => 100, "tmpc" => 35.0, "dwpc" => 5.0, "drct" => 180, "sknt" => 5},
|
||||
%{"pres" => 980.0, "hght" => 300, "tmpc" => 28.0, "dwpc" => 10.0, "drct" => 190, "sknt" => 8},
|
||||
%{"pres" => 950.0, "hght" => 550, "tmpc" => 24.0, "dwpc" => 12.0, "drct" => 200, "sknt" => 10},
|
||||
%{"pres" => 925.0, "hght" => 800, "tmpc" => 20.0, "dwpc" => 10.0, "drct" => 210, "sknt" => 12},
|
||||
%{"pres" => 850.0, "hght" => 1500, "tmpc" => 14.0, "dwpc" => 5.0, "drct" => 230, "sknt" => 18},
|
||||
%{"pres" => 700.0, "hght" => 3100, "tmpc" => 2.0, "dwpc" => -10.0, "drct" => 250, "sknt" => 25},
|
||||
%{"pres" => 500.0, "hght" => 5600, "tmpc" => -18.0, "dwpc" => -30.0, "drct" => 270, "sknt" => 35}
|
||||
]
|
||||
|
||||
describe "sat_vap_pres/1" do
|
||||
test "returns known value at 20°C (~23.4 hPa)" do
|
||||
result = SoundingParams.sat_vap_pres(20.0)
|
||||
assert_in_delta result, 23.4, 0.5
|
||||
end
|
||||
|
||||
test "returns known value at 0°C (~6.1 hPa)" do
|
||||
result = SoundingParams.sat_vap_pres(0.0)
|
||||
assert_in_delta result, 6.1, 0.2
|
||||
end
|
||||
end
|
||||
|
||||
describe "mixing_ratio/2" do
|
||||
test "returns reasonable value for typical surface conditions" do
|
||||
# At 15°C, 1013 hPa, mixing ratio ~10.7 g/kg
|
||||
result = SoundingParams.mixing_ratio(15.0, 1013.0)
|
||||
assert_in_delta result, 10.7, 1.0
|
||||
end
|
||||
end
|
||||
|
||||
describe "derive/1" do
|
||||
test "returns nil for profiles with fewer than 3 valid levels" do
|
||||
short = [
|
||||
%{"pres" => 1013.0, "hght" => 171, "tmpc" => 25.0, "dwpc" => 15.0},
|
||||
%{"pres" => 925.0, "hght" => 800, "tmpc" => 20.0, "dwpc" => 12.0}
|
||||
]
|
||||
|
||||
assert SoundingParams.derive(short) == nil
|
||||
end
|
||||
|
||||
test "returns nil for empty profile" do
|
||||
assert SoundingParams.derive([]) == nil
|
||||
end
|
||||
|
||||
test "computes surface parameters" do
|
||||
result = SoundingParams.derive(@simple_profile)
|
||||
assert result
|
||||
assert_in_delta result.surface_temp_c, 25.0, 0.01
|
||||
assert_in_delta result.surface_dewpoint_c, 15.0, 0.01
|
||||
assert_in_delta result.surface_pressure_mb, 1013.0, 0.01
|
||||
end
|
||||
|
||||
test "computes surface refractivity (N-units)" do
|
||||
result = SoundingParams.derive(@simple_profile)
|
||||
# At 25°C, 15°C dewpoint, 1013 hPa — N should be ~330-350
|
||||
assert result.surface_refractivity > 300
|
||||
assert result.surface_refractivity < 400
|
||||
end
|
||||
|
||||
test "computes dN/dh gradients" do
|
||||
result = SoundingParams.derive(@simple_profile)
|
||||
# Standard atmosphere dN/dh is about -40/km
|
||||
assert result.min_refractivity_gradient < 0
|
||||
end
|
||||
|
||||
test "detects temperature inversions" do
|
||||
result = SoundingParams.derive(@inversion_profile)
|
||||
assert length(result.inversions) > 0
|
||||
|
||||
inv = hd(result.inversions)
|
||||
assert inv.strength > 0
|
||||
assert inv.base >= 0
|
||||
end
|
||||
|
||||
test "no inversions in standard lapse profile" do
|
||||
# Simple profile has monotonic temperature decrease
|
||||
result = SoundingParams.derive(@simple_profile)
|
||||
assert result.inversions == []
|
||||
end
|
||||
|
||||
test "computes precipitable water" do
|
||||
result = SoundingParams.derive(@inversion_profile)
|
||||
# PW should be positive and reasonable (5-80mm)
|
||||
assert result.precipitable_water_mm > 0
|
||||
assert result.precipitable_water_mm < 100
|
||||
end
|
||||
|
||||
test "computes K-Index when standard levels available" do
|
||||
result = SoundingParams.derive(@inversion_profile)
|
||||
# K-Index should be computed (profile has 850, 700, 500 levels)
|
||||
assert result.k_index
|
||||
end
|
||||
|
||||
test "computes Lifted Index when 500 hPa available" do
|
||||
result = SoundingParams.derive(@inversion_profile)
|
||||
assert result.lifted_index
|
||||
end
|
||||
|
||||
test "detects ducting in strong inversion profile" do
|
||||
result = SoundingParams.derive(@ducting_profile)
|
||||
assert result.ducting_detected == true
|
||||
assert length(result.ducts) > 0
|
||||
end
|
||||
|
||||
test "no ducting in standard profile" do
|
||||
result = SoundingParams.derive(@simple_profile)
|
||||
assert result.ducting_detected == false
|
||||
assert result.ducts == []
|
||||
end
|
||||
|
||||
test "computes boundary layer depth" do
|
||||
result = SoundingParams.derive(@inversion_profile)
|
||||
# BL depth should be positive and reasonable
|
||||
assert result.boundary_layer_depth_m > 0
|
||||
assert result.boundary_layer_depth_m < 5000
|
||||
end
|
||||
|
||||
test "returns level_count matching filtered profile" do
|
||||
result = SoundingParams.derive(@simple_profile)
|
||||
assert result.level_count == 3
|
||||
end
|
||||
|
||||
test "filters out levels with nil pressure, temp, or height" do
|
||||
profile_with_nils = [
|
||||
%{"pres" => 1013.0, "hght" => 171, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180, "sknt" => 10},
|
||||
%{"pres" => nil, "hght" => 500, "tmpc" => 22.0, "dwpc" => 13.0, "drct" => 190, "sknt" => 12},
|
||||
%{"pres" => 925.0, "hght" => 800, "tmpc" => 20.0, "dwpc" => 12.0, "drct" => 200, "sknt" => 15},
|
||||
%{"pres" => 850.0, "hght" => 1500, "tmpc" => 15.0, "dwpc" => 8.0, "drct" => 220, "sknt" => 20}
|
||||
]
|
||||
|
||||
result = SoundingParams.derive(profile_with_nils)
|
||||
assert result.level_count == 3
|
||||
end
|
||||
end
|
||||
end
|
||||
115
test/microwaveprop/weather/sounding_test.exs
Normal file
115
test/microwaveprop/weather/sounding_test.exs
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
defmodule Microwaveprop.Weather.SoundingTest do
|
||||
use Microwaveprop.DataCase, async: true
|
||||
|
||||
alias Microwaveprop.Weather.Sounding
|
||||
alias Microwaveprop.Weather.Station
|
||||
|
||||
defp create_sounding_station do
|
||||
%Station{}
|
||||
|> Station.changeset(%{
|
||||
station_code: "FWD",
|
||||
station_type: "sounding",
|
||||
wmo_number: 72_249,
|
||||
name: "Fort Worth",
|
||||
lat: 32.83,
|
||||
lon: -97.30
|
||||
})
|
||||
|> Repo.insert!()
|
||||
end
|
||||
|
||||
@sample_profile [
|
||||
%{"pres" => 1013.0, "hght" => 171, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180, "sknt" => 10},
|
||||
%{"pres" => 925.0, "hght" => 800, "tmpc" => 20.0, "dwpc" => 12.0, "drct" => 200, "sknt" => 15},
|
||||
%{"pres" => 850.0, "hght" => 1500, "tmpc" => 15.0, "dwpc" => 8.0, "drct" => 220, "sknt" => 20}
|
||||
]
|
||||
|
||||
@valid_sounding_attrs %{
|
||||
observed_at: ~U[2026-03-28 12:00:00Z],
|
||||
profile: @sample_profile,
|
||||
level_count: 3,
|
||||
surface_pressure_mb: 1013.0,
|
||||
surface_temp_c: 25.0,
|
||||
surface_dewpoint_c: 15.0,
|
||||
surface_refractivity: 320.5,
|
||||
min_refractivity_gradient: -45.0,
|
||||
boundary_layer_depth_m: 1200.0,
|
||||
precipitable_water_mm: 25.0,
|
||||
k_index: 28.0,
|
||||
lifted_index: -2.0,
|
||||
ducting_detected: false,
|
||||
duct_characteristics: nil
|
||||
}
|
||||
|
||||
describe "changeset/2" do
|
||||
test "valid attributes with station_id" do
|
||||
station = create_sounding_station()
|
||||
attrs = Map.put(@valid_sounding_attrs, :station_id, station.id)
|
||||
changeset = Sounding.changeset(%Sounding{}, attrs)
|
||||
assert changeset.valid?
|
||||
end
|
||||
|
||||
test "requires station_id, observed_at, profile, level_count" do
|
||||
changeset = Sounding.changeset(%Sounding{}, %{})
|
||||
|
||||
assert %{
|
||||
station_id: ["can't be blank"],
|
||||
observed_at: ["can't be blank"],
|
||||
profile: ["can't be blank"],
|
||||
level_count: ["can't be blank"]
|
||||
} = errors_on(changeset)
|
||||
end
|
||||
|
||||
test "derived params are optional" do
|
||||
station = create_sounding_station()
|
||||
|
||||
attrs = %{
|
||||
station_id: station.id,
|
||||
observed_at: ~U[2026-03-28 12:00:00Z],
|
||||
profile: @sample_profile,
|
||||
level_count: 3
|
||||
}
|
||||
|
||||
changeset = Sounding.changeset(%Sounding{}, attrs)
|
||||
assert changeset.valid?
|
||||
end
|
||||
|
||||
test "persists to the database with JSONB profile" do
|
||||
station = create_sounding_station()
|
||||
attrs = Map.put(@valid_sounding_attrs, :station_id, station.id)
|
||||
changeset = Sounding.changeset(%Sounding{}, attrs)
|
||||
assert {:ok, sounding} = Repo.insert(changeset)
|
||||
assert length(sounding.profile) == 3
|
||||
assert hd(sounding.profile)["pres"] == 1013.0
|
||||
assert sounding.ducting_detected == false
|
||||
assert sounding.k_index == 28.0
|
||||
end
|
||||
|
||||
test "persists duct_characteristics as JSONB" do
|
||||
station = create_sounding_station()
|
||||
|
||||
duct_chars = [
|
||||
%{"base" => 200, "top" => 500, "strength" => "5.2"}
|
||||
]
|
||||
|
||||
attrs =
|
||||
@valid_sounding_attrs
|
||||
|> Map.put(:station_id, station.id)
|
||||
|> Map.put(:ducting_detected, true)
|
||||
|> Map.put(:duct_characteristics, duct_chars)
|
||||
|
||||
changeset = Sounding.changeset(%Sounding{}, attrs)
|
||||
assert {:ok, sounding} = Repo.insert(changeset)
|
||||
assert sounding.ducting_detected == true
|
||||
assert [%{"base" => 200, "top" => 500}] = sounding.duct_characteristics
|
||||
end
|
||||
|
||||
test "enforces unique station_id + observed_at" do
|
||||
station = create_sounding_station()
|
||||
attrs = Map.put(@valid_sounding_attrs, :station_id, station.id)
|
||||
|
||||
assert {:ok, _} = %Sounding{} |> Sounding.changeset(attrs) |> Repo.insert()
|
||||
assert {:error, changeset} = %Sounding{} |> Sounding.changeset(attrs) |> Repo.insert()
|
||||
assert %{station_id: ["has already been taken"]} = errors_on(changeset)
|
||||
end
|
||||
end
|
||||
end
|
||||
71
test/microwaveprop/weather/station_test.exs
Normal file
71
test/microwaveprop/weather/station_test.exs
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
defmodule Microwaveprop.Weather.StationTest do
|
||||
use Microwaveprop.DataCase, async: true
|
||||
|
||||
alias Microwaveprop.Weather.Station
|
||||
|
||||
@valid_attrs %{
|
||||
station_code: "KDFW",
|
||||
station_type: "asos",
|
||||
name: "Dallas/Fort Worth International",
|
||||
lat: 32.8998,
|
||||
lon: -97.0403,
|
||||
elevation_m: 171.0,
|
||||
state: "TX"
|
||||
}
|
||||
|
||||
describe "changeset/2" do
|
||||
test "valid attributes" do
|
||||
changeset = Station.changeset(%Station{}, @valid_attrs)
|
||||
assert changeset.valid?
|
||||
end
|
||||
|
||||
test "valid sounding station with wmo_number" do
|
||||
attrs = %{
|
||||
station_code: "FWD",
|
||||
station_type: "sounding",
|
||||
wmo_number: 72_249,
|
||||
name: "Fort Worth",
|
||||
lat: 32.83,
|
||||
lon: -97.30
|
||||
}
|
||||
|
||||
changeset = Station.changeset(%Station{}, attrs)
|
||||
assert changeset.valid?
|
||||
end
|
||||
|
||||
test "requires station_code, station_type, name, lat, lon" do
|
||||
changeset = Station.changeset(%Station{}, %{})
|
||||
|
||||
assert %{
|
||||
station_code: ["can't be blank"],
|
||||
station_type: ["can't be blank"],
|
||||
name: ["can't be blank"],
|
||||
lat: ["can't be blank"],
|
||||
lon: ["can't be blank"]
|
||||
} = errors_on(changeset)
|
||||
end
|
||||
|
||||
test "validates station_type inclusion" do
|
||||
changeset = Station.changeset(%Station{}, %{@valid_attrs | station_type: "radar"})
|
||||
assert %{station_type: ["is invalid"]} = errors_on(changeset)
|
||||
end
|
||||
|
||||
test "persists to the database" do
|
||||
changeset = Station.changeset(%Station{}, @valid_attrs)
|
||||
assert {:ok, station} = Repo.insert(changeset)
|
||||
assert station.station_code == "KDFW"
|
||||
assert station.station_type == "asos"
|
||||
assert station.lat == 32.8998
|
||||
assert station.lon == -97.0403
|
||||
end
|
||||
|
||||
test "enforces unique station_code + station_type" do
|
||||
changeset = Station.changeset(%Station{}, @valid_attrs)
|
||||
assert {:ok, _} = Repo.insert(changeset)
|
||||
|
||||
duplicate = Station.changeset(%Station{}, @valid_attrs)
|
||||
assert {:error, changeset} = Repo.insert(duplicate)
|
||||
assert %{station_code: ["has already been taken"]} = errors_on(changeset)
|
||||
end
|
||||
end
|
||||
end
|
||||
79
test/microwaveprop/weather/surface_observation_test.exs
Normal file
79
test/microwaveprop/weather/surface_observation_test.exs
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
defmodule Microwaveprop.Weather.SurfaceObservationTest do
|
||||
use Microwaveprop.DataCase, async: true
|
||||
|
||||
alias Microwaveprop.Weather.Station
|
||||
alias Microwaveprop.Weather.SurfaceObservation
|
||||
|
||||
defp create_station do
|
||||
%Station{}
|
||||
|> Station.changeset(%{
|
||||
station_code: "KDFW",
|
||||
station_type: "asos",
|
||||
name: "Dallas/Fort Worth International",
|
||||
lat: 32.8998,
|
||||
lon: -97.0403
|
||||
})
|
||||
|> Repo.insert!()
|
||||
end
|
||||
|
||||
@valid_obs_attrs %{
|
||||
observed_at: ~U[2026-03-28 18:00:00Z],
|
||||
temp_f: 75.0,
|
||||
dewpoint_f: 55.0,
|
||||
relative_humidity: 49.0,
|
||||
wind_speed_kts: 12.0,
|
||||
wind_direction_deg: 180,
|
||||
sea_level_pressure_mb: 1013.25,
|
||||
altimeter_setting: 29.92,
|
||||
sky_condition: "SCT"
|
||||
}
|
||||
|
||||
describe "changeset/2" do
|
||||
test "valid attributes with station_id" do
|
||||
station = create_station()
|
||||
attrs = Map.put(@valid_obs_attrs, :station_id, station.id)
|
||||
changeset = SurfaceObservation.changeset(%SurfaceObservation{}, attrs)
|
||||
assert changeset.valid?
|
||||
end
|
||||
|
||||
test "requires station_id and observed_at" do
|
||||
changeset = SurfaceObservation.changeset(%SurfaceObservation{}, %{})
|
||||
|
||||
assert %{
|
||||
station_id: ["can't be blank"],
|
||||
observed_at: ["can't be blank"]
|
||||
} = errors_on(changeset)
|
||||
end
|
||||
|
||||
test "all weather fields are optional" do
|
||||
station = create_station()
|
||||
|
||||
attrs = %{
|
||||
station_id: station.id,
|
||||
observed_at: ~U[2026-03-28 18:00:00Z]
|
||||
}
|
||||
|
||||
changeset = SurfaceObservation.changeset(%SurfaceObservation{}, attrs)
|
||||
assert changeset.valid?
|
||||
end
|
||||
|
||||
test "persists to the database" do
|
||||
station = create_station()
|
||||
attrs = Map.put(@valid_obs_attrs, :station_id, station.id)
|
||||
changeset = SurfaceObservation.changeset(%SurfaceObservation{}, attrs)
|
||||
assert {:ok, obs} = Repo.insert(changeset)
|
||||
assert obs.temp_f == 75.0
|
||||
assert obs.sky_condition == "SCT"
|
||||
assert obs.station_id == station.id
|
||||
end
|
||||
|
||||
test "enforces unique station_id + observed_at" do
|
||||
station = create_station()
|
||||
attrs = Map.put(@valid_obs_attrs, :station_id, station.id)
|
||||
|
||||
assert {:ok, _} = %SurfaceObservation{} |> SurfaceObservation.changeset(attrs) |> Repo.insert()
|
||||
assert {:error, changeset} = %SurfaceObservation{} |> SurfaceObservation.changeset(attrs) |> Repo.insert()
|
||||
assert %{station_id: ["has already been taken"]} = errors_on(changeset)
|
||||
end
|
||||
end
|
||||
end
|
||||
199
test/microwaveprop/weather_test.exs
Normal file
199
test/microwaveprop/weather_test.exs
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
defmodule Microwaveprop.WeatherTest do
|
||||
use Microwaveprop.DataCase, async: true
|
||||
|
||||
alias Microwaveprop.Weather
|
||||
|
||||
@station_attrs %{
|
||||
station_code: "KDFW",
|
||||
station_type: "asos",
|
||||
name: "Dallas/Fort Worth International",
|
||||
lat: 32.8998,
|
||||
lon: -97.0403,
|
||||
elevation_m: 171.0,
|
||||
state: "TX"
|
||||
}
|
||||
|
||||
@sounding_station_attrs %{
|
||||
station_code: "FWD",
|
||||
station_type: "sounding",
|
||||
wmo_number: 72_249,
|
||||
name: "Fort Worth",
|
||||
lat: 32.83,
|
||||
lon: -97.30
|
||||
}
|
||||
|
||||
describe "find_or_create_station/1" do
|
||||
test "creates a new station" do
|
||||
assert {:ok, station} = Weather.find_or_create_station(@station_attrs)
|
||||
assert station.station_code == "KDFW"
|
||||
assert station.station_type == "asos"
|
||||
end
|
||||
|
||||
test "returns existing station if already created" do
|
||||
{:ok, first} = Weather.find_or_create_station(@station_attrs)
|
||||
{:ok, second} = Weather.find_or_create_station(@station_attrs)
|
||||
assert first.id == second.id
|
||||
end
|
||||
|
||||
test "returns error for invalid attributes" do
|
||||
assert {:error, _changeset} = Weather.find_or_create_station(%{station_code: "X"})
|
||||
end
|
||||
end
|
||||
|
||||
describe "upsert_surface_observation/2" do
|
||||
test "inserts a new observation" do
|
||||
{:ok, station} = Weather.find_or_create_station(@station_attrs)
|
||||
|
||||
obs_attrs = %{
|
||||
observed_at: ~U[2026-03-28 18:00:00Z],
|
||||
temp_f: 75.0,
|
||||
dewpoint_f: 55.0,
|
||||
sky_condition: "SCT"
|
||||
}
|
||||
|
||||
assert {:ok, obs} = Weather.upsert_surface_observation(station, obs_attrs)
|
||||
assert obs.temp_f == 75.0
|
||||
assert obs.station_id == station.id
|
||||
end
|
||||
|
||||
test "updates existing observation on conflict" do
|
||||
{:ok, station} = Weather.find_or_create_station(@station_attrs)
|
||||
|
||||
obs_attrs = %{
|
||||
observed_at: ~U[2026-03-28 18:00:00Z],
|
||||
temp_f: 75.0,
|
||||
sky_condition: "SCT"
|
||||
}
|
||||
|
||||
{:ok, first} = Weather.upsert_surface_observation(station, obs_attrs)
|
||||
{:ok, second} = Weather.upsert_surface_observation(station, %{obs_attrs | temp_f: 78.0})
|
||||
|
||||
assert first.id == second.id
|
||||
assert second.temp_f == 78.0
|
||||
end
|
||||
end
|
||||
|
||||
describe "upsert_sounding/2" do
|
||||
@sample_profile [
|
||||
%{"pres" => 1013.0, "hght" => 171, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180, "sknt" => 10},
|
||||
%{"pres" => 925.0, "hght" => 800, "tmpc" => 20.0, "dwpc" => 12.0, "drct" => 200, "sknt" => 15},
|
||||
%{"pres" => 850.0, "hght" => 1500, "tmpc" => 15.0, "dwpc" => 8.0, "drct" => 220, "sknt" => 20}
|
||||
]
|
||||
|
||||
test "inserts a new sounding" do
|
||||
{:ok, station} = Weather.find_or_create_station(@sounding_station_attrs)
|
||||
|
||||
sounding_attrs = %{
|
||||
observed_at: ~U[2026-03-28 12:00:00Z],
|
||||
profile: @sample_profile,
|
||||
level_count: 3,
|
||||
surface_temp_c: 25.0
|
||||
}
|
||||
|
||||
assert {:ok, sounding} = Weather.upsert_sounding(station, sounding_attrs)
|
||||
assert sounding.level_count == 3
|
||||
assert sounding.station_id == station.id
|
||||
end
|
||||
|
||||
test "updates existing sounding on conflict" do
|
||||
{:ok, station} = Weather.find_or_create_station(@sounding_station_attrs)
|
||||
|
||||
sounding_attrs = %{
|
||||
observed_at: ~U[2026-03-28 12:00:00Z],
|
||||
profile: @sample_profile,
|
||||
level_count: 3,
|
||||
surface_temp_c: 25.0
|
||||
}
|
||||
|
||||
{:ok, first} = Weather.upsert_sounding(station, sounding_attrs)
|
||||
{:ok, second} = Weather.upsert_sounding(station, %{sounding_attrs | surface_temp_c: 26.0})
|
||||
|
||||
assert first.id == second.id
|
||||
assert second.surface_temp_c == 26.0
|
||||
end
|
||||
end
|
||||
|
||||
describe "weather_for_qso/2" do
|
||||
test "returns nearby surface observations within time window" do
|
||||
{:ok, station} = Weather.find_or_create_station(@station_attrs)
|
||||
|
||||
{:ok, _obs} =
|
||||
Weather.upsert_surface_observation(station, %{
|
||||
observed_at: ~U[2026-03-28 18:00:00Z],
|
||||
temp_f: 75.0
|
||||
})
|
||||
|
||||
# QSO near the station in space and time
|
||||
result =
|
||||
Weather.weather_for_qso(
|
||||
%{lat: 32.90, lon: -97.05, timestamp: ~U[2026-03-28 18:30:00Z]},
|
||||
radius_km: 100,
|
||||
time_window_hours: 3
|
||||
)
|
||||
|
||||
assert length(result.surface_observations) == 1
|
||||
assert hd(result.surface_observations).temp_f == 75.0
|
||||
end
|
||||
|
||||
test "returns nearby soundings within time window" do
|
||||
{:ok, station} = Weather.find_or_create_station(@sounding_station_attrs)
|
||||
|
||||
{:ok, _sounding} =
|
||||
Weather.upsert_sounding(station, %{
|
||||
observed_at: ~U[2026-03-28 12:00:00Z],
|
||||
profile: [%{"pres" => 1013.0, "hght" => 171, "tmpc" => 25.0, "dwpc" => 15.0, "drct" => 180, "sknt" => 10}],
|
||||
level_count: 1,
|
||||
surface_temp_c: 25.0
|
||||
})
|
||||
|
||||
result =
|
||||
Weather.weather_for_qso(
|
||||
%{lat: 32.85, lon: -97.28, timestamp: ~U[2026-03-28 14:00:00Z]},
|
||||
radius_km: 100,
|
||||
time_window_hours: 6
|
||||
)
|
||||
|
||||
assert length(result.soundings) == 1
|
||||
end
|
||||
|
||||
test "excludes observations outside radius" do
|
||||
{:ok, station} = Weather.find_or_create_station(@station_attrs)
|
||||
|
||||
{:ok, _obs} =
|
||||
Weather.upsert_surface_observation(station, %{
|
||||
observed_at: ~U[2026-03-28 18:00:00Z],
|
||||
temp_f: 75.0
|
||||
})
|
||||
|
||||
# QSO far away
|
||||
result =
|
||||
Weather.weather_for_qso(
|
||||
%{lat: 40.0, lon: -80.0, timestamp: ~U[2026-03-28 18:00:00Z]},
|
||||
radius_km: 100,
|
||||
time_window_hours: 3
|
||||
)
|
||||
|
||||
assert result.surface_observations == []
|
||||
end
|
||||
|
||||
test "excludes observations outside time window" do
|
||||
{:ok, station} = Weather.find_or_create_station(@station_attrs)
|
||||
|
||||
{:ok, _obs} =
|
||||
Weather.upsert_surface_observation(station, %{
|
||||
observed_at: ~U[2026-03-28 06:00:00Z],
|
||||
temp_f: 75.0
|
||||
})
|
||||
|
||||
# QSO much later
|
||||
result =
|
||||
Weather.weather_for_qso(
|
||||
%{lat: 32.90, lon: -97.05, timestamp: ~U[2026-03-28 22:00:00Z]},
|
||||
radius_km: 100,
|
||||
time_window_hours: 3
|
||||
)
|
||||
|
||||
assert result.surface_observations == []
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Reference in a new issue