Graham McIntire
6489f08138
Store surface observations (ASOS) and upper-air soundings (RAOB) alongside QSOs for atmospheric propagation correlation. Three new tables: weather_stations, surface_observations, and soundings with JSONB profiles and pre-computed derived parameters (refractivity, gradients, duct detection, stability indices). Includes IEM API client for historical data import and import script seeded with 95 ASOS + 9 sounding stations from PropCast coverage area.
320 lines
16 KiB
Elixir
320 lines
16 KiB
Elixir
alias Microwaveprop.{Repo, Weather}
|
|
alias Microwaveprop.Weather.{IemClient, SoundingParams}
|
|
|
|
import Ecto.Query
|
|
|
|
# --- Station definitions (from PropCast) ---
|
|
|
|
asos_stations = [
|
|
%{station_code: "KDFW", name: "Dallas/Fort Worth Intl", lat: 32.897, lon: -97.038, state: "TX"},
|
|
%{station_code: "KDAL", name: "Dallas Love Field", lat: 32.847, lon: -96.851, state: "TX"},
|
|
%{station_code: "KADS", name: "Addison", lat: 32.969, lon: -96.836, state: "TX"},
|
|
%{station_code: "KTKI", name: "McKinney National", lat: 33.178, lon: -96.590, state: "TX"},
|
|
%{station_code: "KGPM", name: "Grand Prairie Muni", lat: 32.698, lon: -97.043, state: "TX"},
|
|
%{station_code: "KFWS", name: "Fort Worth Spinks", lat: 32.565, lon: -97.308, state: "TX"},
|
|
%{station_code: "KFTW", name: "Fort Worth Meacham", lat: 32.820, lon: -97.362, state: "TX"},
|
|
%{station_code: "KAFW", name: "Fort Worth Alliance", lat: 32.988, lon: -97.319, state: "TX"},
|
|
%{station_code: "KRBD", name: "Dallas Executive", lat: 32.681, lon: -96.868, state: "TX"},
|
|
%{station_code: "KGKY", name: "Arlington Muni", lat: 32.664, lon: -97.094, state: "TX"},
|
|
%{station_code: "KCPT", name: "Cleburne Regional", lat: 32.354, lon: -97.434, state: "TX"},
|
|
%{station_code: "KHQZ", name: "Mesquite Metro", lat: 32.747, lon: -96.530, state: "TX"},
|
|
%{station_code: "KGYI", name: "Sherman/Denison", lat: 33.729, lon: -96.669, state: "TX"},
|
|
%{station_code: "KGLE", name: "Gainesville Muni", lat: 33.651, lon: -97.197, state: "TX"},
|
|
%{station_code: "KGVT", name: "Greenville/Hunt Co", lat: 33.068, lon: -96.065, state: "TX"},
|
|
%{station_code: "KJDD", name: "Sulphur Springs", lat: 33.152, lon: -95.623, state: "TX"},
|
|
%{station_code: "KGDJ", name: "Granbury Regional", lat: 32.445, lon: -97.817, state: "TX"},
|
|
%{station_code: "KMWL", name: "Mineral Wells", lat: 32.782, lon: -98.060, state: "TX"},
|
|
%{station_code: "KBKD", name: "Breckenridge/Stephens Co", lat: 32.719, lon: -98.890, state: "TX"},
|
|
%{station_code: "KWEA", name: "Weatherford/Parker Co", lat: 32.746, lon: -97.682, state: "TX"},
|
|
%{station_code: "KGGG", name: "Longview East Texas", lat: 32.384, lon: -94.711, state: "TX"},
|
|
%{station_code: "KTYR", name: "Tyler Pounds", lat: 32.354, lon: -95.402, state: "TX"},
|
|
%{station_code: "KCRS", name: "Corsicana/Navarro Co", lat: 32.028, lon: -96.400, state: "TX"},
|
|
%{station_code: "KWXD", name: "Waxahachie/Ellis Co", lat: 32.392, lon: -96.849, state: "TX"},
|
|
%{station_code: "KPNX", name: "Henderson/Rusk Co", lat: 32.152, lon: -94.913, state: "TX"},
|
|
%{station_code: "KPWG", name: "Waco Regional", lat: 31.611, lon: -97.095, state: "TX"},
|
|
%{station_code: "KACT", name: "Waco TSTC", lat: 31.611, lon: -97.226, state: "TX"},
|
|
%{station_code: "KTPL", name: "Temple/Draughon-Miller", lat: 31.152, lon: -97.408, state: "TX"},
|
|
%{station_code: "KILE", name: "Killeen/Fort Cavazos", lat: 31.086, lon: -97.683, state: "TX"},
|
|
%{station_code: "KGRK", name: "Killeen-Fort Hood", lat: 31.067, lon: -97.829, state: "TX"},
|
|
%{station_code: "KAUS", name: "Austin-Bergstrom", lat: 30.198, lon: -97.670, state: "TX"},
|
|
%{station_code: "KSAT", name: "San Antonio Intl", lat: 29.534, lon: -98.470, state: "TX"},
|
|
%{station_code: "KBMQ", name: "Burnet Muni", lat: 30.739, lon: -98.239, state: "TX"},
|
|
%{station_code: "KSSF", name: "San Antonio Stinson", lat: 29.337, lon: -98.471, state: "TX"},
|
|
%{station_code: "KGTU", name: "Georgetown Muni", lat: 30.678, lon: -97.679, state: "TX"},
|
|
%{station_code: "KHYI", name: "San Marcos Rgnl", lat: 29.893, lon: -97.863, state: "TX"},
|
|
%{station_code: "KHOU", name: "Houston Hobby", lat: 29.645, lon: -95.279, state: "TX"},
|
|
%{station_code: "KIAH", name: "Houston Bush Intcntl", lat: 29.980, lon: -95.342, state: "TX"},
|
|
%{station_code: "KCXO", name: "Conroe North Houston", lat: 30.352, lon: -95.415, state: "TX"},
|
|
%{station_code: "KBPT", name: "Beaumont/Port Arthur", lat: 29.951, lon: -94.021, state: "TX"},
|
|
%{station_code: "KLCH", name: "Lake Charles Regional", lat: 30.126, lon: -93.223, state: "LA"},
|
|
%{station_code: "KVCT", name: "Victoria Regional", lat: 28.852, lon: -96.918, state: "TX"},
|
|
%{station_code: "KCRP", name: "Corpus Christi Intl", lat: 27.770, lon: -97.502, state: "TX"},
|
|
%{station_code: "KOKC", name: "Oklahoma City Will Rogers", lat: 35.393, lon: -97.601, state: "OK"},
|
|
%{station_code: "KOUN", name: "Norman/OU", lat: 35.246, lon: -97.472, state: "OK"},
|
|
%{station_code: "KPWA", name: "Wiley Post", lat: 35.534, lon: -97.647, state: "OK"},
|
|
%{station_code: "KSWO", name: "Stillwater Regional", lat: 36.161, lon: -97.086, state: "OK"},
|
|
%{station_code: "KTIK", name: "Tinker AFB", lat: 35.415, lon: -97.387, state: "OK"},
|
|
%{station_code: "KLAW", name: "Lawton/Fort Sill", lat: 34.568, lon: -98.416, state: "OK"},
|
|
%{station_code: "KADU", name: "Ardmore Downtown", lat: 34.303, lon: -97.019, state: "OK"},
|
|
%{station_code: "KSNL", name: "Shawnee Regional", lat: 35.358, lon: -96.943, state: "OK"},
|
|
%{station_code: "KCSM", name: "Clinton/Sherman", lat: 35.339, lon: -99.201, state: "OK"},
|
|
%{station_code: "KGAG", name: "Gage/Ellis Co", lat: 36.296, lon: -99.777, state: "OK"},
|
|
%{station_code: "KWDG", name: "Enid Woodring", lat: 36.379, lon: -97.791, state: "OK"},
|
|
%{station_code: "KBVO", name: "Bartlesville", lat: 36.765, lon: -96.012, state: "OK"},
|
|
%{station_code: "KTUL", name: "Tulsa Intl", lat: 36.198, lon: -95.888, state: "OK"},
|
|
%{station_code: "KMKO", name: "Muskogee/Davis Field", lat: 35.657, lon: -95.367, state: "OK"},
|
|
%{station_code: "KPOC", name: "Pontotoc Co/Ada", lat: 34.802, lon: -96.671, state: "OK"},
|
|
%{station_code: "KGMJ", name: "Grove Muni", lat: 36.607, lon: -94.733, state: "OK"},
|
|
%{station_code: "KABI", name: "Abilene Regional", lat: 32.411, lon: -99.682, state: "TX"},
|
|
%{station_code: "KMAF", name: "Midland Intl", lat: 31.943, lon: -102.201, state: "TX"},
|
|
%{station_code: "KBPG", name: "Big Spring McMahon", lat: 32.213, lon: -101.522, state: "TX"},
|
|
%{station_code: "KSNS", name: "San Angelo Mathis", lat: 31.358, lon: -100.497, state: "TX"},
|
|
%{station_code: "KLBB", name: "Lubbock Preston Smith", lat: 33.664, lon: -101.823, state: "TX"},
|
|
%{station_code: "KAMA", name: "Amarillo Rick Husband", lat: 35.220, lon: -101.706, state: "TX"},
|
|
%{station_code: "KPPA", name: "Pampa/Perry Lefors", lat: 35.613, lon: -100.996, state: "TX"},
|
|
%{station_code: "KPVW", name: "Plainview Hale Co", lat: 34.183, lon: -101.722, state: "TX"},
|
|
%{station_code: "KINK", name: "Wink/Winkler Co", lat: 31.779, lon: -103.201, state: "TX"},
|
|
%{station_code: "KELP", name: "El Paso Intl", lat: 31.807, lon: -106.378, state: "TX"},
|
|
%{station_code: "KCNM", name: "Carlsbad Cavern City", lat: 32.337, lon: -104.263, state: "NM"},
|
|
%{station_code: "KROW", name: "Roswell Intl Air Ctr", lat: 33.302, lon: -104.531, state: "NM"},
|
|
%{station_code: "KCLO", name: "Clovis Muni", lat: 34.434, lon: -103.079, state: "NM"},
|
|
%{station_code: "KHOB", name: "Hobbs Lea Co", lat: 32.688, lon: -103.217, state: "NM"},
|
|
%{station_code: "KTCC", name: "Tucumcari Muni", lat: 35.183, lon: -103.603, state: "NM"},
|
|
%{station_code: "KICT", name: "Wichita Dwight D. Eisenhower", lat: 37.650, lon: -97.433, state: "KS"},
|
|
%{station_code: "KHUT", name: "Hutchinson Regional", lat: 38.065, lon: -97.861, state: "KS"},
|
|
%{station_code: "KDDC", name: "Dodge City Regional", lat: 37.763, lon: -99.965, state: "KS"},
|
|
%{station_code: "KGLD", name: "Goodland Municipal", lat: 39.370, lon: -101.700, state: "KS"},
|
|
%{station_code: "KGBD", name: "Great Bend Municipal", lat: 38.344, lon: -98.859, state: "KS"},
|
|
%{station_code: "KCNU", name: "Chanute Martin Johnson", lat: 37.669, lon: -95.485, state: "KS"},
|
|
%{station_code: "KPTT", name: "Pratt Regional", lat: 37.700, lon: -98.747, state: "KS"},
|
|
%{station_code: "KSGF", name: "Springfield Branson Natl", lat: 37.246, lon: -93.389, state: "MO"},
|
|
%{station_code: "KJLN", name: "Joplin Regional", lat: 37.152, lon: -94.498, state: "MO"},
|
|
%{station_code: "KFSM", name: "Fort Smith Regional", lat: 35.337, lon: -94.368, state: "AR"},
|
|
%{station_code: "KTXK", name: "Texarkana Regional", lat: 33.454, lon: -93.991, state: "TX"},
|
|
%{station_code: "KLIT", name: "Little Rock Clinton", lat: 34.729, lon: -92.224, state: "AR"},
|
|
%{station_code: "KHRO", name: "Harrison Boone Co", lat: 36.261, lon: -93.155, state: "AR"},
|
|
%{station_code: "KFYV", name: "Fayetteville Drake Field", lat: 36.005, lon: -94.170, state: "AR"},
|
|
%{station_code: "KXNA", name: "NW Arkansas Natl", lat: 36.282, lon: -94.307, state: "AR"},
|
|
%{station_code: "KSHV", name: "Shreveport Regional", lat: 32.447, lon: -93.826, state: "LA"},
|
|
%{station_code: "KMLU", name: "Monroe Regional", lat: 32.511, lon: -92.038, state: "LA"},
|
|
%{station_code: "KJAN", name: "Jackson Medgar Wiley Evers", lat: 32.311, lon: -90.076, state: "MS"},
|
|
%{station_code: "KMEI", name: "Meridian Regional", lat: 32.334, lon: -88.752, state: "MS"},
|
|
%{station_code: "KNAC", name: "Nacogdoches/Mangham", lat: 31.578, lon: -94.710, state: "TX"},
|
|
%{station_code: "KNEW", name: "New Orleans Lakefront", lat: 30.042, lon: -90.028, state: "LA"},
|
|
%{station_code: "KMSY", name: "New Orleans Armstrong", lat: 29.993, lon: -90.258, state: "LA"}
|
|
]
|
|
|
|
sounding_stations = [
|
|
%{station_code: "FWD", wmo_number: 72_249, name: "Fort Worth TX", lat: 32.835, lon: -97.298, state: "TX"},
|
|
%{station_code: "OUN", wmo_number: 72_357, name: "Oklahoma City OK", lat: 35.181, lon: -97.436, state: "OK"},
|
|
%{station_code: "SHV", wmo_number: 72_248, name: "Shreveport LA", lat: 32.450, lon: -93.841, state: "LA"},
|
|
%{station_code: "AMA", wmo_number: 72_363, name: "Amarillo TX", lat: 35.233, lon: -101.706, state: "TX"},
|
|
%{station_code: "LCH", wmo_number: 72_240, name: "Lake Charles LA", lat: 30.125, lon: -93.217, state: "LA"},
|
|
%{station_code: "MAF", wmo_number: 72_265, name: "Midland TX", lat: 31.943, lon: -102.189, state: "TX"},
|
|
%{station_code: "LZK", wmo_number: 72_340, name: "Little Rock AR", lat: 34.836, lon: -92.263, state: "AR"},
|
|
%{station_code: "BRO", wmo_number: 72_250, name: "Brownsville TX", lat: 25.910, lon: -97.419, state: "TX"},
|
|
%{station_code: "SPS", wmo_number: 72_451, name: "Wichita Falls TX", lat: 33.989, lon: -98.492, state: "TX"}
|
|
]
|
|
|
|
# --- Helpers ---
|
|
|
|
defmodule ImportHelpers do
|
|
@moduledoc false
|
|
|
|
@km_per_deg 111.0
|
|
|
|
def nearest_stations(lat, lon, stations, radius_km) do
|
|
Enum.filter(stations, fn s ->
|
|
dlat = (s.lat - lat) * @km_per_deg
|
|
dlon = (s.lon - lon) * @km_per_deg * :math.cos(lat * :math.pi() / 180)
|
|
:math.sqrt(dlat * dlat + dlon * dlon) <= radius_km
|
|
end)
|
|
end
|
|
|
|
def sounding_times_around(dt) do
|
|
# Return the 00Z and 12Z sounding times that bracket the QSO time
|
|
date = DateTime.to_date(dt)
|
|
hour = dt.hour
|
|
|
|
times =
|
|
if hour < 12 do
|
|
[
|
|
DateTime.new!(Date.add(date, -1), ~T[12:00:00], "Etc/UTC"),
|
|
DateTime.new!(date, ~T[00:00:00], "Etc/UTC")
|
|
]
|
|
else
|
|
[
|
|
DateTime.new!(date, ~T[00:00:00], "Etc/UTC"),
|
|
DateTime.new!(date, ~T[12:00:00], "Etc/UTC")
|
|
]
|
|
end
|
|
|
|
Enum.uniq(times)
|
|
end
|
|
|
|
def rate_limit(delay_ms \\ 200) do
|
|
Process.sleep(delay_ms)
|
|
end
|
|
end
|
|
|
|
# --- Seed stations ---
|
|
|
|
IO.puts("Seeding #{length(asos_stations)} ASOS stations...")
|
|
|
|
asos_station_map =
|
|
Enum.reduce(asos_stations, %{}, fn attrs, acc ->
|
|
{:ok, station} =
|
|
Weather.find_or_create_station(Map.put(attrs, :station_type, "asos"))
|
|
|
|
Map.put(acc, station.station_code, station)
|
|
end)
|
|
|
|
IO.puts("Seeding #{length(sounding_stations)} sounding stations...")
|
|
|
|
sounding_station_map =
|
|
Enum.reduce(sounding_stations, %{}, fn attrs, acc ->
|
|
{:ok, station} =
|
|
Weather.find_or_create_station(Map.put(attrs, :station_type, "sounding"))
|
|
|
|
Map.put(acc, station.station_code, station)
|
|
end)
|
|
|
|
IO.puts("Stations seeded: #{map_size(asos_station_map)} ASOS, #{map_size(sounding_station_map)} sounding")
|
|
|
|
# --- Fetch weather for each QSO ---
|
|
|
|
qsos =
|
|
Microwaveprop.Radio.Qso
|
|
|> where([q], not is_nil(q.pos1))
|
|
|> order_by([q], q.qso_timestamp)
|
|
|> Repo.all()
|
|
|
|
IO.puts("Processing #{length(qsos)} QSOs with positions...")
|
|
|
|
# Track which station+time combos we've already fetched to avoid duplicates
|
|
asos_fetched = :ets.new(:asos_fetched, [:set, :public])
|
|
raob_fetched = :ets.new(:raob_fetched, [:set, :public])
|
|
|
|
asos_station_list = Enum.map(asos_stations, fn s -> Map.put(s, :station_type, "asos") end)
|
|
|
|
total = length(qsos)
|
|
|
|
qsos
|
|
|> Enum.with_index(1)
|
|
|> Enum.each(fn {qso, idx} ->
|
|
lat = qso.pos1["lat"] || qso.pos1["latitude"]
|
|
lon = qso.pos1["lng"] || qso.pos1["lon"] || qso.pos1["longitude"]
|
|
|
|
if lat && lon do
|
|
if rem(idx, 100) == 0, do: IO.puts(" QSO #{idx}/#{total}")
|
|
|
|
# Find nearby ASOS stations (within 150km)
|
|
nearby_asos = ImportHelpers.nearest_stations(lat, lon, asos_station_list, 150)
|
|
|
|
Enum.each(nearby_asos, fn s ->
|
|
# Fetch a 4-hour window around QSO time
|
|
start_dt = DateTime.add(qso.qso_timestamp, -2 * 3600, :second)
|
|
end_dt = DateTime.add(qso.qso_timestamp, 2 * 3600, :second)
|
|
key = {s.station_code, DateTime.to_date(start_dt), div(start_dt.hour, 4)}
|
|
|
|
unless :ets.member(asos_fetched, key) do
|
|
:ets.insert(asos_fetched, {key, true})
|
|
|
|
station = Map.get(asos_station_map, s.station_code)
|
|
|
|
if station do
|
|
case IemClient.fetch_asos(s.station_code, start_dt, end_dt) do
|
|
{:ok, rows} ->
|
|
Enum.each(rows, fn row ->
|
|
if row.observed_at do
|
|
Weather.upsert_surface_observation(station, row)
|
|
end
|
|
end)
|
|
|
|
{:error, reason} ->
|
|
IO.puts(" ASOS error #{s.station_code}: #{inspect(reason)}")
|
|
end
|
|
|
|
ImportHelpers.rate_limit()
|
|
end
|
|
end
|
|
end)
|
|
|
|
# Find nearby sounding stations (within 300km)
|
|
nearby_soundings = ImportHelpers.nearest_stations(lat, lon, sounding_stations, 300)
|
|
|
|
Enum.each(nearby_soundings, fn s ->
|
|
sounding_times = ImportHelpers.sounding_times_around(qso.qso_timestamp)
|
|
|
|
Enum.each(sounding_times, fn sounding_time ->
|
|
key = {s.station_code, sounding_time}
|
|
|
|
unless :ets.member(raob_fetched, key) do
|
|
:ets.insert(raob_fetched, {key, true})
|
|
|
|
station = Map.get(sounding_station_map, s.station_code)
|
|
|
|
if station do
|
|
case IemClient.fetch_raob(s.station_code, sounding_time) do
|
|
{:ok, [parsed | _]} ->
|
|
params = SoundingParams.derive(parsed.profile)
|
|
|
|
sounding_attrs =
|
|
if params do
|
|
%{
|
|
observed_at: parsed.observed_at,
|
|
profile: parsed.profile,
|
|
level_count: params.level_count,
|
|
surface_pressure_mb: params.surface_pressure_mb,
|
|
surface_temp_c: params.surface_temp_c,
|
|
surface_dewpoint_c: params.surface_dewpoint_c,
|
|
surface_refractivity: params.surface_refractivity,
|
|
min_refractivity_gradient: params.min_refractivity_gradient,
|
|
boundary_layer_depth_m: params.boundary_layer_depth_m,
|
|
precipitable_water_mm: params.precipitable_water_mm,
|
|
k_index: params.k_index,
|
|
lifted_index: params.lifted_index,
|
|
ducting_detected: params.ducting_detected,
|
|
duct_characteristics: params.duct_characteristics
|
|
}
|
|
else
|
|
%{
|
|
observed_at: parsed.observed_at,
|
|
profile: parsed.profile,
|
|
level_count: length(parsed.profile)
|
|
}
|
|
end
|
|
|
|
Weather.upsert_sounding(station, sounding_attrs)
|
|
|
|
{:ok, []} ->
|
|
:ok
|
|
|
|
{:error, reason} ->
|
|
IO.puts(" RAOB error #{s.station_code}: #{inspect(reason)}")
|
|
end
|
|
|
|
ImportHelpers.rate_limit()
|
|
end
|
|
end
|
|
end)
|
|
end)
|
|
end
|
|
end)
|
|
|
|
:ets.delete(asos_fetched)
|
|
:ets.delete(raob_fetched)
|
|
|
|
# --- Summary ---
|
|
|
|
asos_count = Repo.aggregate(Microwaveprop.Weather.SurfaceObservation, :count)
|
|
sounding_count = Repo.aggregate(Microwaveprop.Weather.Sounding, :count)
|
|
station_count = Repo.aggregate(Microwaveprop.Weather.Station, :count)
|
|
|
|
IO.puts("""
|
|
|
|
Import complete:
|
|
Stations: #{station_count}
|
|
Surface observations: #{asos_count}
|
|
Soundings: #{sounding_count}
|
|
""")
|