New AsosAdjustmentWorker runs every 10 minutes: - Fetches latest ASOS observations from all ~2900 US stations via IEM bulk currents API (parallel fetch across 51 state networks) - For each grid point within 75km of a reporting station, re-scores using fresh ASOS data (temp, dewpoint, wind, sky, pressure, precip) with HRRR refractivity gradient from the last hourly computation - Pushes updated scores to the map via PubSub Also stores HRRR profiles in the database during grid computation so the data persists for reference and ASOS blending.
307 lines
8.1 KiB
Elixir
307 lines
8.1 KiB
Elixir
defmodule Microwaveprop.Weather do
|
|
@moduledoc false
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Weather.HrrrProfile
|
|
alias Microwaveprop.Weather.IemreObservation
|
|
alias Microwaveprop.Weather.SolarIndex
|
|
alias Microwaveprop.Weather.Sounding
|
|
alias Microwaveprop.Weather.Station
|
|
alias Microwaveprop.Weather.SurfaceObservation
|
|
|
|
# Approximate km per degree latitude
|
|
@km_per_deg_lat 111.0
|
|
|
|
def find_or_create_station(attrs) do
|
|
code = attrs[:station_code] || attrs["station_code"]
|
|
type = attrs[:station_type] || attrs["station_type"]
|
|
|
|
if code && type do
|
|
case Repo.get_by(Station, station_code: code, station_type: type) do
|
|
nil ->
|
|
%Station{}
|
|
|> Station.changeset(attrs)
|
|
|> Repo.insert()
|
|
|
|
station ->
|
|
{:ok, station}
|
|
end
|
|
else
|
|
%Station{}
|
|
|> Station.changeset(attrs)
|
|
|> Repo.insert()
|
|
end
|
|
end
|
|
|
|
def upsert_surface_observation(%Station{} = station, attrs) do
|
|
attrs = Map.put(attrs, :station_id, station.id)
|
|
|
|
%SurfaceObservation{}
|
|
|> SurfaceObservation.changeset(attrs)
|
|
|> Repo.insert(
|
|
on_conflict: {:replace_all_except, [:id, :station_id, :observed_at, :inserted_at]},
|
|
conflict_target: [:station_id, :observed_at],
|
|
returning: true
|
|
)
|
|
end
|
|
|
|
def upsert_sounding(%Station{} = station, attrs) do
|
|
attrs = Map.put(attrs, :station_id, station.id)
|
|
|
|
%Sounding{}
|
|
|> Sounding.changeset(attrs)
|
|
|> Repo.insert(
|
|
on_conflict: {:replace_all_except, [:id, :station_id, :observed_at, :inserted_at]},
|
|
conflict_target: [:station_id, :observed_at],
|
|
returning: true
|
|
)
|
|
end
|
|
|
|
def upsert_solar_index(attrs) do
|
|
%SolarIndex{}
|
|
|> SolarIndex.changeset(attrs)
|
|
|> Repo.insert(
|
|
on_conflict: {:replace_all_except, [:id, :date, :inserted_at]},
|
|
conflict_target: [:date],
|
|
returning: true
|
|
)
|
|
end
|
|
|
|
def has_surface_observations?(station_id, start_dt, end_dt) do
|
|
SurfaceObservation
|
|
|> where([o], o.station_id == ^station_id)
|
|
|> where([o], o.observed_at >= ^start_dt and o.observed_at <= ^end_dt)
|
|
|> Repo.exists?()
|
|
end
|
|
|
|
def has_sounding?(station_id, observed_at) do
|
|
Sounding
|
|
|> where([s], s.station_id == ^station_id and s.observed_at == ^observed_at)
|
|
|> Repo.exists?()
|
|
end
|
|
|
|
def get_solar_index(date) do
|
|
Repo.get_by(SolarIndex, date: date)
|
|
end
|
|
|
|
def existing_solar_dates do
|
|
SolarIndex
|
|
|> select([s], s.date)
|
|
|> Repo.all()
|
|
|> MapSet.new()
|
|
end
|
|
|
|
def nearby_stations(lat, lon, station_type, radius_km) do
|
|
dlat = radius_km / @km_per_deg_lat
|
|
dlon = radius_km / (@km_per_deg_lat * :math.cos(lat * :math.pi() / 180))
|
|
|
|
Station
|
|
|> where([s], s.station_type == ^station_type)
|
|
|> where(
|
|
[s],
|
|
s.lat >= ^(lat - dlat) and s.lat <= ^(lat + dlat) and
|
|
s.lon >= ^(lon - dlon) and s.lon <= ^(lon + dlon)
|
|
)
|
|
|> Repo.all()
|
|
end
|
|
|
|
def sounding_times_around(dt) do
|
|
date = DateTime.to_date(dt)
|
|
|
|
times =
|
|
if dt.hour < 12 do
|
|
[
|
|
DateTime.new!(Date.add(date, -1), ~T[12:00:00], "Etc/UTC"),
|
|
DateTime.new!(date, ~T[00:00:00], "Etc/UTC")
|
|
]
|
|
else
|
|
[
|
|
DateTime.new!(date, ~T[00:00:00], "Etc/UTC"),
|
|
DateTime.new!(date, ~T[12:00:00], "Etc/UTC")
|
|
]
|
|
end
|
|
|
|
Enum.uniq(times)
|
|
end
|
|
|
|
def weather_for_qso(qso_params, opts \\ []) do
|
|
lat = qso_params[:lat] || qso_params.lat
|
|
lon = qso_params[:lon] || qso_params.lon
|
|
timestamp = qso_params[:timestamp] || qso_params.timestamp
|
|
|
|
radius_km = Keyword.get(opts, :radius_km, 150)
|
|
time_window_hours = Keyword.get(opts, :time_window_hours, 6)
|
|
|
|
# Bounding box in degrees
|
|
dlat = radius_km / @km_per_deg_lat
|
|
dlon = radius_km / (@km_per_deg_lat * :math.cos(lat * :math.pi() / 180))
|
|
|
|
time_start = DateTime.add(timestamp, -time_window_hours * 3600, :second)
|
|
time_end = DateTime.add(timestamp, time_window_hours * 3600, :second)
|
|
|
|
station_ids =
|
|
Station
|
|
|> where(
|
|
[s],
|
|
s.lat >= ^(lat - dlat) and s.lat <= ^(lat + dlat) and
|
|
s.lon >= ^(lon - dlon) and s.lon <= ^(lon + dlon)
|
|
)
|
|
|> select([s], s.id)
|
|
|
|
surface_observations =
|
|
SurfaceObservation
|
|
|> where([o], o.station_id in subquery(station_ids))
|
|
|> where([o], o.observed_at >= ^time_start and o.observed_at <= ^time_end)
|
|
|> preload(:station)
|
|
|> Repo.all()
|
|
|
|
soundings =
|
|
Sounding
|
|
|> where([s], s.station_id in subquery(station_ids))
|
|
|> where([s], s.observed_at >= ^time_start and s.observed_at <= ^time_end)
|
|
|> preload(:station)
|
|
|> Repo.all()
|
|
|
|
%{surface_observations: surface_observations, soundings: soundings}
|
|
end
|
|
|
|
def upsert_hrrr_profile(attrs) do
|
|
%HrrrProfile{}
|
|
|> HrrrProfile.changeset(attrs)
|
|
|> Repo.insert(
|
|
on_conflict: {:replace_all_except, [:id, :inserted_at]},
|
|
conflict_target: [:lat, :lon, :valid_time],
|
|
returning: true
|
|
)
|
|
end
|
|
|
|
def upsert_hrrr_profiles_batch(profiles) do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
entries =
|
|
Enum.map(profiles, fn attrs ->
|
|
Map.merge(attrs, %{
|
|
id: Ecto.UUID.generate(),
|
|
inserted_at: now,
|
|
updated_at: now
|
|
})
|
|
end)
|
|
|
|
Repo.insert_all(HrrrProfile, entries,
|
|
on_conflict: {:replace_all_except, [:id, :inserted_at]},
|
|
conflict_target: [:lat, :lon, :valid_time]
|
|
)
|
|
end
|
|
|
|
def has_hrrr_profile?(lat, lon, valid_time) do
|
|
{rlat, rlon} = round_to_hrrr_grid(lat, lon)
|
|
|
|
HrrrProfile
|
|
|> where([h], h.lat == ^rlat and h.lon == ^rlon and h.valid_time == ^valid_time)
|
|
|> Repo.exists?()
|
|
end
|
|
|
|
def hrrr_for_qso(%{pos1: nil}), do: nil
|
|
|
|
def hrrr_for_qso(qso) do
|
|
lat = qso.pos1["lat"]
|
|
lon = qso.pos1["lon"] || qso.pos1["lng"]
|
|
|
|
if lat && lon do
|
|
find_nearest_hrrr(lat, lon, qso.qso_timestamp)
|
|
end
|
|
end
|
|
|
|
def find_nearest_hrrr(lat, lon, timestamp) do
|
|
dlat = 0.05
|
|
dlon = 0.05
|
|
time_start = DateTime.add(timestamp, -3600, :second)
|
|
time_end = DateTime.add(timestamp, 3600, :second)
|
|
|
|
HrrrProfile
|
|
|> where(
|
|
[h],
|
|
h.lat >= ^(lat - dlat) and h.lat <= ^(lat + dlat) and
|
|
h.lon >= ^(lon - dlon) and h.lon <= ^(lon + dlon) and
|
|
h.valid_time >= ^time_start and h.valid_time <= ^time_end
|
|
)
|
|
|> order_by([h],
|
|
asc:
|
|
fragment(
|
|
"ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))",
|
|
h.lat,
|
|
^lat,
|
|
h.lon,
|
|
^lon,
|
|
h.valid_time,
|
|
^timestamp
|
|
)
|
|
)
|
|
|> limit(1)
|
|
|> Repo.one()
|
|
end
|
|
|
|
def hrrr_profiles_for_path(%{pos1: nil}), do: []
|
|
|
|
def hrrr_profiles_for_path(qso) do
|
|
qso
|
|
|> Microwaveprop.Radio.qso_path_points()
|
|
|> Enum.map(fn {lat, lon} -> find_nearest_hrrr(lat, lon, qso.qso_timestamp) end)
|
|
|> Enum.reject(&is_nil/1)
|
|
end
|
|
|
|
def round_to_hrrr_grid(lat, lon) do
|
|
{Float.round(lat / 1.0, 2), Float.round(lon / 1.0, 2)}
|
|
end
|
|
|
|
def round_to_iemre_grid(lat, lon) do
|
|
{Float.round(lat * 8) / 8, Float.round(lon * 8) / 8}
|
|
end
|
|
|
|
def upsert_iemre_observation(attrs) do
|
|
%IemreObservation{}
|
|
|> IemreObservation.changeset(attrs)
|
|
|> Repo.insert(
|
|
on_conflict: {:replace_all_except, [:id, :inserted_at]},
|
|
conflict_target: [:lat, :lon, :date],
|
|
returning: true
|
|
)
|
|
end
|
|
|
|
def has_iemre_observation?(lat, lon, date) do
|
|
IemreObservation
|
|
|> where([i], i.lat == ^lat and i.lon == ^lon and i.date == ^date)
|
|
|> Repo.exists?()
|
|
end
|
|
|
|
def iemre_for_qso(%{pos1: nil}), do: nil
|
|
|
|
def iemre_for_qso(qso) do
|
|
lat = qso.pos1["lat"]
|
|
lon = qso.pos1["lon"] || qso.pos1["lng"]
|
|
|
|
if lat && lon do
|
|
find_nearest_iemre(lat, lon, qso.qso_timestamp)
|
|
end
|
|
end
|
|
|
|
def find_nearest_iemre(lat, lon, timestamp) do
|
|
{rlat, rlon} = round_to_iemre_grid(lat, lon)
|
|
date = DateTime.to_date(timestamp)
|
|
|
|
IemreObservation
|
|
|> where([i], i.lat == ^rlat and i.lon == ^rlon and i.date == ^date)
|
|
|> Repo.one()
|
|
end
|
|
|
|
def iemre_for_path(%{pos1: nil}), do: []
|
|
|
|
def iemre_for_path(qso) do
|
|
qso
|
|
|> Microwaveprop.Radio.qso_path_points()
|
|
|> Enum.map(fn {lat, lon} -> find_nearest_iemre(lat, lon, qso.qso_timestamp) end)
|
|
|> Enum.reject(&is_nil/1)
|
|
end
|
|
end
|