Ionosphere: GIRO ionosonde ingestion (foF2/foEs/hmF2/MUFD)
First step toward physics-based HF / sporadic-E scoring. Polls GIRO's DIDBase tabular endpoint every 10 minutes for two US ionosondes (Millstone Hill MHJ45, Alpena AL945), parses the plain-text response into observation rows, and upserts into a new ionosonde_observations table keyed on (station_code, valid_time). This gets foEs (E-layer sporadic critical frequency) into the database as a direct measurement — the key input for predicting 144 MHz Es openings via ITU-R P.534-6. foF2 + MUFD are the F2-layer inputs for HF MUF predictions. Not yet wired into scoring. Boulder/Wallops/Austin/Idaho/Point Arguello are in the GIRO catalog but were silent when probed — add them back if/when they come online. Next steps: SWPC JSON (Kp, F10.7, sunspot), GOES X-ray flux, D-RAP text, and the P.534-6 Es scoring factor that uses foEs at midpoint for the 144/440 band configs.
This commit is contained in:
parent
139f9f72ea
commit
361b9dec5a
12 changed files with 635 additions and 4 deletions
|
|
@ -67,7 +67,8 @@ config :microwaveprop, Oban,
|
|||
# forecast-hour chain job.
|
||||
propagation: 2,
|
||||
admin: 1,
|
||||
nexrad: 2
|
||||
nexrad: 2,
|
||||
ionosphere: 1
|
||||
],
|
||||
plugins: [
|
||||
{Oban.Plugins.Pruner, max_age: 3600 * 24},
|
||||
|
|
@ -85,7 +86,10 @@ config :microwaveprop, Oban,
|
|||
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
||||
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
|
||||
{"30 1 * * *", CanadianSoundingFetchWorker},
|
||||
{"30 13 * * *", CanadianSoundingFetchWorker}
|
||||
{"30 13 * * *", CanadianSoundingFetchWorker},
|
||||
# GIRO ionosonde stations publish at ~7.5 min cadence; poll every
|
||||
# 10 min to pick up fresh foF2/foEs/hmF2 within one cycle.
|
||||
{"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker}
|
||||
]}
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -195,7 +195,8 @@ if config_env() == :prod do
|
|||
rtma: 2,
|
||||
backfill_enqueue: 1,
|
||||
admin: 1,
|
||||
nexrad: 2
|
||||
nexrad: 2,
|
||||
ionosphere: 1
|
||||
],
|
||||
plugins: [
|
||||
{Oban.Plugins.Pruner, max_age: 3600 * 24},
|
||||
|
|
@ -236,7 +237,11 @@ if config_env() == :prod do
|
|||
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}},
|
||||
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
|
||||
{"30 1 * * *", CanadianSoundingFetchWorker},
|
||||
{"30 13 * * *", CanadianSoundingFetchWorker}
|
||||
{"30 13 * * *", CanadianSoundingFetchWorker},
|
||||
# GIRO DIDBase publishes foF2/foEs/hmF2 at ~7.5 min cadence; poll
|
||||
# every 10 min per station (Millstone Hill, Alpena) for the 144
|
||||
# MHz sporadic-E and HF MUF scoring inputs.
|
||||
{"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker}
|
||||
]}
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ config :microwaveprop, cache_contact_count: false
|
|||
config :microwaveprop, cache_contact_map: false
|
||||
config :microwaveprop, elevation_req_options: [plug: {Req.Test, Microwaveprop.Terrain.ElevationClient}, retry: false]
|
||||
config :microwaveprop, era5_req_options: [plug: {Req.Test, Microwaveprop.Weather.Era5Client}, retry: false]
|
||||
config :microwaveprop, giro_req_options: [plug: {Req.Test, Microwaveprop.Ionosphere.GiroClient}, retry: false]
|
||||
config :microwaveprop, hrrr_req_options: [plug: {Req.Test, Microwaveprop.Weather.HrrrClient}, retry: false]
|
||||
config :microwaveprop, iem_req_options: [plug: {Req.Test, Microwaveprop.Weather.IemClient}, retry: false]
|
||||
|
||||
|
|
|
|||
52
lib/microwaveprop/ionosphere.ex
Normal file
52
lib/microwaveprop/ionosphere.ex
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
defmodule Microwaveprop.Ionosphere do
|
||||
@moduledoc """
|
||||
Context for ionospheric observations (GIRO ionosonde data). Used by
|
||||
the propagation scorer for VHF sporadic-E and HF MUF predictions.
|
||||
"""
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias Microwaveprop.Ionosphere.Observation
|
||||
alias Microwaveprop.Repo
|
||||
|
||||
@doc """
|
||||
Upsert a list of parsed `GiroClient` observations for a station.
|
||||
Conflicts on `(station_code, valid_time)` replace every column except
|
||||
the primary key and inserted_at. Returns `{:ok, count}`.
|
||||
"""
|
||||
@spec upsert_observations(String.t(), [map()]) :: {:ok, non_neg_integer()}
|
||||
def upsert_observations(_station_code, []), do: {:ok, 0}
|
||||
|
||||
def upsert_observations(station_code, observations) when is_list(observations) do
|
||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||
|
||||
rows =
|
||||
Enum.map(observations, fn obs ->
|
||||
obs
|
||||
|> Map.put(:station_code, station_code)
|
||||
|> Map.put(:inserted_at, now)
|
||||
|> Map.put(:updated_at, now)
|
||||
end)
|
||||
|
||||
{count, _} =
|
||||
Repo.insert_all(Observation, rows,
|
||||
on_conflict: {:replace_all_except, [:id, :inserted_at]},
|
||||
conflict_target: [:station_code, :valid_time]
|
||||
)
|
||||
|
||||
{:ok, count}
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns the most recent `Observation` row for a station, or nil.
|
||||
"""
|
||||
@spec latest_observation(String.t()) :: Observation.t() | nil
|
||||
def latest_observation(station_code) do
|
||||
Repo.one(
|
||||
from o in Observation,
|
||||
where: o.station_code == ^station_code,
|
||||
order_by: [desc: o.valid_time],
|
||||
limit: 1
|
||||
)
|
||||
end
|
||||
end
|
||||
195
lib/microwaveprop/ionosphere/giro_client.ex
Normal file
195
lib/microwaveprop/ionosphere/giro_client.ex
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
defmodule Microwaveprop.Ionosphere.GiroClient do
|
||||
@moduledoc """
|
||||
Client for GIRO's `DIDBGetValues` tabular characteristic endpoint.
|
||||
|
||||
GIRO (Global Ionospheric Radio Observatory, operated by LGDC at UMass
|
||||
Lowell) publishes real-time ionosonde measurements for stations
|
||||
worldwide. We poll the scaled characteristics for a handful of US
|
||||
stations every ~10 minutes so the propagation scorer has a live view
|
||||
of the ionosphere for 144 MHz sporadic-E and HF MUF predictions.
|
||||
|
||||
The endpoint returns a plain-text tabular response, not JSON:
|
||||
|
||||
# Global Ionospheric Radio Observatory
|
||||
# Location: GEO 42.6N 288.5E, URSI-Code MHJ45 MILLSTONE HILL
|
||||
# ...header comment block...
|
||||
#Time CS foF2 QD MUFD QD foEs QD foE QD hmF2 QD
|
||||
2026-04-15T18:00:00.000Z 80 7.113 // 23.00 // 3.08 // 3.10 // 242.0 //
|
||||
|
||||
Columns:
|
||||
- `Time` — ISO-8601 UTC
|
||||
- `CS` — autoscaling confidence score (0–100, 999 = manual, -1 = unknown)
|
||||
- `foF2`, `foE`, `foEs` — critical frequencies in MHz
|
||||
- `hmF2` — F2 peak height in km
|
||||
- `MUFD` — maximum usable frequency for the distance D requested
|
||||
|
||||
Each characteristic is followed by a QD (quality descriptor) flag — we
|
||||
drop the QD column when indexing. Missing values are encoded as `---`
|
||||
with a `__` QD flag and become `nil` in the parsed map.
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
@base_url "https://lgdc.uml.edu/common/DIDBGetValues"
|
||||
|
||||
@default_chars ~w(foF2 foE foEs hmF2 MUFD)
|
||||
@default_dmuf 3000
|
||||
@default_timeout_ms 15_000
|
||||
|
||||
@type observation :: %{
|
||||
valid_time: DateTime.t(),
|
||||
confidence_score: integer() | nil,
|
||||
fo_f2_mhz: float() | nil,
|
||||
fo_e_mhz: float() | nil,
|
||||
fo_es_mhz: float() | nil,
|
||||
hm_f2_km: float() | nil,
|
||||
mufd_mhz: float() | nil
|
||||
}
|
||||
|
||||
@doc """
|
||||
Fetches scaled characteristics for a station between two UTC DateTimes.
|
||||
Returns `{:ok, [observation]}` or `{:error, reason}`.
|
||||
"""
|
||||
@spec fetch(String.t(), DateTime.t(), DateTime.t()) ::
|
||||
{:ok, [observation()]} | {:error, term()}
|
||||
def fetch(ursi_code, %DateTime{} = from_dt, %DateTime{} = to_dt) do
|
||||
url = build_url(ursi_code, from_dt, to_dt)
|
||||
|
||||
case Req.get(url, [receive_timeout: @default_timeout_ms] ++ req_options()) do
|
||||
{:ok, %{status: 200, body: body}} when is_binary(body) ->
|
||||
parse_tabular(body)
|
||||
|
||||
{:ok, %{status: status, body: body}} ->
|
||||
{:error, "GIRO HTTP #{status}: #{inspect(body)}"}
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, "GIRO request failed: #{inspect(reason)}"}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Parses a GIRO tabular response body into a list of observation maps.
|
||||
"""
|
||||
@spec parse_tabular(String.t()) :: {:ok, [observation()]} | {:error, atom()}
|
||||
def parse_tabular(body) when is_binary(body) do
|
||||
lines = String.split(body, ~r/\r?\n/, trim: true)
|
||||
|
||||
case find_header(lines) do
|
||||
{:ok, columns} ->
|
||||
rows =
|
||||
lines
|
||||
|> Enum.reject(&header_or_comment?/1)
|
||||
|> Enum.flat_map(&parse_data_line(&1, columns))
|
||||
|
||||
{:ok, rows}
|
||||
|
||||
:error ->
|
||||
{:error, :no_header}
|
||||
end
|
||||
end
|
||||
|
||||
defp build_url(ursi_code, from_dt, to_dt) do
|
||||
params = [
|
||||
{"ursiCode", ursi_code},
|
||||
{"charName", Enum.join(@default_chars, ",")},
|
||||
{"DMUF", Integer.to_string(@default_dmuf)},
|
||||
{"fromDate", format_giro_date(from_dt)},
|
||||
{"toDate", format_giro_date(to_dt)}
|
||||
]
|
||||
|
||||
query = Enum.map_join(params, "&", fn {k, v} -> "#{k}=#{URI.encode(v)}" end)
|
||||
"#{@base_url}?#{query}"
|
||||
end
|
||||
|
||||
# GIRO wants `YYYY.MM.DD+HH:MM:SS` with a literal `+`. Using `/` or
|
||||
# url-escaping the space silently returns "no data".
|
||||
defp format_giro_date(%DateTime{} = dt) do
|
||||
dt
|
||||
|> DateTime.truncate(:second)
|
||||
|> Calendar.strftime("%Y.%m.%d+%H:%M:%S")
|
||||
end
|
||||
|
||||
# Column header line starts with `#Time`. Returns the list of value
|
||||
# column names, stripping the `Time`/`CS` prefix columns and the `QD`
|
||||
# filler tokens (which are paired with each value).
|
||||
defp find_header(lines) do
|
||||
case Enum.find(lines, &String.starts_with?(&1, "#Time")) do
|
||||
nil ->
|
||||
:error
|
||||
|
||||
line ->
|
||||
columns =
|
||||
line
|
||||
# Drop the leading "#"
|
||||
|> String.trim_leading("#")
|
||||
|> String.split(~r/\s+/, trim: true)
|
||||
|> Enum.reject(&(&1 in ["Time", "CS", "QD"]))
|
||||
|
||||
{:ok, columns}
|
||||
end
|
||||
end
|
||||
|
||||
defp header_or_comment?(line), do: String.starts_with?(line, "#")
|
||||
|
||||
defp parse_data_line(line, columns) do
|
||||
tokens = String.split(line, ~r/\s+/, trim: true)
|
||||
# Expected: time + CS + 2 tokens per column (value + QD)
|
||||
expected = 2 + length(columns) * 2
|
||||
|
||||
if length(tokens) == expected do
|
||||
[time_str, cs_str | rest] = tokens
|
||||
|
||||
case DateTime.from_iso8601(time_str) do
|
||||
{:ok, valid_time, _} ->
|
||||
values = take_values(rest)
|
||||
row = columns |> Enum.zip(values) |> Map.new() |> then(&build_observation(valid_time, cs_str, &1))
|
||||
[row]
|
||||
|
||||
_ ->
|
||||
[]
|
||||
end
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
# Drop the QD flags by keeping every other element.
|
||||
defp take_values([]), do: []
|
||||
defp take_values([value, _qd | rest]), do: [value | take_values(rest)]
|
||||
defp take_values([value]), do: [value]
|
||||
|
||||
defp build_observation(valid_time, cs_str, values_by_col) do
|
||||
%{
|
||||
valid_time: DateTime.truncate(valid_time, :second),
|
||||
confidence_score: parse_int(cs_str),
|
||||
fo_f2_mhz: parse_float(values_by_col["foF2"]),
|
||||
fo_e_mhz: parse_float(values_by_col["foE"]),
|
||||
fo_es_mhz: parse_float(values_by_col["foEs"]),
|
||||
hm_f2_km: parse_float(values_by_col["hmF2"]),
|
||||
mufd_mhz: parse_float(values_by_col["MUFD"])
|
||||
}
|
||||
end
|
||||
|
||||
defp parse_int(nil), do: nil
|
||||
|
||||
defp parse_int(s) do
|
||||
case Integer.parse(s) do
|
||||
{n, ""} -> n
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_float(nil), do: nil
|
||||
defp parse_float("---"), do: nil
|
||||
|
||||
defp parse_float(s) do
|
||||
case Float.parse(s) do
|
||||
{f, ""} -> f
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp req_options do
|
||||
Application.get_env(:microwaveprop, :giro_req_options, [])
|
||||
end
|
||||
end
|
||||
39
lib/microwaveprop/ionosphere/observation.ex
Normal file
39
lib/microwaveprop/ionosphere/observation.ex
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
defmodule Microwaveprop.Ionosphere.Observation do
|
||||
@moduledoc """
|
||||
A single ionosonde measurement for one station at one time. Populated
|
||||
from GIRO's DIDBase scaled-characteristic feed and used by the
|
||||
propagation scorer for sporadic-E and HF MUF predictions.
|
||||
"""
|
||||
use Ecto.Schema
|
||||
|
||||
import Ecto.Changeset
|
||||
|
||||
@primary_key {:id, :binary_id, autogenerate: true}
|
||||
@foreign_key_type :binary_id
|
||||
|
||||
schema "ionosonde_observations" do
|
||||
field :station_code, :string
|
||||
field :valid_time, :utc_datetime
|
||||
field :confidence_score, :integer
|
||||
field :fo_f2_mhz, :float
|
||||
field :fo_e_mhz, :float
|
||||
field :fo_es_mhz, :float
|
||||
field :hm_f2_km, :float
|
||||
field :mufd_mhz, :float
|
||||
|
||||
timestamps(type: :utc_datetime)
|
||||
end
|
||||
|
||||
@type t :: %__MODULE__{}
|
||||
|
||||
@required_fields ~w(station_code valid_time)a
|
||||
@optional_fields ~w(confidence_score fo_f2_mhz fo_e_mhz fo_es_mhz hm_f2_km mufd_mhz)a
|
||||
|
||||
@spec changeset(t() | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t()
|
||||
def changeset(observation, attrs) do
|
||||
observation
|
||||
|> cast(attrs, @required_fields ++ @optional_fields)
|
||||
|> validate_required(@required_fields)
|
||||
|> unique_constraint([:station_code, :valid_time])
|
||||
end
|
||||
end
|
||||
62
lib/microwaveprop/workers/ionosphere_fetch_worker.ex
Normal file
62
lib/microwaveprop/workers/ionosphere_fetch_worker.ex
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
defmodule Microwaveprop.Workers.IonosphereFetchWorker do
|
||||
@moduledoc """
|
||||
Polls GIRO DIDBase for real-time ionosonde measurements (foF2, foE,
|
||||
foEs, hmF2, MUFD) from a handful of CONUS stations. Drives the 144
|
||||
MHz sporadic-E and HF MUF scoring inputs.
|
||||
|
||||
Runs on a ~10-minute cron — GIRO publishes each station at 7.5-minute
|
||||
cadence so a 10-minute poll guarantees we pick up every new sample
|
||||
within one cycle without hammering the endpoint.
|
||||
|
||||
Pulls a 2-hour window each poll to catch any late-arriving samples
|
||||
and to be resilient to the occasional missed run. Upserts are
|
||||
idempotent on (station_code, valid_time).
|
||||
"""
|
||||
|
||||
use Oban.Worker,
|
||||
queue: :ionosphere,
|
||||
max_attempts: 3,
|
||||
unique: [period: 300, states: [:available, :scheduled, :executing, :retryable]]
|
||||
|
||||
alias Microwaveprop.Ionosphere
|
||||
alias Microwaveprop.Ionosphere.GiroClient
|
||||
|
||||
require Logger
|
||||
|
||||
# CONUS stations that have been publishing reliably. Others (Boulder,
|
||||
# Wallops, Austin, Idaho, Point Arguello) are in the GIRO catalog but
|
||||
# intermittent — add them here once they come back online.
|
||||
@stations [
|
||||
%{code: "MHJ45", name: "Millstone Hill MA", lat: 42.6, lon: -71.5},
|
||||
%{code: "AL945", name: "Alpena MI", lat: 45.1, lon: -83.6}
|
||||
]
|
||||
|
||||
@lookback_seconds 2 * 3600
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{}) do
|
||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||
from_dt = DateTime.add(now, -@lookback_seconds, :second)
|
||||
|
||||
Enum.each(@stations, fn station ->
|
||||
fetch_and_upsert(station, from_dt, now)
|
||||
end)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
@doc false
|
||||
@spec stations() :: [map()]
|
||||
def stations, do: @stations
|
||||
|
||||
defp fetch_and_upsert(%{code: code, name: name}, from_dt, to_dt) do
|
||||
case GiroClient.fetch(code, from_dt, to_dt) do
|
||||
{:ok, observations} ->
|
||||
{:ok, count} = Ionosphere.upsert_observations(code, observations)
|
||||
Logger.info("IonosphereFetch: #{code} (#{name}) upserted #{count} observations")
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.warning("IonosphereFetch: #{code} (#{name}) failed: #{inspect(reason)}")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
defmodule Microwaveprop.Repo.Migrations.CreateIonosondeObservations do
|
||||
use Ecto.Migration
|
||||
|
||||
def change do
|
||||
create table(:ionosonde_observations, primary_key: false) do
|
||||
add :id, :binary_id, primary_key: true
|
||||
add :station_code, :string, null: false
|
||||
add :valid_time, :utc_datetime, null: false
|
||||
add :confidence_score, :integer
|
||||
add :fo_f2_mhz, :float
|
||||
add :fo_e_mhz, :float
|
||||
add :fo_es_mhz, :float
|
||||
add :hm_f2_km, :float
|
||||
add :mufd_mhz, :float
|
||||
|
||||
timestamps(type: :utc_datetime)
|
||||
end
|
||||
|
||||
create unique_index(:ionosonde_observations, [:station_code, :valid_time])
|
||||
create index(:ionosonde_observations, [:valid_time])
|
||||
end
|
||||
end
|
||||
50
test/fixtures/giro/mhj45_sample.txt
vendored
Normal file
50
test/fixtures/giro/mhj45_sample.txt
vendored
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
# Global Ionospheric Radio Observatory
|
||||
# GIRO Tabulated Ionospheric Characteristics, Version 1.0 Revision B
|
||||
# Generated by DIDBGetValues on 2026-04-15T19:32:24.784Z
|
||||
#
|
||||
# Location: GEO 42.6N 288.5E, URSI-Code MHJ45 MILLSTONE HILL
|
||||
# Instrument: Ionosonde, Model: DPS-4D
|
||||
#
|
||||
# Query for measurement intervals of time:
|
||||
# 2026-04-15T16:00:00.000Z - 2026-04-15T19:00:00.000Z
|
||||
#
|
||||
# Data Selection:
|
||||
# CS is Autoscaling Confidence Score (from 0 to 100, 999 if manual scaling, -1 if unknown)
|
||||
# foF2 [MHz] - F2 layer critical frequency
|
||||
# MUFD [MHz] - Maximum usable frequency for ground distance D
|
||||
# foEs [MHz] - Es layer critical frequency
|
||||
# foE [MHz] - E layer critical frequency
|
||||
# hmF2 [km] - Peak height F2-layer
|
||||
# Distance D for MUF calculations: 3000 km
|
||||
#
|
||||
# All GIRO measurements are released under CC-BY-NC-SA 4.0 license
|
||||
# Please follow the Lowell GIRO Data Center RULES OF THE ROAD
|
||||
# https://ulcar.uml.edu/DIDBase/RulesOfTheRoadForDIDBase.htm
|
||||
# Requires acknowledgement of MHJ45 data provider
|
||||
#
|
||||
#Time CS foF2 QD MUFD QD foEs QD foE QD hmF2 QD
|
||||
2026-04-15T16:00:00.000Z 60 6.856 // 23.63 // --- __ 3.50 // 217.2 //
|
||||
2026-04-15T16:07:30.000Z 90 6.763 // 21.10 // 3.42 // 3.47 // 260.3 //
|
||||
2026-04-15T16:15:00.000Z 70 6.863 // 22.72 // 3.50 // 3.50 // 225.1 //
|
||||
2026-04-15T16:22:30.000Z 70 6.863 // 22.39 // 3.45 // 3.45 // 231.8 //
|
||||
2026-04-15T16:30:00.000Z 75 6.775 // 22.72 // --- __ 3.30 // 224.3 //
|
||||
2026-04-15T16:37:30.000Z 70 6.913 // 23.39 // 3.08 // 3.15 // 226.6 //
|
||||
2026-04-15T16:45:00.000Z 60 6.925 // 22.41 // 3.05 // 3.13 // 226.6 //
|
||||
2026-04-15T16:52:30.000Z 70 6.975 // 23.32 // --- __ 3.30 // 212.0 //
|
||||
2026-04-15T17:00:00.000Z 90 6.907 // 21.87 // --- __ 3.20 // 232.4 //
|
||||
2026-04-15T17:07:30.000Z 60 7.063 // 23.52 // 3.13 // 3.15 // 236.4 //
|
||||
2026-04-15T17:15:00.000Z 75 7.225 // 23.38 // 3.20 // 3.38 // 244.2 //
|
||||
2026-04-15T17:22:30.000Z 70 7.138 // 23.45 // 3.40 // 3.35 // 234.8 //
|
||||
2026-04-15T17:30:00.000Z 75 7.063 // 22.66 // --- __ 3.35 // 238.8 //
|
||||
2026-04-15T17:37:30.000Z 70 7.025 // 22.19 // 3.08 // 3.28 // 240.9 //
|
||||
2026-04-15T17:45:00.000Z 70 7.138 // 22.25 // --- __ 3.08 // 256.2 //
|
||||
2026-04-15T17:52:30.000Z 70 7.138 // 22.97 // 3.13 // 2.97 // 242.6 //
|
||||
2026-04-15T18:00:00.000Z 80 7.113 // 23.00 // 3.08 // 3.10 // 242.0 //
|
||||
2026-04-15T18:07:30.000Z 80 7.088 // 22.76 // 3.35 // 3.13 // 246.0 //
|
||||
2026-04-15T18:15:00.000Z 70 7.150 // 22.09 // 3.42 // 2.92 // 266.7 //
|
||||
2026-04-15T18:22:30.000Z 75 7.200 // 22.37 // 3.45 // 2.80 // 262.0 //
|
||||
2026-04-15T18:30:00.000Z 70 7.225 // 23.34 // 3.22 // 2.85 // 246.5 //
|
||||
2026-04-15T18:37:30.000Z 75 7.213 // 23.03 // --- __ 3.17 // 250.9 //
|
||||
2026-04-15T18:45:00.000Z 75 7.338 // 22.77 // 3.20 // 2.72 // 263.5 //
|
||||
2026-04-15T18:52:30.000Z 75 7.413 // 23.50 // 3.33 // 3.10 // 256.5 //
|
||||
2026-04-15T19:00:00.000Z 75 7.400 // 23.64 // 3.50 // 3.15 // 249.4 //
|
||||
69
test/microwaveprop/ionosphere/giro_client_test.exs
Normal file
69
test/microwaveprop/ionosphere/giro_client_test.exs
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
defmodule Microwaveprop.Ionosphere.GiroClientTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Microwaveprop.Ionosphere.GiroClient
|
||||
|
||||
describe "parse_tabular/1" do
|
||||
test "decodes the real Millstone Hill fixture into observation maps" do
|
||||
body = File.read!("test/fixtures/giro/mhj45_sample.txt")
|
||||
|
||||
assert {:ok, rows} = GiroClient.parse_tabular(body)
|
||||
|
||||
# The fixture is 2026-04-15 16:00 → 19:00 UTC at 7.5-minute cadence,
|
||||
# so 25 observations.
|
||||
assert length(rows) == 25
|
||||
|
||||
first = hd(rows)
|
||||
assert first.valid_time == ~U[2026-04-15 16:00:00Z]
|
||||
assert first.confidence_score == 60
|
||||
assert_in_delta first.fo_f2_mhz, 6.856, 0.001
|
||||
assert_in_delta first.mufd_mhz, 23.63, 0.001
|
||||
# foEs was missing ("---") at 16:00.
|
||||
assert first.fo_es_mhz == nil
|
||||
assert_in_delta first.fo_e_mhz, 3.50, 0.001
|
||||
assert_in_delta first.hm_f2_km, 217.2, 0.1
|
||||
|
||||
# 16:07:30 has foEs == 3.42.
|
||||
second = Enum.at(rows, 1)
|
||||
assert second.valid_time == ~U[2026-04-15 16:07:30Z]
|
||||
assert_in_delta second.fo_es_mhz, 3.42, 0.001
|
||||
|
||||
last = List.last(rows)
|
||||
assert last.valid_time == ~U[2026-04-15 19:00:00Z]
|
||||
assert_in_delta last.fo_f2_mhz, 7.400, 0.001
|
||||
end
|
||||
|
||||
test "returns {:ok, []} for a body with only the header" do
|
||||
body = """
|
||||
# Global Ionospheric Radio Observatory
|
||||
# Location: GEO 42.6N 288.5E, URSI-Code MHJ45 MILLSTONE HILL
|
||||
#
|
||||
#Time CS foF2 QD MUFD QD foEs QD foE QD hmF2 QD
|
||||
"""
|
||||
|
||||
assert {:ok, []} = GiroClient.parse_tabular(body)
|
||||
end
|
||||
|
||||
test "returns {:error, :no_header} when the column header is missing" do
|
||||
body = """
|
||||
# some junk
|
||||
2026-04-15T18:00:00.000Z 80 7.113 // 23.00 //
|
||||
"""
|
||||
|
||||
assert {:error, :no_header} = GiroClient.parse_tabular(body)
|
||||
end
|
||||
|
||||
test "skips data lines whose token count doesn't match the header" do
|
||||
# Header declares 5 value columns (10 tokens after time+CS) but the
|
||||
# data row is truncated. Should be skipped rather than crashing.
|
||||
body = """
|
||||
#Time CS foF2 QD MUFD QD foEs QD foE QD hmF2 QD
|
||||
2026-04-15T18:00:00.000Z 80 7.113 //
|
||||
2026-04-15T18:07:30.000Z 80 7.088 // 22.76 // 3.35 // 3.13 // 246.0 //
|
||||
"""
|
||||
|
||||
assert {:ok, [row]} = GiroClient.parse_tabular(body)
|
||||
assert row.valid_time == ~U[2026-04-15 18:07:30Z]
|
||||
end
|
||||
end
|
||||
end
|
||||
83
test/microwaveprop/ionosphere/ionosphere_test.exs
Normal file
83
test/microwaveprop/ionosphere/ionosphere_test.exs
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
defmodule Microwaveprop.IonosphereTest do
|
||||
use Microwaveprop.DataCase, async: true
|
||||
|
||||
alias Microwaveprop.Ionosphere
|
||||
alias Microwaveprop.Ionosphere.Observation
|
||||
|
||||
@valid_attrs %{
|
||||
station_code: "MHJ45",
|
||||
valid_time: ~U[2026-04-15 18:00:00Z],
|
||||
confidence_score: 80,
|
||||
fo_f2_mhz: 7.113,
|
||||
fo_e_mhz: 3.10,
|
||||
fo_es_mhz: 3.08,
|
||||
hm_f2_km: 242.0,
|
||||
mufd_mhz: 23.00
|
||||
}
|
||||
|
||||
describe "upsert_observations/2" do
|
||||
test "inserts new observations and returns the inserted count" do
|
||||
assert {:ok, 2} =
|
||||
Ionosphere.upsert_observations("MHJ45", [
|
||||
Map.delete(@valid_attrs, :station_code),
|
||||
@valid_attrs
|
||||
|> Map.delete(:station_code)
|
||||
|> Map.put(:valid_time, ~U[2026-04-15 18:07:30Z])
|
||||
])
|
||||
|
||||
assert Repo.aggregate(Observation, :count, :id) == 2
|
||||
end
|
||||
|
||||
test "re-upserting the same (station, valid_time) replaces fields and does not duplicate" do
|
||||
{:ok, 1} = Ionosphere.upsert_observations("MHJ45", [Map.delete(@valid_attrs, :station_code)])
|
||||
|
||||
updated =
|
||||
@valid_attrs
|
||||
|> Map.delete(:station_code)
|
||||
|> Map.put(:fo_es_mhz, 9.5)
|
||||
|
||||
assert {:ok, 1} = Ionosphere.upsert_observations("MHJ45", [updated])
|
||||
assert Repo.aggregate(Observation, :count, :id) == 1
|
||||
|
||||
[row] = Repo.all(Observation)
|
||||
assert row.fo_es_mhz == 9.5
|
||||
end
|
||||
|
||||
test "stamps station_code on each observation" do
|
||||
{:ok, 1} =
|
||||
Ionosphere.upsert_observations("AL945", [
|
||||
Map.delete(@valid_attrs, :station_code)
|
||||
])
|
||||
|
||||
[row] = Repo.all(Observation)
|
||||
assert row.station_code == "AL945"
|
||||
end
|
||||
|
||||
test "returns 0 and does nothing for an empty list" do
|
||||
assert {:ok, 0} = Ionosphere.upsert_observations("MHJ45", [])
|
||||
assert Repo.aggregate(Observation, :count, :id) == 0
|
||||
end
|
||||
end
|
||||
|
||||
describe "latest_observation/1" do
|
||||
test "returns the most recent observation for a station" do
|
||||
older = Map.delete(@valid_attrs, :station_code)
|
||||
|
||||
newer =
|
||||
@valid_attrs
|
||||
|> Map.delete(:station_code)
|
||||
|> Map.put(:valid_time, ~U[2026-04-15 19:00:00Z])
|
||||
|> Map.put(:fo_es_mhz, 10.0)
|
||||
|
||||
{:ok, _} = Ionosphere.upsert_observations("MHJ45", [older, newer])
|
||||
|
||||
row = Ionosphere.latest_observation("MHJ45")
|
||||
assert row.valid_time == ~U[2026-04-15 19:00:00Z]
|
||||
assert row.fo_es_mhz == 10.0
|
||||
end
|
||||
|
||||
test "returns nil when no observations exist for the station" do
|
||||
assert Ionosphere.latest_observation("MHJ45") == nil
|
||||
end
|
||||
end
|
||||
end
|
||||
49
test/microwaveprop/workers/ionosphere_fetch_worker_test.exs
Normal file
49
test/microwaveprop/workers/ionosphere_fetch_worker_test.exs
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
defmodule Microwaveprop.Workers.IonosphereFetchWorkerTest do
|
||||
use Microwaveprop.DataCase, async: false
|
||||
use Oban.Testing, repo: Microwaveprop.Repo
|
||||
|
||||
alias Microwaveprop.Ionosphere
|
||||
alias Microwaveprop.Ionosphere.GiroClient
|
||||
alias Microwaveprop.Ionosphere.Observation
|
||||
alias Microwaveprop.Workers.IonosphereFetchWorker
|
||||
|
||||
@sample File.read!("test/fixtures/giro/mhj45_sample.txt")
|
||||
|
||||
describe "perform/1" do
|
||||
test "pulls each configured station and upserts the parsed rows" do
|
||||
Req.Test.stub(GiroClient, fn conn ->
|
||||
assert conn.method == "GET"
|
||||
params = Plug.Conn.fetch_query_params(conn).query_params
|
||||
assert params["ursiCode"] in ~w(MHJ45 AL945)
|
||||
|
||||
Plug.Conn.resp(conn, 200, @sample)
|
||||
end)
|
||||
|
||||
assert :ok = perform_job(IonosphereFetchWorker, %{})
|
||||
|
||||
# 25 observations per station × 2 stations = 50 rows (fixture has 25
|
||||
# data lines; upserts go to each station code separately).
|
||||
assert Repo.aggregate(Observation, :count, :id) == 50
|
||||
assert Ionosphere.latest_observation("MHJ45")
|
||||
assert Ionosphere.latest_observation("AL945")
|
||||
end
|
||||
|
||||
test "survives a station's fetch error and still ingests the others" do
|
||||
Req.Test.stub(GiroClient, fn conn ->
|
||||
params = Plug.Conn.fetch_query_params(conn).query_params
|
||||
|
||||
if params["ursiCode"] == "MHJ45" do
|
||||
Plug.Conn.resp(conn, 500, "boom")
|
||||
else
|
||||
Plug.Conn.resp(conn, 200, @sample)
|
||||
end
|
||||
end)
|
||||
|
||||
assert :ok = perform_job(IonosphereFetchWorker, %{})
|
||||
|
||||
# MHJ45 failed, AL945 ingested.
|
||||
assert Ionosphere.latest_observation("MHJ45") == nil
|
||||
assert Ionosphere.latest_observation("AL945")
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Reference in a new issue