From 361b9dec5a7456c9ef39da327309f0035ce2236d Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Wed, 15 Apr 2026 14:37:42 -0500 Subject: [PATCH] Ionosphere: GIRO ionosonde ingestion (foF2/foEs/hmF2/MUFD) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First step toward physics-based HF / sporadic-E scoring. Polls GIRO's DIDBase tabular endpoint every 10 minutes for two US ionosondes (Millstone Hill MHJ45, Alpena AL945), parses the plain-text response into observation rows, and upserts into a new ionosonde_observations table keyed on (station_code, valid_time). This gets foEs (E-layer sporadic critical frequency) into the database as a direct measurement — the key input for predicting 144 MHz Es openings via ITU-R P.534-6. foF2 + MUFD are the F2-layer inputs for HF MUF predictions. Not yet wired into scoring. Boulder/Wallops/Austin/Idaho/Point Arguello are in the GIRO catalog but were silent when probed — add them back if/when they come online. Next steps: SWPC JSON (Kp, F10.7, sunspot), GOES X-ray flux, D-RAP text, and the P.534-6 Es scoring factor that uses foEs at midpoint for the 144/440 band configs. --- config/config.exs | 8 +- config/runtime.exs | 9 +- config/test.exs | 1 + lib/microwaveprop/ionosphere.ex | 52 +++++ lib/microwaveprop/ionosphere/giro_client.ex | 195 ++++++++++++++++++ lib/microwaveprop/ionosphere/observation.ex | 39 ++++ .../workers/ionosphere_fetch_worker.ex | 62 ++++++ ...15193413_create_ionosonde_observations.exs | 22 ++ test/fixtures/giro/mhj45_sample.txt | 50 +++++ .../ionosphere/giro_client_test.exs | 69 +++++++ .../ionosphere/ionosphere_test.exs | 83 ++++++++ .../workers/ionosphere_fetch_worker_test.exs | 49 +++++ 12 files changed, 635 insertions(+), 4 deletions(-) create mode 100644 lib/microwaveprop/ionosphere.ex create mode 100644 lib/microwaveprop/ionosphere/giro_client.ex create mode 100644 lib/microwaveprop/ionosphere/observation.ex create mode 100644 lib/microwaveprop/workers/ionosphere_fetch_worker.ex create mode 100644 priv/repo/migrations/20260415193413_create_ionosonde_observations.exs create mode 100644 test/fixtures/giro/mhj45_sample.txt create mode 100644 test/microwaveprop/ionosphere/giro_client_test.exs create mode 100644 test/microwaveprop/ionosphere/ionosphere_test.exs create mode 100644 test/microwaveprop/workers/ionosphere_fetch_worker_test.exs diff --git a/config/config.exs b/config/config.exs index 91b10a2c..ff9ef825 100644 --- a/config/config.exs +++ b/config/config.exs @@ -67,7 +67,8 @@ config :microwaveprop, Oban, # forecast-hour chain job. propagation: 2, admin: 1, - nexrad: 2 + nexrad: 2, + ionosphere: 1 ], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, @@ -85,7 +86,10 @@ config :microwaveprop, Oban, {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch {"30 1 * * *", CanadianSoundingFetchWorker}, - {"30 13 * * *", CanadianSoundingFetchWorker} + {"30 13 * * *", CanadianSoundingFetchWorker}, + # GIRO ionosonde stations publish at ~7.5 min cadence; poll every + # 10 min to pick up fresh foF2/foEs/hmF2 within one cycle. + {"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker} ]} ] diff --git a/config/runtime.exs b/config/runtime.exs index e5a3f83a..da81377a 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -195,7 +195,8 @@ if config_env() == :prod do rtma: 2, backfill_enqueue: 1, admin: 1, - nexrad: 2 + nexrad: 2, + ionosphere: 1 ], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, @@ -236,7 +237,11 @@ if config_env() == :prod do args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}}, # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch {"30 1 * * *", CanadianSoundingFetchWorker}, - {"30 13 * * *", CanadianSoundingFetchWorker} + {"30 13 * * *", CanadianSoundingFetchWorker}, + # GIRO DIDBase publishes foF2/foEs/hmF2 at ~7.5 min cadence; poll + # every 10 min per station (Millstone Hill, Alpena) for the 144 + # MHz sporadic-E and HF MUF scoring inputs. + {"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker} ]} ] diff --git a/config/test.exs b/config/test.exs index 331c7972..a4f7c28a 100644 --- a/config/test.exs +++ b/config/test.exs @@ -59,6 +59,7 @@ config :microwaveprop, cache_contact_count: false config :microwaveprop, cache_contact_map: false config :microwaveprop, elevation_req_options: [plug: {Req.Test, Microwaveprop.Terrain.ElevationClient}, retry: false] config :microwaveprop, era5_req_options: [plug: {Req.Test, Microwaveprop.Weather.Era5Client}, retry: false] +config :microwaveprop, giro_req_options: [plug: {Req.Test, Microwaveprop.Ionosphere.GiroClient}, retry: false] config :microwaveprop, hrrr_req_options: [plug: {Req.Test, Microwaveprop.Weather.HrrrClient}, retry: false] config :microwaveprop, iem_req_options: [plug: {Req.Test, Microwaveprop.Weather.IemClient}, retry: false] diff --git a/lib/microwaveprop/ionosphere.ex b/lib/microwaveprop/ionosphere.ex new file mode 100644 index 00000000..80b8e6e8 --- /dev/null +++ b/lib/microwaveprop/ionosphere.ex @@ -0,0 +1,52 @@ +defmodule Microwaveprop.Ionosphere do + @moduledoc """ + Context for ionospheric observations (GIRO ionosonde data). Used by + the propagation scorer for VHF sporadic-E and HF MUF predictions. + """ + + import Ecto.Query + + alias Microwaveprop.Ionosphere.Observation + alias Microwaveprop.Repo + + @doc """ + Upsert a list of parsed `GiroClient` observations for a station. + Conflicts on `(station_code, valid_time)` replace every column except + the primary key and inserted_at. Returns `{:ok, count}`. + """ + @spec upsert_observations(String.t(), [map()]) :: {:ok, non_neg_integer()} + def upsert_observations(_station_code, []), do: {:ok, 0} + + def upsert_observations(station_code, observations) when is_list(observations) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + rows = + Enum.map(observations, fn obs -> + obs + |> Map.put(:station_code, station_code) + |> Map.put(:inserted_at, now) + |> Map.put(:updated_at, now) + end) + + {count, _} = + Repo.insert_all(Observation, rows, + on_conflict: {:replace_all_except, [:id, :inserted_at]}, + conflict_target: [:station_code, :valid_time] + ) + + {:ok, count} + end + + @doc """ + Returns the most recent `Observation` row for a station, or nil. + """ + @spec latest_observation(String.t()) :: Observation.t() | nil + def latest_observation(station_code) do + Repo.one( + from o in Observation, + where: o.station_code == ^station_code, + order_by: [desc: o.valid_time], + limit: 1 + ) + end +end diff --git a/lib/microwaveprop/ionosphere/giro_client.ex b/lib/microwaveprop/ionosphere/giro_client.ex new file mode 100644 index 00000000..6fcc2803 --- /dev/null +++ b/lib/microwaveprop/ionosphere/giro_client.ex @@ -0,0 +1,195 @@ +defmodule Microwaveprop.Ionosphere.GiroClient do + @moduledoc """ + Client for GIRO's `DIDBGetValues` tabular characteristic endpoint. + + GIRO (Global Ionospheric Radio Observatory, operated by LGDC at UMass + Lowell) publishes real-time ionosonde measurements for stations + worldwide. We poll the scaled characteristics for a handful of US + stations every ~10 minutes so the propagation scorer has a live view + of the ionosphere for 144 MHz sporadic-E and HF MUF predictions. + + The endpoint returns a plain-text tabular response, not JSON: + + # Global Ionospheric Radio Observatory + # Location: GEO 42.6N 288.5E, URSI-Code MHJ45 MILLSTONE HILL + # ...header comment block... + #Time CS foF2 QD MUFD QD foEs QD foE QD hmF2 QD + 2026-04-15T18:00:00.000Z 80 7.113 // 23.00 // 3.08 // 3.10 // 242.0 // + + Columns: + - `Time` — ISO-8601 UTC + - `CS` — autoscaling confidence score (0–100, 999 = manual, -1 = unknown) + - `foF2`, `foE`, `foEs` — critical frequencies in MHz + - `hmF2` — F2 peak height in km + - `MUFD` — maximum usable frequency for the distance D requested + + Each characteristic is followed by a QD (quality descriptor) flag — we + drop the QD column when indexing. Missing values are encoded as `---` + with a `__` QD flag and become `nil` in the parsed map. + """ + + require Logger + + @base_url "https://lgdc.uml.edu/common/DIDBGetValues" + + @default_chars ~w(foF2 foE foEs hmF2 MUFD) + @default_dmuf 3000 + @default_timeout_ms 15_000 + + @type observation :: %{ + valid_time: DateTime.t(), + confidence_score: integer() | nil, + fo_f2_mhz: float() | nil, + fo_e_mhz: float() | nil, + fo_es_mhz: float() | nil, + hm_f2_km: float() | nil, + mufd_mhz: float() | nil + } + + @doc """ + Fetches scaled characteristics for a station between two UTC DateTimes. + Returns `{:ok, [observation]}` or `{:error, reason}`. + """ + @spec fetch(String.t(), DateTime.t(), DateTime.t()) :: + {:ok, [observation()]} | {:error, term()} + def fetch(ursi_code, %DateTime{} = from_dt, %DateTime{} = to_dt) do + url = build_url(ursi_code, from_dt, to_dt) + + case Req.get(url, [receive_timeout: @default_timeout_ms] ++ req_options()) do + {:ok, %{status: 200, body: body}} when is_binary(body) -> + parse_tabular(body) + + {:ok, %{status: status, body: body}} -> + {:error, "GIRO HTTP #{status}: #{inspect(body)}"} + + {:error, reason} -> + {:error, "GIRO request failed: #{inspect(reason)}"} + end + end + + @doc """ + Parses a GIRO tabular response body into a list of observation maps. + """ + @spec parse_tabular(String.t()) :: {:ok, [observation()]} | {:error, atom()} + def parse_tabular(body) when is_binary(body) do + lines = String.split(body, ~r/\r?\n/, trim: true) + + case find_header(lines) do + {:ok, columns} -> + rows = + lines + |> Enum.reject(&header_or_comment?/1) + |> Enum.flat_map(&parse_data_line(&1, columns)) + + {:ok, rows} + + :error -> + {:error, :no_header} + end + end + + defp build_url(ursi_code, from_dt, to_dt) do + params = [ + {"ursiCode", ursi_code}, + {"charName", Enum.join(@default_chars, ",")}, + {"DMUF", Integer.to_string(@default_dmuf)}, + {"fromDate", format_giro_date(from_dt)}, + {"toDate", format_giro_date(to_dt)} + ] + + query = Enum.map_join(params, "&", fn {k, v} -> "#{k}=#{URI.encode(v)}" end) + "#{@base_url}?#{query}" + end + + # GIRO wants `YYYY.MM.DD+HH:MM:SS` with a literal `+`. Using `/` or + # url-escaping the space silently returns "no data". + defp format_giro_date(%DateTime{} = dt) do + dt + |> DateTime.truncate(:second) + |> Calendar.strftime("%Y.%m.%d+%H:%M:%S") + end + + # Column header line starts with `#Time`. Returns the list of value + # column names, stripping the `Time`/`CS` prefix columns and the `QD` + # filler tokens (which are paired with each value). + defp find_header(lines) do + case Enum.find(lines, &String.starts_with?(&1, "#Time")) do + nil -> + :error + + line -> + columns = + line + # Drop the leading "#" + |> String.trim_leading("#") + |> String.split(~r/\s+/, trim: true) + |> Enum.reject(&(&1 in ["Time", "CS", "QD"])) + + {:ok, columns} + end + end + + defp header_or_comment?(line), do: String.starts_with?(line, "#") + + defp parse_data_line(line, columns) do + tokens = String.split(line, ~r/\s+/, trim: true) + # Expected: time + CS + 2 tokens per column (value + QD) + expected = 2 + length(columns) * 2 + + if length(tokens) == expected do + [time_str, cs_str | rest] = tokens + + case DateTime.from_iso8601(time_str) do + {:ok, valid_time, _} -> + values = take_values(rest) + row = columns |> Enum.zip(values) |> Map.new() |> then(&build_observation(valid_time, cs_str, &1)) + [row] + + _ -> + [] + end + else + [] + end + end + + # Drop the QD flags by keeping every other element. + defp take_values([]), do: [] + defp take_values([value, _qd | rest]), do: [value | take_values(rest)] + defp take_values([value]), do: [value] + + defp build_observation(valid_time, cs_str, values_by_col) do + %{ + valid_time: DateTime.truncate(valid_time, :second), + confidence_score: parse_int(cs_str), + fo_f2_mhz: parse_float(values_by_col["foF2"]), + fo_e_mhz: parse_float(values_by_col["foE"]), + fo_es_mhz: parse_float(values_by_col["foEs"]), + hm_f2_km: parse_float(values_by_col["hmF2"]), + mufd_mhz: parse_float(values_by_col["MUFD"]) + } + end + + defp parse_int(nil), do: nil + + defp parse_int(s) do + case Integer.parse(s) do + {n, ""} -> n + _ -> nil + end + end + + defp parse_float(nil), do: nil + defp parse_float("---"), do: nil + + defp parse_float(s) do + case Float.parse(s) do + {f, ""} -> f + _ -> nil + end + end + + defp req_options do + Application.get_env(:microwaveprop, :giro_req_options, []) + end +end diff --git a/lib/microwaveprop/ionosphere/observation.ex b/lib/microwaveprop/ionosphere/observation.ex new file mode 100644 index 00000000..72286796 --- /dev/null +++ b/lib/microwaveprop/ionosphere/observation.ex @@ -0,0 +1,39 @@ +defmodule Microwaveprop.Ionosphere.Observation do + @moduledoc """ + A single ionosonde measurement for one station at one time. Populated + from GIRO's DIDBase scaled-characteristic feed and used by the + propagation scorer for sporadic-E and HF MUF predictions. + """ + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + schema "ionosonde_observations" do + field :station_code, :string + field :valid_time, :utc_datetime + field :confidence_score, :integer + field :fo_f2_mhz, :float + field :fo_e_mhz, :float + field :fo_es_mhz, :float + field :hm_f2_km, :float + field :mufd_mhz, :float + + timestamps(type: :utc_datetime) + end + + @type t :: %__MODULE__{} + + @required_fields ~w(station_code valid_time)a + @optional_fields ~w(confidence_score fo_f2_mhz fo_e_mhz fo_es_mhz hm_f2_km mufd_mhz)a + + @spec changeset(t() | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t() + def changeset(observation, attrs) do + observation + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) + |> unique_constraint([:station_code, :valid_time]) + end +end diff --git a/lib/microwaveprop/workers/ionosphere_fetch_worker.ex b/lib/microwaveprop/workers/ionosphere_fetch_worker.ex new file mode 100644 index 00000000..1c6b051d --- /dev/null +++ b/lib/microwaveprop/workers/ionosphere_fetch_worker.ex @@ -0,0 +1,62 @@ +defmodule Microwaveprop.Workers.IonosphereFetchWorker do + @moduledoc """ + Polls GIRO DIDBase for real-time ionosonde measurements (foF2, foE, + foEs, hmF2, MUFD) from a handful of CONUS stations. Drives the 144 + MHz sporadic-E and HF MUF scoring inputs. + + Runs on a ~10-minute cron — GIRO publishes each station at 7.5-minute + cadence so a 10-minute poll guarantees we pick up every new sample + within one cycle without hammering the endpoint. + + Pulls a 2-hour window each poll to catch any late-arriving samples + and to be resilient to the occasional missed run. Upserts are + idempotent on (station_code, valid_time). + """ + + use Oban.Worker, + queue: :ionosphere, + max_attempts: 3, + unique: [period: 300, states: [:available, :scheduled, :executing, :retryable]] + + alias Microwaveprop.Ionosphere + alias Microwaveprop.Ionosphere.GiroClient + + require Logger + + # CONUS stations that have been publishing reliably. Others (Boulder, + # Wallops, Austin, Idaho, Point Arguello) are in the GIRO catalog but + # intermittent — add them here once they come back online. + @stations [ + %{code: "MHJ45", name: "Millstone Hill MA", lat: 42.6, lon: -71.5}, + %{code: "AL945", name: "Alpena MI", lat: 45.1, lon: -83.6} + ] + + @lookback_seconds 2 * 3600 + + @impl Oban.Worker + def perform(%Oban.Job{}) do + now = DateTime.truncate(DateTime.utc_now(), :second) + from_dt = DateTime.add(now, -@lookback_seconds, :second) + + Enum.each(@stations, fn station -> + fetch_and_upsert(station, from_dt, now) + end) + + :ok + end + + @doc false + @spec stations() :: [map()] + def stations, do: @stations + + defp fetch_and_upsert(%{code: code, name: name}, from_dt, to_dt) do + case GiroClient.fetch(code, from_dt, to_dt) do + {:ok, observations} -> + {:ok, count} = Ionosphere.upsert_observations(code, observations) + Logger.info("IonosphereFetch: #{code} (#{name}) upserted #{count} observations") + + {:error, reason} -> + Logger.warning("IonosphereFetch: #{code} (#{name}) failed: #{inspect(reason)}") + end + end +end diff --git a/priv/repo/migrations/20260415193413_create_ionosonde_observations.exs b/priv/repo/migrations/20260415193413_create_ionosonde_observations.exs new file mode 100644 index 00000000..ef19cb77 --- /dev/null +++ b/priv/repo/migrations/20260415193413_create_ionosonde_observations.exs @@ -0,0 +1,22 @@ +defmodule Microwaveprop.Repo.Migrations.CreateIonosondeObservations do + use Ecto.Migration + + def change do + create table(:ionosonde_observations, primary_key: false) do + add :id, :binary_id, primary_key: true + add :station_code, :string, null: false + add :valid_time, :utc_datetime, null: false + add :confidence_score, :integer + add :fo_f2_mhz, :float + add :fo_e_mhz, :float + add :fo_es_mhz, :float + add :hm_f2_km, :float + add :mufd_mhz, :float + + timestamps(type: :utc_datetime) + end + + create unique_index(:ionosonde_observations, [:station_code, :valid_time]) + create index(:ionosonde_observations, [:valid_time]) + end +end diff --git a/test/fixtures/giro/mhj45_sample.txt b/test/fixtures/giro/mhj45_sample.txt new file mode 100644 index 00000000..3e9549d7 --- /dev/null +++ b/test/fixtures/giro/mhj45_sample.txt @@ -0,0 +1,50 @@ +# Global Ionospheric Radio Observatory +# GIRO Tabulated Ionospheric Characteristics, Version 1.0 Revision B +# Generated by DIDBGetValues on 2026-04-15T19:32:24.784Z +# +# Location: GEO 42.6N 288.5E, URSI-Code MHJ45 MILLSTONE HILL +# Instrument: Ionosonde, Model: DPS-4D +# +# Query for measurement intervals of time: +# 2026-04-15T16:00:00.000Z - 2026-04-15T19:00:00.000Z +# +# Data Selection: +# CS is Autoscaling Confidence Score (from 0 to 100, 999 if manual scaling, -1 if unknown) +# foF2 [MHz] - F2 layer critical frequency +# MUFD [MHz] - Maximum usable frequency for ground distance D +# foEs [MHz] - Es layer critical frequency +# foE [MHz] - E layer critical frequency +# hmF2 [km] - Peak height F2-layer +# Distance D for MUF calculations: 3000 km +# +# All GIRO measurements are released under CC-BY-NC-SA 4.0 license +# Please follow the Lowell GIRO Data Center RULES OF THE ROAD +# https://ulcar.uml.edu/DIDBase/RulesOfTheRoadForDIDBase.htm +# Requires acknowledgement of MHJ45 data provider +# +#Time CS foF2 QD MUFD QD foEs QD foE QD hmF2 QD +2026-04-15T16:00:00.000Z 60 6.856 // 23.63 // --- __ 3.50 // 217.2 // +2026-04-15T16:07:30.000Z 90 6.763 // 21.10 // 3.42 // 3.47 // 260.3 // +2026-04-15T16:15:00.000Z 70 6.863 // 22.72 // 3.50 // 3.50 // 225.1 // +2026-04-15T16:22:30.000Z 70 6.863 // 22.39 // 3.45 // 3.45 // 231.8 // +2026-04-15T16:30:00.000Z 75 6.775 // 22.72 // --- __ 3.30 // 224.3 // +2026-04-15T16:37:30.000Z 70 6.913 // 23.39 // 3.08 // 3.15 // 226.6 // +2026-04-15T16:45:00.000Z 60 6.925 // 22.41 // 3.05 // 3.13 // 226.6 // +2026-04-15T16:52:30.000Z 70 6.975 // 23.32 // --- __ 3.30 // 212.0 // +2026-04-15T17:00:00.000Z 90 6.907 // 21.87 // --- __ 3.20 // 232.4 // +2026-04-15T17:07:30.000Z 60 7.063 // 23.52 // 3.13 // 3.15 // 236.4 // +2026-04-15T17:15:00.000Z 75 7.225 // 23.38 // 3.20 // 3.38 // 244.2 // +2026-04-15T17:22:30.000Z 70 7.138 // 23.45 // 3.40 // 3.35 // 234.8 // +2026-04-15T17:30:00.000Z 75 7.063 // 22.66 // --- __ 3.35 // 238.8 // +2026-04-15T17:37:30.000Z 70 7.025 // 22.19 // 3.08 // 3.28 // 240.9 // +2026-04-15T17:45:00.000Z 70 7.138 // 22.25 // --- __ 3.08 // 256.2 // +2026-04-15T17:52:30.000Z 70 7.138 // 22.97 // 3.13 // 2.97 // 242.6 // +2026-04-15T18:00:00.000Z 80 7.113 // 23.00 // 3.08 // 3.10 // 242.0 // +2026-04-15T18:07:30.000Z 80 7.088 // 22.76 // 3.35 // 3.13 // 246.0 // +2026-04-15T18:15:00.000Z 70 7.150 // 22.09 // 3.42 // 2.92 // 266.7 // +2026-04-15T18:22:30.000Z 75 7.200 // 22.37 // 3.45 // 2.80 // 262.0 // +2026-04-15T18:30:00.000Z 70 7.225 // 23.34 // 3.22 // 2.85 // 246.5 // +2026-04-15T18:37:30.000Z 75 7.213 // 23.03 // --- __ 3.17 // 250.9 // +2026-04-15T18:45:00.000Z 75 7.338 // 22.77 // 3.20 // 2.72 // 263.5 // +2026-04-15T18:52:30.000Z 75 7.413 // 23.50 // 3.33 // 3.10 // 256.5 // +2026-04-15T19:00:00.000Z 75 7.400 // 23.64 // 3.50 // 3.15 // 249.4 // diff --git a/test/microwaveprop/ionosphere/giro_client_test.exs b/test/microwaveprop/ionosphere/giro_client_test.exs new file mode 100644 index 00000000..9b45d675 --- /dev/null +++ b/test/microwaveprop/ionosphere/giro_client_test.exs @@ -0,0 +1,69 @@ +defmodule Microwaveprop.Ionosphere.GiroClientTest do + use ExUnit.Case, async: true + + alias Microwaveprop.Ionosphere.GiroClient + + describe "parse_tabular/1" do + test "decodes the real Millstone Hill fixture into observation maps" do + body = File.read!("test/fixtures/giro/mhj45_sample.txt") + + assert {:ok, rows} = GiroClient.parse_tabular(body) + + # The fixture is 2026-04-15 16:00 → 19:00 UTC at 7.5-minute cadence, + # so 25 observations. + assert length(rows) == 25 + + first = hd(rows) + assert first.valid_time == ~U[2026-04-15 16:00:00Z] + assert first.confidence_score == 60 + assert_in_delta first.fo_f2_mhz, 6.856, 0.001 + assert_in_delta first.mufd_mhz, 23.63, 0.001 + # foEs was missing ("---") at 16:00. + assert first.fo_es_mhz == nil + assert_in_delta first.fo_e_mhz, 3.50, 0.001 + assert_in_delta first.hm_f2_km, 217.2, 0.1 + + # 16:07:30 has foEs == 3.42. + second = Enum.at(rows, 1) + assert second.valid_time == ~U[2026-04-15 16:07:30Z] + assert_in_delta second.fo_es_mhz, 3.42, 0.001 + + last = List.last(rows) + assert last.valid_time == ~U[2026-04-15 19:00:00Z] + assert_in_delta last.fo_f2_mhz, 7.400, 0.001 + end + + test "returns {:ok, []} for a body with only the header" do + body = """ + # Global Ionospheric Radio Observatory + # Location: GEO 42.6N 288.5E, URSI-Code MHJ45 MILLSTONE HILL + # + #Time CS foF2 QD MUFD QD foEs QD foE QD hmF2 QD + """ + + assert {:ok, []} = GiroClient.parse_tabular(body) + end + + test "returns {:error, :no_header} when the column header is missing" do + body = """ + # some junk + 2026-04-15T18:00:00.000Z 80 7.113 // 23.00 // + """ + + assert {:error, :no_header} = GiroClient.parse_tabular(body) + end + + test "skips data lines whose token count doesn't match the header" do + # Header declares 5 value columns (10 tokens after time+CS) but the + # data row is truncated. Should be skipped rather than crashing. + body = """ + #Time CS foF2 QD MUFD QD foEs QD foE QD hmF2 QD + 2026-04-15T18:00:00.000Z 80 7.113 // + 2026-04-15T18:07:30.000Z 80 7.088 // 22.76 // 3.35 // 3.13 // 246.0 // + """ + + assert {:ok, [row]} = GiroClient.parse_tabular(body) + assert row.valid_time == ~U[2026-04-15 18:07:30Z] + end + end +end diff --git a/test/microwaveprop/ionosphere/ionosphere_test.exs b/test/microwaveprop/ionosphere/ionosphere_test.exs new file mode 100644 index 00000000..8392b3a1 --- /dev/null +++ b/test/microwaveprop/ionosphere/ionosphere_test.exs @@ -0,0 +1,83 @@ +defmodule Microwaveprop.IonosphereTest do + use Microwaveprop.DataCase, async: true + + alias Microwaveprop.Ionosphere + alias Microwaveprop.Ionosphere.Observation + + @valid_attrs %{ + station_code: "MHJ45", + valid_time: ~U[2026-04-15 18:00:00Z], + confidence_score: 80, + fo_f2_mhz: 7.113, + fo_e_mhz: 3.10, + fo_es_mhz: 3.08, + hm_f2_km: 242.0, + mufd_mhz: 23.00 + } + + describe "upsert_observations/2" do + test "inserts new observations and returns the inserted count" do + assert {:ok, 2} = + Ionosphere.upsert_observations("MHJ45", [ + Map.delete(@valid_attrs, :station_code), + @valid_attrs + |> Map.delete(:station_code) + |> Map.put(:valid_time, ~U[2026-04-15 18:07:30Z]) + ]) + + assert Repo.aggregate(Observation, :count, :id) == 2 + end + + test "re-upserting the same (station, valid_time) replaces fields and does not duplicate" do + {:ok, 1} = Ionosphere.upsert_observations("MHJ45", [Map.delete(@valid_attrs, :station_code)]) + + updated = + @valid_attrs + |> Map.delete(:station_code) + |> Map.put(:fo_es_mhz, 9.5) + + assert {:ok, 1} = Ionosphere.upsert_observations("MHJ45", [updated]) + assert Repo.aggregate(Observation, :count, :id) == 1 + + [row] = Repo.all(Observation) + assert row.fo_es_mhz == 9.5 + end + + test "stamps station_code on each observation" do + {:ok, 1} = + Ionosphere.upsert_observations("AL945", [ + Map.delete(@valid_attrs, :station_code) + ]) + + [row] = Repo.all(Observation) + assert row.station_code == "AL945" + end + + test "returns 0 and does nothing for an empty list" do + assert {:ok, 0} = Ionosphere.upsert_observations("MHJ45", []) + assert Repo.aggregate(Observation, :count, :id) == 0 + end + end + + describe "latest_observation/1" do + test "returns the most recent observation for a station" do + older = Map.delete(@valid_attrs, :station_code) + + newer = + @valid_attrs + |> Map.delete(:station_code) + |> Map.put(:valid_time, ~U[2026-04-15 19:00:00Z]) + |> Map.put(:fo_es_mhz, 10.0) + + {:ok, _} = Ionosphere.upsert_observations("MHJ45", [older, newer]) + + row = Ionosphere.latest_observation("MHJ45") + assert row.valid_time == ~U[2026-04-15 19:00:00Z] + assert row.fo_es_mhz == 10.0 + end + + test "returns nil when no observations exist for the station" do + assert Ionosphere.latest_observation("MHJ45") == nil + end + end +end diff --git a/test/microwaveprop/workers/ionosphere_fetch_worker_test.exs b/test/microwaveprop/workers/ionosphere_fetch_worker_test.exs new file mode 100644 index 00000000..27616f29 --- /dev/null +++ b/test/microwaveprop/workers/ionosphere_fetch_worker_test.exs @@ -0,0 +1,49 @@ +defmodule Microwaveprop.Workers.IonosphereFetchWorkerTest do + use Microwaveprop.DataCase, async: false + use Oban.Testing, repo: Microwaveprop.Repo + + alias Microwaveprop.Ionosphere + alias Microwaveprop.Ionosphere.GiroClient + alias Microwaveprop.Ionosphere.Observation + alias Microwaveprop.Workers.IonosphereFetchWorker + + @sample File.read!("test/fixtures/giro/mhj45_sample.txt") + + describe "perform/1" do + test "pulls each configured station and upserts the parsed rows" do + Req.Test.stub(GiroClient, fn conn -> + assert conn.method == "GET" + params = Plug.Conn.fetch_query_params(conn).query_params + assert params["ursiCode"] in ~w(MHJ45 AL945) + + Plug.Conn.resp(conn, 200, @sample) + end) + + assert :ok = perform_job(IonosphereFetchWorker, %{}) + + # 25 observations per station × 2 stations = 50 rows (fixture has 25 + # data lines; upserts go to each station code separately). + assert Repo.aggregate(Observation, :count, :id) == 50 + assert Ionosphere.latest_observation("MHJ45") + assert Ionosphere.latest_observation("AL945") + end + + test "survives a station's fetch error and still ingests the others" do + Req.Test.stub(GiroClient, fn conn -> + params = Plug.Conn.fetch_query_params(conn).query_params + + if params["ursiCode"] == "MHJ45" do + Plug.Conn.resp(conn, 500, "boom") + else + Plug.Conn.resp(conn, 200, @sample) + end + end) + + assert :ok = perform_job(IonosphereFetchWorker, %{}) + + # MHJ45 failed, AL945 ingested. + assert Ionosphere.latest_observation("MHJ45") == nil + assert Ionosphere.latest_observation("AL945") + end + end +end