From 923c8a468dddf6187c8f6067eb854d5fb2006dd2 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Wed, 15 Apr 2026 14:55:23 -0500 Subject: [PATCH] SpaceWeather: SWPC JSON ingestion (Kp, F10.7, GOES X-ray) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the second ionospheric data source layer. Polls NOAA SWPC's free public JSON services every 5 minutes for three products: * Planetary K-index (1-min cadence) → geomagnetic_observations * 10.7 cm solar flux (hourly) → solar_flux_observations * GOES X-ray flux (1-min, 0.1-0.8nm long-wave band only) → solar_xray_observations All three products feed HF / Es scoring inputs that the propagation engine will start using in a follow-up commit: * Kp drives HF absorption and Es damping (memory notes Kp is inversely correlated with tropo/Es stability). * F10.7 is the SSN proxy for ITU-R P.533 HF MUF prediction. * GOES X-ray long-wave flux is the short-wave fade (SWF) / D-layer absorption proxy — a C-class flare starts attenuating HF within seconds of the X-ray peak. New context `Microwaveprop.SpaceWeather` exposes `upsert_*/1` and `latest_*/0` per product; `SpaceWeatherFetchWorker` pulls all three independently so one endpoint's outage doesn't block the others. Fixtures captured from live SWPC endpoints on 2026-04-15 for the parser tests. Not yet wired into scoring — that's a separate commit. --- config/config.exs | 8 +- config/runtime.exs | 8 +- config/test.exs | 1 + lib/microwaveprop/space_weather.ex | 78 ++++++ .../space_weather/geomagnetic_observation.ex | 30 +++ .../space_weather/solar_flux_observation.ex | 32 +++ .../space_weather/solar_xray_observation.ex | 32 +++ .../space_weather/swpc_client.ex | 227 ++++++++++++++++++ .../workers/space_weather_fetch_worker.ex | 43 ++++ ...5208_create_space_weather_observations.exs | 42 ++++ test/fixtures/swpc/f107_sample.json | 92 +++++++ test/fixtures/swpc/kp_sample.json | 62 +++++ test/fixtures/swpc/xrays_sample.json | 110 +++++++++ .../space_weather/space_weather_test.exs | 124 ++++++++++ .../space_weather/swpc_client_test.exs | 63 +++++ .../space_weather_fetch_worker_test.exs | 55 +++++ 16 files changed, 1003 insertions(+), 4 deletions(-) create mode 100644 lib/microwaveprop/space_weather.ex create mode 100644 lib/microwaveprop/space_weather/geomagnetic_observation.ex create mode 100644 lib/microwaveprop/space_weather/solar_flux_observation.ex create mode 100644 lib/microwaveprop/space_weather/solar_xray_observation.ex create mode 100644 lib/microwaveprop/space_weather/swpc_client.ex create mode 100644 lib/microwaveprop/workers/space_weather_fetch_worker.ex create mode 100644 priv/repo/migrations/20260415195208_create_space_weather_observations.exs create mode 100644 test/fixtures/swpc/f107_sample.json create mode 100644 test/fixtures/swpc/kp_sample.json create mode 100644 test/fixtures/swpc/xrays_sample.json create mode 100644 test/microwaveprop/space_weather/space_weather_test.exs create mode 100644 test/microwaveprop/space_weather/swpc_client_test.exs create mode 100644 test/microwaveprop/workers/space_weather_fetch_worker_test.exs diff --git a/config/config.exs b/config/config.exs index ff9ef825..b095dd43 100644 --- a/config/config.exs +++ b/config/config.exs @@ -68,7 +68,8 @@ config :microwaveprop, Oban, propagation: 2, admin: 1, nexrad: 2, - ionosphere: 1 + ionosphere: 1, + space_weather: 1 ], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, @@ -89,7 +90,10 @@ config :microwaveprop, Oban, {"30 13 * * *", CanadianSoundingFetchWorker}, # GIRO ionosonde stations publish at ~7.5 min cadence; poll every # 10 min to pick up fresh foF2/foEs/hmF2 within one cycle. - {"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker} + {"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker}, + # NOAA SWPC publishes Kp and GOES X-ray at 1-min cadence and F10.7 + # hourly; poll every 5 min for reasonable freshness vs request volume. + {"*/5 * * * *", Microwaveprop.Workers.SpaceWeatherFetchWorker} ]} ] diff --git a/config/runtime.exs b/config/runtime.exs index da81377a..c8e71051 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -196,7 +196,8 @@ if config_env() == :prod do backfill_enqueue: 1, admin: 1, nexrad: 2, - ionosphere: 1 + ionosphere: 1, + space_weather: 1 ], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, @@ -241,7 +242,10 @@ if config_env() == :prod do # GIRO DIDBase publishes foF2/foEs/hmF2 at ~7.5 min cadence; poll # every 10 min per station (Millstone Hill, Alpena) for the 144 # MHz sporadic-E and HF MUF scoring inputs. - {"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker} + {"*/10 * * * *", Microwaveprop.Workers.IonosphereFetchWorker}, + # NOAA SWPC: Kp / GOES X-ray at 1-min cadence, F10.7 hourly. + # 5-min poll balances freshness vs request volume. + {"*/5 * * * *", Microwaveprop.Workers.SpaceWeatherFetchWorker} ]} ] diff --git a/config/test.exs b/config/test.exs index a4f7c28a..c5187a0f 100644 --- a/config/test.exs +++ b/config/test.exs @@ -68,6 +68,7 @@ config :microwaveprop, nexrad_req_options: [plug: {Req.Test, Microwaveprop.Weath config :microwaveprop, solar_req_options: [plug: {Req.Test, Microwaveprop.Weather.SolarClient}] config :microwaveprop, srtm_req_options: [plug: {Req.Test, Microwaveprop.Terrain.Srtm}, retry: false] config :microwaveprop, start_freshness_monitor: false +config :microwaveprop, swpc_req_options: [plug: {Req.Test, Microwaveprop.SpaceWeather.SwpcClient}, retry: false] config :microwaveprop, uwyo_req_options: [plug: {Req.Test, Microwaveprop.Weather.UwyoSoundingClient}, retry: false] # Initialize plugs at runtime for faster test compilation diff --git a/lib/microwaveprop/space_weather.ex b/lib/microwaveprop/space_weather.ex new file mode 100644 index 00000000..14c38781 --- /dev/null +++ b/lib/microwaveprop/space_weather.ex @@ -0,0 +1,78 @@ +defmodule Microwaveprop.SpaceWeather do + @moduledoc """ + Context for NOAA SWPC space-weather observations: planetary K-index, + 10.7 cm solar flux, and GOES X-ray flux. Used by the propagation + scorer for HF MUF prediction (F10.7 → SSN proxy), geomagnetic storm + effects (Kp), and short-wave fade detection (GOES X-ray flares). + """ + + import Ecto.Query + + alias Microwaveprop.Repo + alias Microwaveprop.SpaceWeather.GeomagneticObservation + alias Microwaveprop.SpaceWeather.SolarFluxObservation + alias Microwaveprop.SpaceWeather.SolarXrayObservation + + # ---------- Kp ---------- + + @spec upsert_kp([map()]) :: {:ok, non_neg_integer()} + def upsert_kp([]), do: {:ok, 0} + + def upsert_kp(rows) when is_list(rows) do + stamp_upsert(GeomagneticObservation, rows, conflict_target: :valid_time) + end + + @spec latest_kp() :: GeomagneticObservation.t() | nil + def latest_kp do + Repo.one(from o in GeomagneticObservation, order_by: [desc: o.valid_time], limit: 1) + end + + # ---------- F10.7 ---------- + + @spec upsert_solar_flux([map()]) :: {:ok, non_neg_integer()} + def upsert_solar_flux([]), do: {:ok, 0} + + def upsert_solar_flux(rows) when is_list(rows) do + stamp_upsert(SolarFluxObservation, rows, conflict_target: :valid_time) + end + + @spec latest_f107() :: SolarFluxObservation.t() | nil + def latest_f107 do + Repo.one(from o in SolarFluxObservation, order_by: [desc: o.valid_time], limit: 1) + end + + # ---------- GOES X-ray ---------- + + @spec upsert_xray([map()]) :: {:ok, non_neg_integer()} + def upsert_xray([]), do: {:ok, 0} + + def upsert_xray(rows) when is_list(rows) do + stamp_upsert(SolarXrayObservation, rows, conflict_target: [:valid_time, :energy_band]) + end + + @spec latest_xray() :: SolarXrayObservation.t() | nil + def latest_xray do + Repo.one(from o in SolarXrayObservation, order_by: [desc: o.valid_time], limit: 1) + end + + # ---------- Shared ---------- + + defp stamp_upsert(schema, rows, opts) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + stamped = + Enum.map(rows, fn row -> + row + |> Map.put(:inserted_at, now) + |> Map.put(:updated_at, now) + end) + + {count, _} = + Repo.insert_all(schema, stamped, + on_conflict: {:replace_all_except, [:id, :inserted_at]}, + conflict_target: Keyword.fetch!(opts, :conflict_target) + ) + + {:ok, count} + end +end diff --git a/lib/microwaveprop/space_weather/geomagnetic_observation.ex b/lib/microwaveprop/space_weather/geomagnetic_observation.ex new file mode 100644 index 00000000..1b94e5bb --- /dev/null +++ b/lib/microwaveprop/space_weather/geomagnetic_observation.ex @@ -0,0 +1,30 @@ +defmodule Microwaveprop.SpaceWeather.GeomagneticObservation do + @moduledoc false + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + schema "geomagnetic_observations" do + field :valid_time, :utc_datetime + field :kp_index, :integer + field :estimated_kp, :float + + timestamps(type: :utc_datetime) + end + + @type t :: %__MODULE__{} + + @required_fields ~w(valid_time)a + @optional_fields ~w(kp_index estimated_kp)a + + @spec changeset(t() | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t() + def changeset(obs, attrs) do + obs + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) + |> unique_constraint(:valid_time) + end +end diff --git a/lib/microwaveprop/space_weather/solar_flux_observation.ex b/lib/microwaveprop/space_weather/solar_flux_observation.ex new file mode 100644 index 00000000..d87e5e03 --- /dev/null +++ b/lib/microwaveprop/space_weather/solar_flux_observation.ex @@ -0,0 +1,32 @@ +defmodule Microwaveprop.SpaceWeather.SolarFluxObservation do + @moduledoc false + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + schema "solar_flux_observations" do + field :valid_time, :utc_datetime + field :frequency_mhz, :integer + field :flux_sfu, :float + field :reporting_schedule, :string + field :ninety_day_mean, :float + + timestamps(type: :utc_datetime) + end + + @type t :: %__MODULE__{} + + @required_fields ~w(valid_time)a + @optional_fields ~w(frequency_mhz flux_sfu reporting_schedule ninety_day_mean)a + + @spec changeset(t() | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t() + def changeset(obs, attrs) do + obs + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) + |> unique_constraint(:valid_time) + end +end diff --git a/lib/microwaveprop/space_weather/solar_xray_observation.ex b/lib/microwaveprop/space_weather/solar_xray_observation.ex new file mode 100644 index 00000000..88e45840 --- /dev/null +++ b/lib/microwaveprop/space_weather/solar_xray_observation.ex @@ -0,0 +1,32 @@ +defmodule Microwaveprop.SpaceWeather.SolarXrayObservation do + @moduledoc false + use Ecto.Schema + + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + schema "solar_xray_observations" do + field :valid_time, :utc_datetime + field :satellite, :integer + field :flux_wm2, :float + field :energy_band, :string + field :electron_contaminated, :boolean, default: false + + timestamps(type: :utc_datetime) + end + + @type t :: %__MODULE__{} + + @required_fields ~w(valid_time)a + @optional_fields ~w(satellite flux_wm2 energy_band electron_contaminated)a + + @spec changeset(t() | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t() + def changeset(obs, attrs) do + obs + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) + |> unique_constraint([:valid_time, :energy_band]) + end +end diff --git a/lib/microwaveprop/space_weather/swpc_client.ex b/lib/microwaveprop/space_weather/swpc_client.ex new file mode 100644 index 00000000..7a58a069 --- /dev/null +++ b/lib/microwaveprop/space_weather/swpc_client.ex @@ -0,0 +1,227 @@ +defmodule Microwaveprop.SpaceWeather.SwpcClient do + @moduledoc """ + Client for NOAA SWPC's public JSON services (https://services.swpc.noaa.gov/). + + All three products we care about for HF / Es scoring are free, no-auth + JSON endpoints: + + - `/json/planetary_k_index_1m.json` — planetary K 1-min cadence + - `/json/f107_cm_flux.json` — 10.7 cm solar flux (hourly) + - `/json/goes/primary/xrays-1-day.json` — GOES X-ray flux 1-min, 2 bands + + Each product has its own wire format so each has its own `fetch_*` + + `parse_*` pair. Parsed rows are shaped for direct insert into the + corresponding schemas via `Microwaveprop.SpaceWeather` context. + """ + + require Logger + + @base_url "https://services.swpc.noaa.gov" + @default_timeout_ms 15_000 + + # ---------- Kp ---------- + + @doc """ + Fetch and parse the planetary K 1-min feed. Returns `{:ok, [row]}` where + each row is `%{valid_time, kp_index, estimated_kp}`. + """ + @spec fetch_kp() :: {:ok, [map()]} | {:error, term()} + def fetch_kp, do: fetch_and_parse("/json/planetary_k_index_1m.json", &parse_kp/1) + + @doc false + @spec parse_kp(String.t()) :: {:ok, [map()]} | {:error, term()} + def parse_kp(body) do + with {:ok, decoded} <- decode_array(body) do + rows = + decoded + |> Enum.map(&parse_kp_row/1) + |> Enum.reject(&is_nil/1) + + {:ok, rows} + end + end + + defp parse_kp_row(%{"time_tag" => t, "kp_index" => kp_index} = row) do + case parse_naive_iso(t) do + {:ok, valid_time} -> + %{ + valid_time: valid_time, + kp_index: to_integer(kp_index), + estimated_kp: to_float(Map.get(row, "estimated_kp")) + } + + :error -> + nil + end + end + + defp parse_kp_row(_), do: nil + + # ---------- F10.7 ---------- + + @doc """ + Fetch and parse the F10.7 cm flux feed. Returns `{:ok, [row]}` where + each row is `%{valid_time, frequency_mhz, flux_sfu, reporting_schedule, + ninety_day_mean}`. + """ + @spec fetch_f107() :: {:ok, [map()]} | {:error, term()} + def fetch_f107, do: fetch_and_parse("/json/f107_cm_flux.json", &parse_f107/1) + + @doc false + @spec parse_f107(String.t()) :: {:ok, [map()]} | {:error, term()} + def parse_f107(body) do + with {:ok, decoded} <- decode_array(body) do + rows = + decoded + |> Enum.map(&parse_f107_row/1) + |> Enum.reject(&is_nil/1) + + {:ok, rows} + end + end + + defp parse_f107_row(%{"time_tag" => t, "flux" => flux} = row) do + case parse_naive_iso(t) do + {:ok, valid_time} -> + %{ + valid_time: valid_time, + frequency_mhz: to_integer(Map.get(row, "frequency")), + flux_sfu: to_float(flux), + reporting_schedule: Map.get(row, "reporting_schedule"), + ninety_day_mean: to_float(Map.get(row, "ninety_day_mean")) + } + + :error -> + nil + end + end + + defp parse_f107_row(_), do: nil + + # ---------- GOES X-ray ---------- + + @long_band "0.1-0.8nm" + + @doc """ + Fetch and parse the GOES primary 1-day X-ray feed. Only keeps the + 0.1–0.8nm long-wavelength band (the standard SWF / D-layer absorption + proxy). Returns `{:ok, [row]}` where each row is `%{valid_time, + satellite, flux_wm2, energy_band, electron_contaminated}`. + """ + @spec fetch_xrays() :: {:ok, [map()]} | {:error, term()} + def fetch_xrays, do: fetch_and_parse("/json/goes/primary/xrays-1-day.json", &parse_xrays/1) + + @doc false + @spec parse_xrays(String.t()) :: {:ok, [map()]} | {:error, term()} + def parse_xrays(body) do + with {:ok, decoded} <- decode_array(body) do + rows = + decoded + |> Enum.filter(&long_band?/1) + |> Enum.map(&parse_xray_row/1) + |> Enum.reject(&is_nil/1) + + {:ok, rows} + end + end + + defp long_band?(%{"energy" => @long_band}), do: true + defp long_band?(_), do: false + + defp parse_xray_row(%{"time_tag" => t, "flux" => flux} = row) do + case parse_iso_z(t) do + {:ok, valid_time} -> + %{ + valid_time: valid_time, + satellite: to_integer(Map.get(row, "satellite")), + flux_wm2: to_float(flux), + energy_band: Map.get(row, "energy"), + # SWPC's field name is misspelled "electron_contaminaton" upstream. + electron_contaminated: !!Map.get(row, "electron_contaminaton") + } + + :error -> + nil + end + end + + defp parse_xray_row(_), do: nil + + # ---------- Shared HTTP / parsing helpers ---------- + + defp fetch_and_parse(path, parser) do + url = @base_url <> path + + case Req.get(url, [receive_timeout: @default_timeout_ms] ++ req_options()) do + {:ok, %{status: 200, body: body}} -> + parser.(body) + + {:ok, %{status: status, body: body}} -> + {:error, "SWPC HTTP #{status}: #{inspect(body)}"} + + {:error, reason} -> + {:error, "SWPC request failed: #{inspect(reason)}"} + end + end + + defp decode_array(body) when is_binary(body) do + case Jason.decode(body) do + {:ok, list} when is_list(list) -> {:ok, list} + {:ok, _other} -> {:error, :not_a_list} + {:error, reason} -> {:error, reason} + end + end + + # Req's JSON auto-decode may already have parsed the body if the + # upstream set Content-Type: application/json. In that case body is a + # list, not a binary. + defp decode_array(list) when is_list(list), do: {:ok, list} + defp decode_array(_), do: {:error, :not_a_list} + + # SWPC's Kp/F10.7 time_tag is naive ISO ("2026-04-15T13:49:00") — + # documented UTC but without a Z suffix. Add the Z so DateTime.from_iso8601 + # parses it as UTC. + defp parse_naive_iso(s) when is_binary(s) do + case DateTime.from_iso8601(s <> "Z") do + {:ok, dt, _} -> {:ok, DateTime.truncate(dt, :second)} + _ -> :error + end + end + + defp parse_naive_iso(_), do: :error + + defp parse_iso_z(s) when is_binary(s) do + case DateTime.from_iso8601(s) do + {:ok, dt, _} -> {:ok, DateTime.truncate(dt, :second)} + _ -> :error + end + end + + defp parse_iso_z(_), do: :error + + defp to_integer(nil), do: nil + defp to_integer(n) when is_integer(n), do: n + defp to_integer(n) when is_float(n), do: trunc(n) + + defp to_integer(s) when is_binary(s) do + case Integer.parse(s) do + {n, ""} -> n + _ -> nil + end + end + + defp to_float(nil), do: nil + defp to_float(n) when is_float(n), do: n + defp to_float(n) when is_integer(n), do: n * 1.0 + + defp to_float(s) when is_binary(s) do + case Float.parse(s) do + {f, ""} -> f + _ -> nil + end + end + + defp req_options do + Application.get_env(:microwaveprop, :swpc_req_options, []) + end +end diff --git a/lib/microwaveprop/workers/space_weather_fetch_worker.ex b/lib/microwaveprop/workers/space_weather_fetch_worker.ex new file mode 100644 index 00000000..02202101 --- /dev/null +++ b/lib/microwaveprop/workers/space_weather_fetch_worker.ex @@ -0,0 +1,43 @@ +defmodule Microwaveprop.Workers.SpaceWeatherFetchWorker do + @moduledoc """ + Polls NOAA SWPC's free public JSON services for the three space-weather + products we use for HF / Es scoring: + + - Planetary K-index (1-min cadence) — geomagnetic disturbance level + - 10.7 cm solar flux (hourly) — SSN proxy for HF MUF prediction + - GOES X-ray flux (1-min, 0.1-0.8nm band) — short-wave fade from flares + + Runs on a 5-minute cron. SWPC publishes Kp and X-ray at 1-min cadence; + 5 minutes strikes a balance between freshness and request volume. Each + fetch is independent: a failure in one product doesn't block the others. + """ + + use Oban.Worker, + queue: :space_weather, + max_attempts: 3, + unique: [period: 120, states: [:available, :scheduled, :executing, :retryable]] + + alias Microwaveprop.SpaceWeather + alias Microwaveprop.SpaceWeather.SwpcClient + + require Logger + + @impl Oban.Worker + def perform(%Oban.Job{}) do + fetch_and_upsert(:kp, &SwpcClient.fetch_kp/0, &SpaceWeather.upsert_kp/1) + fetch_and_upsert(:f107, &SwpcClient.fetch_f107/0, &SpaceWeather.upsert_solar_flux/1) + fetch_and_upsert(:xrays, &SwpcClient.fetch_xrays/0, &SpaceWeather.upsert_xray/1) + :ok + end + + defp fetch_and_upsert(name, fetcher, upserter) do + case fetcher.() do + {:ok, rows} -> + {:ok, count} = upserter.(rows) + Logger.info("SpaceWeatherFetch: #{name} upserted #{count} rows") + + {:error, reason} -> + Logger.warning("SpaceWeatherFetch: #{name} failed: #{inspect(reason)}") + end + end +end diff --git a/priv/repo/migrations/20260415195208_create_space_weather_observations.exs b/priv/repo/migrations/20260415195208_create_space_weather_observations.exs new file mode 100644 index 00000000..5b6b5c7a --- /dev/null +++ b/priv/repo/migrations/20260415195208_create_space_weather_observations.exs @@ -0,0 +1,42 @@ +defmodule Microwaveprop.Repo.Migrations.CreateSpaceWeatherObservations do + use Ecto.Migration + + def change do + create table(:geomagnetic_observations, primary_key: false) do + add :id, :binary_id, primary_key: true + add :valid_time, :utc_datetime, null: false + add :kp_index, :integer + add :estimated_kp, :float + + timestamps(type: :utc_datetime) + end + + create unique_index(:geomagnetic_observations, [:valid_time]) + + create table(:solar_flux_observations, primary_key: false) do + add :id, :binary_id, primary_key: true + add :valid_time, :utc_datetime, null: false + add :frequency_mhz, :integer + add :flux_sfu, :float + add :reporting_schedule, :string + add :ninety_day_mean, :float + + timestamps(type: :utc_datetime) + end + + create unique_index(:solar_flux_observations, [:valid_time]) + + create table(:solar_xray_observations, primary_key: false) do + add :id, :binary_id, primary_key: true + add :valid_time, :utc_datetime, null: false + add :satellite, :integer + add :flux_wm2, :float + add :energy_band, :string + add :electron_contaminated, :boolean, default: false, null: false + + timestamps(type: :utc_datetime) + end + + create unique_index(:solar_xray_observations, [:valid_time, :energy_band]) + end +end diff --git a/test/fixtures/swpc/f107_sample.json b/test/fixtures/swpc/f107_sample.json new file mode 100644 index 00000000..1d65aac3 --- /dev/null +++ b/test/fixtures/swpc/f107_sample.json @@ -0,0 +1,92 @@ +[ + { + "time_tag": "2026-04-15T17:00:00", + "frequency": 2800, + "flux": 112.0, + "reporting_schedule": "Morning", + "avg_begin_date": null, + "ninety_day_mean": null, + "rec_count": null + }, + { + "time_tag": "2026-04-14T22:00:00", + "frequency": 2800, + "flux": 100.0, + "reporting_schedule": "Afternoon", + "avg_begin_date": null, + "ninety_day_mean": null, + "rec_count": null + }, + { + "time_tag": "2026-04-14T20:00:00", + "frequency": 2800, + "flux": 101.0, + "reporting_schedule": "Noon", + "avg_begin_date": "2026-01-15T20:00:00", + "ninety_day_mean": 135.0, + "rec_count": 90 + }, + { + "time_tag": "2026-04-14T17:00:00", + "frequency": 2800, + "flux": 100.0, + "reporting_schedule": "Morning", + "avg_begin_date": null, + "ninety_day_mean": null, + "rec_count": null + }, + { + "time_tag": "2026-04-13T22:00:00", + "frequency": 2800, + "flux": 100.0, + "reporting_schedule": "Afternoon", + "avg_begin_date": null, + "ninety_day_mean": null, + "rec_count": null + }, + { + "time_tag": "2026-04-13T20:00:00", + "frequency": 2800, + "flux": 99.0, + "reporting_schedule": "Noon", + "avg_begin_date": "2026-01-14T20:00:00", + "ninety_day_mean": 135.0, + "rec_count": 90 + }, + { + "time_tag": "2026-04-13T17:00:00", + "frequency": 2800, + "flux": 99.0, + "reporting_schedule": "Morning", + "avg_begin_date": null, + "ninety_day_mean": null, + "rec_count": null + }, + { + "time_tag": "2026-04-12T22:00:00", + "frequency": 2800, + "flux": 99.0, + "reporting_schedule": "Afternoon", + "avg_begin_date": null, + "ninety_day_mean": null, + "rec_count": null + }, + { + "time_tag": "2026-04-12T20:00:00", + "frequency": 2800, + "flux": 99.0, + "reporting_schedule": "Noon", + "avg_begin_date": "2026-01-13T20:00:00", + "ninety_day_mean": 135.0, + "rec_count": 90 + }, + { + "time_tag": "2026-04-12T17:00:00", + "frequency": 2800, + "flux": 98.0, + "reporting_schedule": "Morning", + "avg_begin_date": null, + "ninety_day_mean": null, + "rec_count": null + } +] diff --git a/test/fixtures/swpc/kp_sample.json b/test/fixtures/swpc/kp_sample.json new file mode 100644 index 00000000..576e2d4f --- /dev/null +++ b/test/fixtures/swpc/kp_sample.json @@ -0,0 +1,62 @@ +[ + { + "time_tag": "2026-04-15T13:49:00", + "kp_index": 0, + "estimated_kp": 0.33, + "kp": "0P" + }, + { + "time_tag": "2026-04-15T13:50:00", + "kp_index": 0, + "estimated_kp": 0.33, + "kp": "0P" + }, + { + "time_tag": "2026-04-15T13:51:00", + "kp_index": 1, + "estimated_kp": 0.67, + "kp": "1M" + }, + { + "time_tag": "2026-04-15T13:52:00", + "kp_index": 1, + "estimated_kp": 0.67, + "kp": "1M" + }, + { + "time_tag": "2026-04-15T13:53:00", + "kp_index": 1, + "estimated_kp": 0.67, + "kp": "1M" + }, + { + "time_tag": "2026-04-15T13:54:00", + "kp_index": 1, + "estimated_kp": 0.67, + "kp": "1M" + }, + { + "time_tag": "2026-04-15T13:55:00", + "kp_index": 1, + "estimated_kp": 0.67, + "kp": "1M" + }, + { + "time_tag": "2026-04-15T13:56:00", + "kp_index": 1, + "estimated_kp": 0.67, + "kp": "1M" + }, + { + "time_tag": "2026-04-15T13:57:00", + "kp_index": 1, + "estimated_kp": 0.67, + "kp": "1M" + }, + { + "time_tag": "2026-04-15T13:58:00", + "kp_index": 1, + "estimated_kp": 0.67, + "kp": "1M" + } +] diff --git a/test/fixtures/swpc/xrays_sample.json b/test/fixtures/swpc/xrays_sample.json new file mode 100644 index 00000000..1621e293 --- /dev/null +++ b/test/fixtures/swpc/xrays_sample.json @@ -0,0 +1,110 @@ +[ + { + "time_tag": "2026-04-14T19:49:00Z", + "satellite": 18, + "flux": 8.99991015046453e-09, + "observed_flux": 5.089760790610853e-08, + "electron_correction": 4.189769953200084e-08, + "electron_contaminaton": true, + "energy": "0.05-0.4nm" + }, + { + "time_tag": "2026-04-14T19:49:00Z", + "satellite": 18, + "flux": 3.523403790950397e-07, + "observed_flux": 3.811346402926574e-07, + "electron_correction": 2.8794250539476707e-08, + "electron_contaminaton": false, + "energy": "0.1-0.8nm" + }, + { + "time_tag": "2026-04-14T19:50:00Z", + "satellite": 18, + "flux": 8.762290448771637e-09, + "observed_flux": 5.0545377661137536e-08, + "electron_correction": 4.178308898872274e-08, + "electron_contaminaton": true, + "energy": "0.05-0.4nm" + }, + { + "time_tag": "2026-04-14T19:50:00Z", + "satellite": 18, + "flux": 3.47408843026642e-07, + "observed_flux": 3.757511706226069e-07, + "electron_correction": 2.8342315161467013e-08, + "electron_contaminaton": false, + "energy": "0.1-0.8nm" + }, + { + "time_tag": "2026-04-14T19:51:00Z", + "satellite": 18, + "flux": 1.0952255102836261e-08, + "observed_flux": 5.269238911864704e-08, + "electron_correction": 4.174013312763236e-08, + "electron_contaminaton": true, + "energy": "0.05-0.4nm" + }, + { + "time_tag": "2026-04-14T19:51:00Z", + "satellite": 18, + "flux": 3.443850573603413e-07, + "observed_flux": 3.7320211276892223e-07, + "electron_correction": 2.8817035868655694e-08, + "electron_contaminaton": false, + "energy": "0.1-0.8nm" + }, + { + "time_tag": "2026-04-14T19:52:00Z", + "satellite": 18, + "flux": 1.1434187818792907e-08, + "observed_flux": 5.2151044371839816e-08, + "electron_correction": 4.071685566486849e-08, + "electron_contaminaton": true, + "energy": "0.05-0.4nm" + }, + { + "time_tag": "2026-04-14T19:52:00Z", + "satellite": 18, + "flux": 3.4263317161276063e-07, + "observed_flux": 3.709357656589418e-07, + "electron_correction": 2.8302592269824345e-08, + "electron_contaminaton": false, + "energy": "0.1-0.8nm" + }, + { + "time_tag": "2026-04-14T19:53:00Z", + "satellite": 18, + "flux": 1.2688251338488499e-08, + "observed_flux": 5.3903328023352515e-08, + "electron_correction": 4.1215077573042436e-08, + "electron_contaminaton": true, + "energy": "0.05-0.4nm" + }, + { + "time_tag": "2026-04-14T19:53:00Z", + "satellite": 18, + "flux": 3.4440770946275734e-07, + "observed_flux": 3.729552133791003e-07, + "electron_correction": 2.854748792913142e-08, + "electron_contaminaton": false, + "energy": "0.1-0.8nm" + }, + { + "time_tag": "2026-04-14T19:54:00Z", + "satellite": 18, + "flux": 1.4324521124819967e-08, + "observed_flux": 5.4740130650543506e-08, + "electron_correction": 4.041560686118828e-08, + "electron_contaminaton": true, + "energy": "0.05-0.4nm" + }, + { + "time_tag": "2026-04-14T19:54:00Z", + "satellite": 18, + "flux": 3.448363941060961e-07, + "observed_flux": 3.734814697509137e-07, + "electron_correction": 2.8645072092103874e-08, + "electron_contaminaton": false, + "energy": "0.1-0.8nm" + } +] diff --git a/test/microwaveprop/space_weather/space_weather_test.exs b/test/microwaveprop/space_weather/space_weather_test.exs new file mode 100644 index 00000000..2faf4fa1 --- /dev/null +++ b/test/microwaveprop/space_weather/space_weather_test.exs @@ -0,0 +1,124 @@ +defmodule Microwaveprop.SpaceWeatherTest do + use Microwaveprop.DataCase, async: true + + alias Microwaveprop.SpaceWeather + alias Microwaveprop.SpaceWeather.GeomagneticObservation + alias Microwaveprop.SpaceWeather.SolarFluxObservation + alias Microwaveprop.SpaceWeather.SolarXrayObservation + + describe "upsert_kp/1" do + test "inserts new rows and returns the count" do + rows = [ + %{valid_time: ~U[2026-04-15 18:00:00Z], kp_index: 2, estimated_kp: 1.67}, + %{valid_time: ~U[2026-04-15 18:01:00Z], kp_index: 2, estimated_kp: 2.0} + ] + + assert {:ok, 2} = SpaceWeather.upsert_kp(rows) + assert Repo.aggregate(GeomagneticObservation, :count, :id) == 2 + end + + test "re-upsert replaces by valid_time (no dupes)" do + row = %{valid_time: ~U[2026-04-15 18:00:00Z], kp_index: 2, estimated_kp: 1.67} + {:ok, 1} = SpaceWeather.upsert_kp([row]) + + updated = %{row | kp_index: 5, estimated_kp: 5.3} + assert {:ok, 1} = SpaceWeather.upsert_kp([updated]) + + assert [stored] = Repo.all(GeomagneticObservation) + assert stored.kp_index == 5 + assert stored.estimated_kp == 5.3 + end + + test "empty list is a no-op returning {:ok, 0}" do + assert {:ok, 0} = SpaceWeather.upsert_kp([]) + end + end + + describe "upsert_solar_flux/1" do + test "inserts rows and replaces by valid_time" do + rows = [ + %{ + valid_time: ~U[2026-04-15 17:00:00Z], + frequency_mhz: 2800, + flux_sfu: 112.0, + reporting_schedule: "Morning", + ninety_day_mean: nil + } + ] + + assert {:ok, 1} = SpaceWeather.upsert_solar_flux(rows) + + updated = rows |> List.first() |> Map.put(:flux_sfu, 120.0) + assert {:ok, 1} = SpaceWeather.upsert_solar_flux([updated]) + + assert [stored] = Repo.all(SolarFluxObservation) + assert stored.flux_sfu == 120.0 + end + end + + describe "upsert_xray/1" do + test "inserts rows and replaces by (valid_time, energy_band)" do + rows = [ + %{ + valid_time: ~U[2026-04-14 19:49:00Z], + satellite: 18, + flux_wm2: 3.5e-7, + energy_band: "0.1-0.8nm", + electron_contaminated: false + } + ] + + assert {:ok, 1} = SpaceWeather.upsert_xray(rows) + assert Repo.aggregate(SolarXrayObservation, :count, :id) == 1 + + # Re-upserting updates existing row + updated = rows |> List.first() |> Map.put(:flux_wm2, 4.0e-7) + assert {:ok, 1} = SpaceWeather.upsert_xray([updated]) + assert Repo.aggregate(SolarXrayObservation, :count, :id) == 1 + + [stored] = Repo.all(SolarXrayObservation) + assert stored.flux_wm2 == 4.0e-7 + end + end + + describe "latest_kp/0" do + test "returns the most recent geomagnetic observation, or nil" do + assert SpaceWeather.latest_kp() == nil + + {:ok, _} = + SpaceWeather.upsert_kp([ + %{valid_time: ~U[2026-04-15 17:00:00Z], kp_index: 2, estimated_kp: 2.0}, + %{valid_time: ~U[2026-04-15 18:00:00Z], kp_index: 3, estimated_kp: 3.0} + ]) + + row = SpaceWeather.latest_kp() + assert row.valid_time == ~U[2026-04-15 18:00:00Z] + assert row.kp_index == 3 + end + end + + describe "latest_f107/0 and latest_xray/0" do + test "return the most recent rows, or nil" do + assert SpaceWeather.latest_f107() == nil + assert SpaceWeather.latest_xray() == nil + + {:ok, _} = + SpaceWeather.upsert_solar_flux([ + %{valid_time: ~U[2026-04-15 17:00:00Z], frequency_mhz: 2800, flux_sfu: 112.0} + ]) + + {:ok, _} = + SpaceWeather.upsert_xray([ + %{ + valid_time: ~U[2026-04-15 17:00:00Z], + satellite: 18, + flux_wm2: 1.0e-7, + energy_band: "0.1-0.8nm" + } + ]) + + assert %SolarFluxObservation{flux_sfu: 112.0} = SpaceWeather.latest_f107() + assert %SolarXrayObservation{flux_wm2: 1.0e-7} = SpaceWeather.latest_xray() + end + end +end diff --git a/test/microwaveprop/space_weather/swpc_client_test.exs b/test/microwaveprop/space_weather/swpc_client_test.exs new file mode 100644 index 00000000..acd0faea --- /dev/null +++ b/test/microwaveprop/space_weather/swpc_client_test.exs @@ -0,0 +1,63 @@ +defmodule Microwaveprop.SpaceWeather.SwpcClientTest do + use ExUnit.Case, async: true + + alias Microwaveprop.SpaceWeather.SwpcClient + + describe "parse_kp/1" do + test "decodes the SWPC planetary_k_index_1m fixture" do + body = File.read!("test/fixtures/swpc/kp_sample.json") + + assert {:ok, rows} = SwpcClient.parse_kp(body) + assert length(rows) == 10 + + first = hd(rows) + assert first.valid_time == ~U[2026-04-15 13:49:00Z] + assert first.kp_index == 0 + assert_in_delta first.estimated_kp, 0.33, 0.001 + end + + test "returns {:error, _} for malformed bodies" do + assert {:error, _} = SwpcClient.parse_kp("not json") + assert {:error, _} = SwpcClient.parse_kp(~s({"not": "an array"})) + end + end + + describe "parse_f107/1" do + test "decodes the SWPC f107_cm_flux fixture" do + body = File.read!("test/fixtures/swpc/f107_sample.json") + + assert {:ok, rows} = SwpcClient.parse_f107(body) + assert length(rows) == 10 + + first = hd(rows) + assert first.valid_time == ~U[2026-04-15 17:00:00Z] + assert first.frequency_mhz == 2800 + assert first.flux_sfu == 112.0 + assert first.reporting_schedule == "Morning" + end + + test "keeps nullable historical stats as nil when absent" do + body = File.read!("test/fixtures/swpc/f107_sample.json") + {:ok, [first | _]} = SwpcClient.parse_f107(body) + assert first.ninety_day_mean == nil + end + end + + describe "parse_xrays/1" do + test "decodes the GOES xrays-1-day fixture, keeping the 0.1-0.8nm band only" do + body = File.read!("test/fixtures/swpc/xrays_sample.json") + + assert {:ok, rows} = SwpcClient.parse_xrays(body) + # Fixture has 12 records (6 timestamps × 2 bands). We keep the + # long-wavelength band (0.1-0.8nm) which is the standard SWF / + # D-layer absorption proxy. + assert length(rows) == 6 + assert Enum.all?(rows, fn r -> r.energy_band == "0.1-0.8nm" end) + + first = hd(rows) + assert first.valid_time == ~U[2026-04-14 19:49:00Z] + assert first.satellite == 18 + assert_in_delta first.flux_wm2, 3.523e-07, 1.0e-09 + end + end +end diff --git a/test/microwaveprop/workers/space_weather_fetch_worker_test.exs b/test/microwaveprop/workers/space_weather_fetch_worker_test.exs new file mode 100644 index 00000000..302127bf --- /dev/null +++ b/test/microwaveprop/workers/space_weather_fetch_worker_test.exs @@ -0,0 +1,55 @@ +defmodule Microwaveprop.Workers.SpaceWeatherFetchWorkerTest do + use Microwaveprop.DataCase, async: false + use Oban.Testing, repo: Microwaveprop.Repo + + alias Microwaveprop.SpaceWeather + alias Microwaveprop.SpaceWeather.GeomagneticObservation + alias Microwaveprop.SpaceWeather.SolarFluxObservation + alias Microwaveprop.SpaceWeather.SolarXrayObservation + alias Microwaveprop.SpaceWeather.SwpcClient + alias Microwaveprop.Workers.SpaceWeatherFetchWorker + + @kp File.read!("test/fixtures/swpc/kp_sample.json") + @f107 File.read!("test/fixtures/swpc/f107_sample.json") + @xrays File.read!("test/fixtures/swpc/xrays_sample.json") + + describe "perform/1" do + test "pulls all three SWPC products and upserts them" do + Req.Test.stub(SwpcClient, fn conn -> + case conn.request_path do + "/json/planetary_k_index_1m.json" -> Plug.Conn.resp(conn, 200, @kp) + "/json/f107_cm_flux.json" -> Plug.Conn.resp(conn, 200, @f107) + "/json/goes/primary/xrays-1-day.json" -> Plug.Conn.resp(conn, 200, @xrays) + end + end) + + assert :ok = perform_job(SpaceWeatherFetchWorker, %{}) + + assert Repo.aggregate(GeomagneticObservation, :count, :id) == 10 + assert Repo.aggregate(SolarFluxObservation, :count, :id) == 10 + # Xray fixture has 12 records (6 timestamps × 2 bands); we only + # store the 0.1-0.8nm band. + assert Repo.aggregate(SolarXrayObservation, :count, :id) == 6 + + assert SpaceWeather.latest_kp() + assert SpaceWeather.latest_f107() + assert SpaceWeather.latest_xray() + end + + test "ingests the remaining products when one endpoint fails" do + Req.Test.stub(SwpcClient, fn conn -> + case conn.request_path do + "/json/planetary_k_index_1m.json" -> Plug.Conn.resp(conn, 500, "boom") + "/json/f107_cm_flux.json" -> Plug.Conn.resp(conn, 200, @f107) + "/json/goes/primary/xrays-1-day.json" -> Plug.Conn.resp(conn, 200, @xrays) + end + end) + + assert :ok = perform_job(SpaceWeatherFetchWorker, %{}) + + assert Repo.aggregate(GeomagneticObservation, :count, :id) == 0 + assert Repo.aggregate(SolarFluxObservation, :count, :id) == 10 + assert Repo.aggregate(SolarXrayObservation, :count, :id) == 6 + end + end +end