diff --git a/config/config.exs b/config/config.exs index 8f10a744..716dfdce 100644 --- a/config/config.exs +++ b/config/config.exs @@ -7,6 +7,8 @@ # General application configuration import Config +alias Microwaveprop.Workers.CanadianSoundingFetchWorker + # Configure esbuild (the version is required) config :esbuild, version: "0.27.4", @@ -75,7 +77,10 @@ config :microwaveprop, Oban, {"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker}, {"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker}, {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker}, - {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker} + {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, + # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch + {"30 1 * * *", CanadianSoundingFetchWorker}, + {"30 13 * * *", CanadianSoundingFetchWorker} ]} ] diff --git a/config/runtime.exs b/config/runtime.exs index 8dcd9aa4..a73d4d76 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -16,6 +16,8 @@ import Config # # Alternatively, you can use `mix phx.gen.release` to generate a `bin/server` # script that automatically sets the env var above. +alias Microwaveprop.Workers.CanadianSoundingFetchWorker + if System.get_env("PHX_SERVER") do config :microwaveprop, MicrowavepropWeb.Endpoint, server: true end @@ -188,7 +190,10 @@ if config_env() == :prod do {"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker}, {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker, - args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}} + args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}}, + # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch + {"30 1 * * *", CanadianSoundingFetchWorker}, + {"30 13 * * *", CanadianSoundingFetchWorker} ]} ] diff --git a/config/test.exs b/config/test.exs index 80c27bc3..c31606bf 100644 --- a/config/test.exs +++ b/config/test.exs @@ -58,6 +58,7 @@ config :microwaveprop, nexrad_req_options: [plug: {Req.Test, Microwaveprop.Weath config :microwaveprop, solar_req_options: [plug: {Req.Test, Microwaveprop.Weather.SolarClient}] config :microwaveprop, srtm_req_options: [plug: {Req.Test, Microwaveprop.Terrain.Srtm}, retry: false] config :microwaveprop, start_freshness_monitor: false +config :microwaveprop, uwyo_req_options: [plug: {Req.Test, Microwaveprop.Weather.UwyoSoundingClient}, retry: false] # Initialize plugs at runtime for faster test compilation config :phoenix, :plug_init_mode, :runtime diff --git a/docs/plans/2026-04-13-hrdps-canadian-prop-grid.md b/docs/plans/2026-04-13-hrdps-canadian-prop-grid.md new file mode 100644 index 00000000..1a6d31ce --- /dev/null +++ b/docs/plans/2026-04-13-hrdps-canadian-prop-grid.md @@ -0,0 +1,311 @@ +# HRDPS Canadian Propagation Grid Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Extend the propagation scoring grid to cover all of Canada at HRDPS's 2.5 km resolution — the Canadian analog to HRRR. This lights up the `/map` for VE stations, gives terrain-diffraction analysis for VE QSOs, and unlocks cross-border path analysis for KP/W7-to-VE7 tropo paths. + +**Architecture:** Build a parallel NWP ingestion pipeline (`HrdpsClient` + `HrdpsGridWorker` + `hrdps_profiles` table) that matches the HRRR structure one-for-one, with three major differences: rotated lat/lon projection, CMC file naming (`CMC_hrdps_continental_*_ps2.5km_*_{FFF}.grib2`), and a different native-level grid spec for duct analysis. A new `propagation_scores` region tag distinguishes HRRR-sourced vs HRDPS-sourced cells so the front-end can render them uniformly. + +**Tech Stack:** Elixir 1.15, Req, Nx (optional for duct analysis), Postgres, Oban, wgrib2, proj (for projection conversions if needed) + +**Prerequisite:** Task 6 of the RDPS plan (`2026-04-13-rdps-vertical-profiles.md`) — or at minimum a working pattern for "score grid_data from a non-HRRR source". If you're starting HRDPS fresh, Task 6 below stands alone but will duplicate some work from the RDPS plan. + +--- + +## Pre-flight research (1–2 hours) + +**Step R.1: Confirm HRDPS datamart URL pattern** + +```bash +curl -s https://dd.weather.gc.ca/model_hrdps/continental/2.5km/ 2>&1 | head -30 +curl -s https://dd.weather.gc.ca/model_hrdps/continental/2.5km/00/000/ 2>&1 | head -30 +``` + +Expected: file list with entries like `CMC_hrdps_continental_TMP_TGL_2_ps2.5km_{YYYYMMDDHH}_P{FFF}-00.grib2`. + +**Step R.2: Download one sample + inspect with wgrib2** + +```bash +wgrib2 CMC_hrdps_continental_TMP_TGL_2_ps2.5km_2026041300_P000-00.grib2 -grid -v +``` + +Expected: confirms grid is rotated lat/lon, gives dimensions (typically ~2500×1800), and lists all variables. + +**Step R.3: Confirm pressure-level files exist** + +HRDPS publishes surface + pressure-level files separately (like HRRR wrfsfc vs wrfprs). Find the pressure-level variant: + +```bash +curl -s https://dd.weather.gc.ca/model_hrdps/continental/2.5km/00/000/ | grep -i 'ISBL' +``` + +If separate files per pressure level, the ingestion must fan out like `HrrrClient.pressure_messages/0`. If a single combined file, one download gets everything. + +**Step R.4: Decide projection strategy** + +wgrib2 can extract points from rotated lat/lon directly with `-lon LON LAT`. For extracting the full grid as a flat array, you need `proj` or a wgrib2 `-new_grid` reprojection. **Recommended**: never reproject — always extract point-by-point for the grid points we care about. Slower than bulk extraction but avoids projection bugs. + +**Step R.5: Confirm run frequency and forecast range** + +HRDPS: 4× daily (00/06/12/18Z), forecasts to 48h. Verify via datamart. This is half HRRR's run frequency (hourly) and 2.7× longer (48h vs 18h). + +**Decision gate:** After R.5, confirm with your human partner which forecast range to ingest. Recommendation: start with f00–f24 to match the "current + 1 day" use case. Extending to f48 doubles storage and compute and probably isn't worth it for amateur radio planning. + +--- + +## Task 1: `HrdpsClient` — surface + pressure fetch + +**Files:** +- Create: `lib/microwaveprop/weather/hrdps_client.ex` +- Test: `test/microwaveprop/weather/hrdps_client_test.exs` +- Fixture: `test/support/fixtures/hrdps_sample_sfc.grib2` + `test/support/fixtures/hrdps_sample_prs.grib2` + +**Step 1.1: Write failing URL builder test** + +```elixir +test "builds correct CMC datamart URL for surface temp" do + url = HrdpsClient.grib_url(~D[2026-04-13], 12, 6, :surface, :tmp) + assert url =~ "/model_hrdps/continental/2.5km/12/006/CMC_hrdps_continental_TMP_TGL_2_ps2.5km_2026041312_P006-00.grib2" +end +``` + +Adjust the format string after pre-flight research confirms the exact naming. + +**Step 1.2–1.4: Write → fail → implement → pass** + +Reuse `HrrrClient.parse_idx/1`, `HrrrClient.byte_ranges_for_messages/2`, and `HrrrClient.download_grib_ranges/2` — they're format-agnostic. Only the URL pattern and the message set differ. + +**Step 1.5: Implement `fetch_grid/3`** + +```elixir +@spec fetch_grid([{float(), float()}], DateTime.t(), keyword()) :: + {:ok, %{{float(), float()} => map()}} | {:error, term()} +def fetch_grid(points, run_time, opts \\ []) +``` + +Same signature as `HrrrClient.fetch_grid/3` so `PropagationGridWorker.process_forecast_hour/4` can call either one polymorphically in Task 6. + +**Step 1.6: Nail the projection** + +Write a focused test: pick a known Canadian airport (CYYR = Goose Bay, 53.32°N, 60.42°W) and verify the HRDPS profile at that point matches the UWYO radiosonde for the same `valid_time` within a reasonable tolerance (surface temp ±3°C, dewpoint ±4°C). If they don't match, the projection is wrong and you need to debug wgrib2's `-lon` extraction vs the CMC rotated grid. + +**Step 1.7: Commit** + +--- + +## Task 2: Native-level duct metrics via `HrdpsNativeClient` + +**Files:** +- Create: `lib/microwaveprop/weather/hrdps_native_client.ex` +- Test: `test/microwaveprop/weather/hrdps_native_client_test.exs` + +**Step 2.1: Confirm HRDPS publishes native hybrid levels** + +HRDPS uses CMC's GEM dynamical core with hybrid sigma-pressure levels (62 of them). The datamart file naming for native levels is `*_HY_*` or within `model_hrdps/continental/2.5km/native/` — confirm in pre-flight. + +**If native levels are NOT published**: this task is impossible; skip it and fall back to pressure-level duct analysis (coarser — 31 levels vs 62 — but still works). The map's ducting detection quality will be slightly worse over VE airspace than over CONUS, which is acceptable for a first pass. + +**If native levels ARE published**: mirror `HrrrNativeClient.fetch_native_duct_grid/4`. Use `Microwaveprop.Propagation.Duct.analyze/1` which is projection-agnostic — it just needs a list of levels with temperature, pressure, and specific humidity. + +**Step 2.2–2.5: TDD cycle as above** + +**Step 2.6: Commit** + +--- + +## Task 3: `hrdps_profiles` table + schema + +**Files:** +- Create: `priv/repo/migrations/{timestamp}_create_hrdps_profiles.exs` +- Create: `lib/microwaveprop/weather/hrdps_profile.ex` +- Test: `test/microwaveprop/weather/hrdps_profile_test.exs` + +**Step 3.1–3.4**: Mirror `hrrr_profiles` exactly. Same columns, same indexes. The only reason to have a separate table is operational isolation — don't contaminate the 10M-row HRRR partition with 8M-row HRDPS rows, and let you drop/reprocess HRDPS without touching HRRR. + +Use daily partitioning on `valid_time` just like `hrrr_profiles`. + +**Step 3.5: Partition-pruning migration** + +Add an `Oban.Plugins.Cron` entry for `HrdpsPartitionCleanup` similar to `HrrrPartitionCleanup` if one exists. + +**Step 3.6: Commit** + +--- + +## Task 4: `HrdpsGridWorker` — hourly compute over Canadian grid + +**Files:** +- Create: `lib/microwaveprop/workers/hrdps_grid_worker.ex` +- Test: `test/microwaveprop/workers/hrdps_grid_worker_test.exs` +- Modify: `lib/microwaveprop/propagation/grid.ex` (add `canadian_points/0`) + +**Step 4.1: Define the Canadian grid** + +At HRDPS 2.5 km resolution, Canada (42N-83N, -141 to -52W) has ~1.5M cells. Too many to score every hour. Instead: +- Sample at 0.125° (14 km) to match HRRR's output grid → ~100K points +- For the subset of points that are INSIDE HRRR's CONUS footprint, prefer HRRR; OUTSIDE, use HRDPS +- Store the "outside HRRR" mask as `Grid.hrdps_only_points/0` + +**Step 4.2: Worker perform/1** + +Mirror `PropagationGridWorker.process_forecast_hour/4` but with `HrdpsClient.fetch_grid/3` and `HrdpsNativeClient.fetch_native_duct_grid/4`. + +**Step 4.3: Scoring** + +Call `Propagation.score_grid_point/4` — it's already projection-agnostic. It takes a profile map and valid_time and produces per-band scores. No changes needed to the scorer. + +**Step 4.4: Upsert into `propagation_scores`** + +`grid_scores` is already indexed by `(lat, lon, valid_time, band_mhz)`. HRDPS and HRRR both write into the same table — the unique key prevents collisions because HRDPS cells have different (lat, lon) than HRRR cells at the shared 0.125° resolution IF you don't re-score HRRR's cells from HRDPS. Use the `hrdps_only_points/0` mask to guarantee no overlap. + +**Step 4.5: TDD cycle, commit** + +--- + +## Task 5: Oban queue + cron + +**Files:** +- Modify: `config/config.exs`, `config/runtime.exs` + +**Step 5.1: Add `hrdps` queue** + +```elixir +queues: [ + # ... existing + hrdps: 1 # HRDPS fetch is sequential like HRRR (memory pressure) +] +``` + +**Step 5.2: Add cron** + +```elixir +# HRDPS runs 00/06/12/18Z. Aligned with RDPS cadence. +{"35 */6 * * *", Microwaveprop.Workers.HrdpsGridWorker} +``` + +Note: 35 past the hour, not 5, because PropagationGridWorker (HRRR) fires at 5 past. Stagger by 30 minutes so they don't compete for the DB and the wgrib2 subprocess slot. + +**Step 5.3: `mix compile` sanity-check** + +**Step 5.4: Commit** + +--- + +## Task 6: Map overlay — extend prop map to cover Canada + +**Files:** +- Modify: `lib/microwaveprop_web/live/map_live.ex` — `@initial_bounds`, timeline +- Modify: `assets/js/propagation_map_hook.ts` — heatmap rendering bounds +- Modify: `lib/microwaveprop/propagation/grid.ex` — export `north_america_points/0` that returns HRRR ∪ HRDPS + +**Step 6.1: Write failing LiveView test** + +```elixir +test "/map renders scores for a Canadian grid cell (53N, -113W)" do + {:ok, view, html} = live(conn, ~p"/map") + # after PropagationGridWorker (HRRR) and HrdpsGridWorker (HRDPS) have both written scores + assert html =~ "53.0" # or whatever assertion proves the HRDPS cell made it to the client +end +``` + +**Step 6.2: Widen `@initial_bounds` in map_live.ex** + +From the CONUS-only `(23.5, 49.5, -124.5, -66.9)` to `(23.5, 83.0, -141.0, -52.0)`. Zoom out default to show all of North America. + +**Step 6.3: Verify front-end renders Canadian points** + +```bash +mix phx.server +# Open http://localhost:4000/map +# Confirm colored cells north of 49N after the first HRDPS run completes +``` + +**Step 6.4: Commit** + +--- + +## Task 7: Terrain + QSO path analysis north of the border + +**Files:** +- Modify: `lib/microwaveprop/terrain/terrain_analysis.ex` (if needed) +- Modify: `lib/microwaveprop/workers/terrain_profile_worker.ex` +- Possibly: mount SRTM data covering Canada onto the production pods + +**Step 7.1: Check SRTM coverage** + +SRTM1 covers 60°S–60°N. Canadian radiosondes and stations above 60°N (Yellowknife, Whitehorse, Inuvik, Resolute) have NO SRTM coverage. The terrain analysis pipeline needs a fallback — likely CDEM (Canadian Digital Elevation Model) or ASTER GDEM. + +**Decision gate with your human partner**: Does the feature need to work above 60°N? If yes, this is a multi-day sub-project (Task 7 becomes a whole new plan). If no — scope to 42–60°N and explicitly document that Inuvik paths won't get terrain analysis. + +**Step 7.2 onward**: Out of scope for this plan. Write a follow-up plan for Arctic terrain if your human partner wants coverage. + +**Step 7.3: Commit the scope decision to `docs/plans/` or a memory file** + +--- + +## Task 8: Freshness monitoring + observability + +**Files:** +- Modify: `lib/microwaveprop/propagation/freshness_monitor.ex` +- Modify: `lib/microwaveprop_web/live/status_live.ex` + +**Step 8.1: Extend FreshnessMonitor to track HRDPS run time** + +Add a `hrdps_last_run_at` field to the monitor state, surface it on `/status`. + +**Step 8.2: Alert when HRDPS run_time is > 4 hours stale** + +HRDPS runs every 6 hours, publishes ~90 min after, so a healthy pipeline never exceeds ~2.5 hours stale. 4 hours = something broken. + +**Step 8.3: Commit** + +--- + +## Task 9: Precommit, docs, PR + +**Step 9.1: `mix precommit`** + +Must show 0 failures. + +**Step 9.2: Update `CLAUDE.md`** + +- Add HRDPS to `### Key Data Sources` +- Add `HrdpsGridWorker` + `HrdpsFetchWorker` to the background queue table +- Add a note on the CONUS → North America scope change + +**Step 9.3: Update `memory/reference_hrdps_canada.md`** + +Mark the plan as executed — include the final ingestion rate, queue config, and any projection gotchas that surfaced. + +**Step 9.4: Commit, push to `github` and `origin`, let Flux deploy** + +Do NOT push to `dokku`. + +--- + +## Known risks + gotchas + +- **Projection**: HRDPS rotated lat/lon is the #1 source of bugs. Burn a day in Task 1.6 validating a dozen points against UWYO or ASOS surface obs before moving on. A 10 km projection offset is invisible on the propagation map but destroys terrain-path analysis. +- **Memory pressure**: The existing HRRR worker hits `:system_memory_high_watermark` alarms at ~4 GB. HRDPS grid is 3× larger in cell count. Plan for 8 GB per pod for the prop namespace, or offload HRDPS fetch to a dedicated node via Kubernetes node affinity. +- **Publish time**: CMC's publish delay can be erratic — 90 min on good days, 3+ hours under load. Build retry-with-backoff into `HrdpsGridWorker.perform/1` so the cron doesn't fire, find no data, and dead-letter immediately. +- **Arctic terrain**: SRTM stops at 60°N. Accept or fix — don't silently serve wrong diffraction losses to stations in Yellowknife. +- **HRRR/HRDPS overlap**: The two models disagree on the US/Canadian border. Pick a rule (prefer HRRR south of 49°N, prefer HRDPS north) and encode it in `Grid.point_source/2`. +- **Scope creep**: This plan is already ~5 days of work. Resist adding "while we're at it, let's also fix the native-level ducting for rotated grids" side-quests. Ship the minimum viable Canadian prop grid first. + +## Rough effort estimate + +| Task | Estimate | +|---|---| +| Pre-flight research | 2 hours | +| Task 1: HrdpsClient | 1 day (projection debug dominates) | +| Task 2: HrdpsNativeClient | 0.5 day (or skipped if no native) | +| Task 3: hrdps_profiles schema | 2 hours | +| Task 4: HrdpsGridWorker | 1 day | +| Task 5: Cron wiring | 1 hour | +| Task 6: Map overlay | 0.5 day | +| Task 7: Terrain coverage decision | 2 hours (punt if Arctic gets out-of-scope) | +| Task 8: Freshness monitoring | 2 hours | +| Task 9: Precommit + docs | 2 hours | +| **Total** | **~5 days** | + +Compare to RDPS plan (~2 days) and the UWYO Canadian soundings (~4 hours, already done). Do HRDPS only when the RDPS 10 km coverage feels limiting, not before. diff --git a/docs/plans/2026-04-13-rdps-vertical-profiles.md b/docs/plans/2026-04-13-rdps-vertical-profiles.md new file mode 100644 index 00000000..2003b71d --- /dev/null +++ b/docs/plans/2026-04-13-rdps-vertical-profiles.md @@ -0,0 +1,307 @@ +# RDPS Vertical Profiles (Canadian Ducting Outside HRRR Footprint) Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Extend the refractivity/ducting analysis beyond HRRR's Lambert CONUS footprint by ingesting ECCC's Regional Deterministic Prediction System (RDPS) vertical profiles. This gives us forecast ducting across all of Canada + northern-tier US gaps without the full HRDPS ingestion complexity. + +**Architecture:** RDPS publishes a dedicated "vertical profiles" subset at 10 km resolution, 84h forecast, 4×/day. The files contain pressure-level T/Td/height — exactly the shape `SoundingParams.derive/1` + `Propagation.Duct.analyze/1` already consume. Fetch one forecast cycle per run, decode, store as time-indexed profile rows in a new `rdps_profiles` table (parallel to `hrrr_profiles` but on a 10 km grid), and surface it through a new `/weather-ca` map or extend the existing weather map timeline to "fall back to RDPS outside the HRRR footprint". + +**Tech Stack:** Elixir 1.15, Req (HTTP), Nx optional, Postgres, Oban, wgrib2 (already installed for HRRR native duct grid) + +--- + +## Pre-flight: URL and format research + +Before writing any code, confirm the RDPS vertical-profiles format. MSC docs are thin; the cheapest way is to hit the datamart directly. + +**Files to touch:** None — this is read-only research. + +**Step R.1: Find today's run directory** + +Run: + +```bash +curl -s https://dd.weather.gc.ca/model_gem_regional/10km/grib2/00/000/ | grep -oE 'href="[^"]+\.grib2"' | head -20 +``` + +Expected: file list including a `*vertical*profile*.grib2` or similar naming. + +**Step R.2: Confirm vertical-profiles subdirectory** + +Actual RDPS directory on dd.weather.gc.ca is `/model_gem_regional/10km/grib2/{HH}/{FFF}/` — where HH is 00/06/12/18 and FFF is forecast hour. The vertical-profile subset may live under a different path; check: + +```bash +curl -s https://dd.weather.gc.ca/ensemble/rdps/grib2/ 2>/dev/null +curl -s https://dd.weather.gc.ca/20260413/WXO-DD/model_gem_regional/ 2>/dev/null | grep -oE 'href="[^"]+/?' +``` + +If no dedicated vertical-profiles subset exists, fall back to pulling TMP/DEPR/HGT at all 31 pressure levels of the regular RDPS pressure-level files (same approach `HrrrClient.fetch_grid/3` uses for HRRR). + +**Step R.3: Download one sample file and inspect with wgrib2** + +```bash +wgrib2 /tmp/sample.grib2 -var -v | head -30 +``` + +Confirm it contains TMP, DEPR (or SPFH), HGT at 1000, 925, 850, 700, 500, 400, 300, 250, 200 mb (minimum). + +**Decision after R.3:** If RDPS ships a ready-made vertical-profiles product with structured per-cell levels, continue with Task 1. If it's just the full pressure-level grid (like HRRR), this plan is still viable but the first task becomes "adapt HrrrClient pressure-level extraction to RDPS rotated lat/lon projection" — still simpler than full HRDPS. + +--- + +## Task 1: New `RdpsClient` module for GRIB2 fetch + decode + +**Files:** +- Create: `lib/microwaveprop/weather/rdps_client.ex` +- Test: `test/microwaveprop/weather/rdps_client_test.exs` +- Fixture: `test/support/fixtures/rdps_sample.grib2` (capture one real ~50 MB file via curl, commit it) + +**Step 1.1: Write failing test for URL builder** + +```elixir +test "builds the correct datamart URL for a run/forecast hour/pressure level" do + url = RdpsClient.grib_url(~D[2026-04-13], 12, 6, :tmp, 850) + assert url =~ "/20260413/WXO-DD/model_gem_regional/10km/grib2/12/006/" + assert url =~ "TMP_ISBL_850" +end +``` + +**Step 1.2: Verify test fails** + +``` +mix test test/microwaveprop/weather/rdps_client_test.exs -v +``` + +Expected: FAIL with UndefinedFunctionError. + +**Step 1.3: Implement `grib_url/5`** + +Match the actual dd.weather.gc.ca pattern discovered in pre-flight research. Copy the index-file + byte-range download pattern from `HrrrClient.fetch_product/4` — RDPS GRIB2 files also ship `.idx` sidecars. + +**Step 1.4: Run, confirm PASS** + +**Step 1.5: Write failing test for `fetch_cell_profile/3`** + +Given a lat/lon and run time, return `%{"pres", "hght", "tmpc", "dwpc"}` maps for all standard pressure levels, matching the shape `SoundingParams.derive/1` expects. Use the committed fixture GRIB2 via `Req.Test` stub or directly from disk. + +**Step 1.6: Implement fetch_cell_profile/3 using wgrib2 subprocess** + +Reuse `Microwaveprop.Weather.Grib2.Wgrib2.extract_points/3` — the module already wraps wgrib2 for HRRR extraction and supports arbitrary GRIB2 files. + +**Key nuance — rotated lat/lon projection:** RDPS uses rotated_latitude_longitude, not Lambert conformal. wgrib2 handles this internally for `-lon`, `-match ":(TMP|DEPR|HGT):(1000 mb|925 mb|850 mb|700 mb|500 mb|400 mb|300 mb|250 mb|200 mb)"` extraction — verify by comparing a known point against UWYO sounding for Winnipeg/YWG. + +**Step 1.7: Commit** + +```bash +git add lib/microwaveprop/weather/rdps_client.ex test/microwaveprop/weather/rdps_client_test.exs test/support/fixtures/rdps_sample.grib2 +git commit -m "Add RdpsClient for Canadian pressure-level weather fetch" +``` + +--- + +## Task 2: `rdps_profiles` schema + migration + +**Files:** +- Create: `priv/repo/migrations/{timestamp}_create_rdps_profiles.exs` (use `mix ecto.gen.migration create_rdps_profiles`) +- Create: `lib/microwaveprop/weather/rdps_profile.ex` +- Test: `test/microwaveprop/weather/rdps_profile_test.exs` + +**Step 2.1: Write failing changeset test** + +```elixir +test "requires valid_time, lat, lon, and profile" do + changeset = RdpsProfile.changeset(%RdpsProfile{}, %{}) + refute changeset.valid? + assert %{valid_time: [_], lat: [_], lon: [_], profile: [_]} = errors_on(changeset) +end +``` + +**Step 2.2: Generate migration** + +```bash +mix ecto.gen.migration create_rdps_profiles +``` + +Columns (mirror `hrrr_profiles` minus the native-duct fields RDPS doesn't provide): +- `id` binary_id +- `valid_time` utc_datetime +- `run_time` utc_datetime +- `lat` float +- `lon` float +- `profile` jsonb (array of pressure-level maps) +- `surface_pressure_mb`, `surface_temp_c`, `surface_dewpoint_c` floats +- `surface_refractivity`, `min_refractivity_gradient` floats +- `ducting_detected` boolean, `duct_characteristics` jsonb +- `pwat_mm`, `hpbl_m` floats (if RDPS publishes them; otherwise null) +- unique index on `(lat, lon, valid_time)` +- timestamps + +**Step 2.3: Implement schema module** + +Copy shape from `lib/microwaveprop/weather/hrrr_profile.ex`, strip anything RDPS doesn't supply. + +**Step 2.4: Run migration against dev DB and verify** + +```bash +mix ecto.migrate +psql -h localhost -d microwaveprop_dev -c "\d rdps_profiles" +``` + +**Step 2.5: Run schema test, verify PASS** + +**Step 2.6: Commit** + +--- + +## Task 3: `RdpsFetchWorker` — one cell per job, queue-limited + +**Files:** +- Create: `lib/microwaveprop/workers/rdps_fetch_worker.ex` +- Test: `test/microwaveprop/workers/rdps_fetch_worker_test.exs` + +**Step 3.1: Write failing test** + +Worker takes `%{"lat" => lat, "lon" => lon, "valid_time" => iso}`, stubs `RdpsClient.fetch_cell_profile/3`, asserts an `RdpsProfile` row is upserted with derived SoundingParams fields filled in. + +**Step 3.2: Implement worker** + +- `use Oban.Worker, queue: :rdps, max_attempts: 3, unique: ...` +- `perform/1` calls `RdpsClient.fetch_cell_profile`, runs `SoundingParams.derive/1` on the returned profile, upserts via `Weather.upsert_rdps_profile/1` (add this function). + +**Step 3.3: Verify test passes** + +**Step 3.4: Commit** + +--- + +## Task 4: `RdpsGridWorker` — hourly-ish batch driver + +**Files:** +- Create: `lib/microwaveprop/workers/rdps_grid_worker.ex` +- Test: `test/microwaveprop/workers/rdps_grid_worker_test.exs` + +**Step 4.1: Write failing test** + +Test that `perform/1` enqueues one `RdpsFetchWorker` per grid point in a Canadian bounding box `(south, north, west, east) = (42, 83, -141, -52)` sampled at 0.5° (≈1500 points). + +**Step 4.2: Implement** + +Mirror `PropagationGridWorker.perform/1` — compute run_time (2 hours ago nearest RDPS synoptic hour 00/06/12/18), loop over Canadian grid, enqueue. Don't couple to HRRR; RDPS runs on a different cadence (every 6 hours vs HRRR every hour) and this keeps the two pipelines independent. + +**Step 4.3: Verify** + +**Step 4.4: Commit** + +--- + +## Task 5: Oban cron wiring + queue limits + +**Files:** +- Modify: `config/config.exs` (dev + shared) +- Modify: `config/runtime.exs` (prod Oban config) + +**Step 5.1: Add `rdps` queue** + +```elixir +queues: [ + # ... existing + rdps: 2 +] +``` + +Dev: 2. Prod: 2 per pod × 3 pods = 6 workers, plus a global_limit if MSC starts rate-limiting. + +**Step 5.2: Add cron** + +```elixir +# RDPS runs 00/06/12/18Z. Data typically publishes ~90 minutes after synoptic. +{"30 */6 * * *", Microwaveprop.Workers.RdpsGridWorker} +``` + +**Step 5.3: `mix compile` sanity-check** + +**Step 5.4: Commit** + +--- + +## Task 6: Score integration — fall back to RDPS outside HRRR footprint + +**Files:** +- Modify: `lib/microwaveprop/propagation.ex:~244` (`scores_at/3`) +- Modify: `lib/microwaveprop/workers/propagation_grid_worker.ex:~74` (`process_forecast_hour/4`) +- Test: `test/microwaveprop/propagation/rdps_fallback_test.exs` + +**Step 6.1: Write failing test** + +For a lat/lon inside HRRR's CONUS footprint, `Propagation.score_grid_point/4` uses the HRRR profile. For a lat/lon in northern Canada (65N, -90W), it falls back to the `RdpsProfile` for the same valid_time. + +**Step 6.2: Add `rdps_profile_at/3` helper in `Weather`** + +```elixir +def rdps_profile_at(lat, lon, valid_time) do + # nearest-neighbor lookup in rdps_profiles +end +``` + +**Step 6.3: Modify `process_forecast_hour/4` to also score RDPS points** + +Two-pass: first the HRRR CONUS grid as today, then a second pass over RDPS cells that fall outside HRRR's Lambert domain. Extend `grid_data` to include both sources, mark each point with `source: :hrrr` or `source: :rdps` in factors. + +**Step 6.4: Verify tests + precommit** + +**Step 6.5: Commit** + +--- + +## Task 7: Map overlay — extend CONUS prop map north + +**Files:** +- Modify: `lib/microwaveprop/propagation/grid.ex` — `conus_points/0` → `north_america_points/0` or similar +- Modify: `assets/js/propagation_map_hook.ts` — extend bounds, label RDPS-sourced cells + +**Step 7.1: Decide UX** + +Pick one: +- **Option A**: Silent extension — the existing /map just shows scores north of 49N without visual distinction. +- **Option B**: Dashed line on the map at the HRRR/RDPS boundary, tooltip notes data source. + +Default to Option A unless your human partner prefers B. + +**Step 7.2: Extend grid bounds** + +Change the `conus_points/0` generator to include cells in `(42, 83, -141, -52)` for RDPS. Keep CONUS cells on the current 0.125° grid, add RDPS cells at 0.5° (coarser). + +**Step 7.3: Verify `/map` renders Canadian cells** + +Manual: `mix phx.server`, open http://localhost:4000/map, zoom to Ontario/Quebec, confirm colored cells appear. + +**Step 7.4: Commit** + +--- + +## Task 8: Precommit + PR + +**Step 8.1: `mix precommit`** + +Must show 0 failures. + +**Step 8.2: Update `CLAUDE.md` with the new pipeline** + +Add an `RdpsFetchWorker`/`RdpsGridWorker` row to the Background Job Queues table and a bullet under `## Key Data Sources` for RDPS. + +**Step 8.3: Update `memory/reference_hrdps_canada.md`** + +Add a note: "RDPS vertical profiles (10 km) ingestion landed YYYY-MM-DD — gives Canadian ducting without the full HRDPS rewrite." + +**Step 8.4: Commit and push to both `github` and `origin` remotes** + +Do NOT push to `dokku`. Let Flux roll out the new image from the CI build. + +--- + +## Notes / risks + +- **Rotated lat/lon projection**: wgrib2 handles it correctly for point extraction but caution needed if the team ever decides to store full-grid binary rasters. +- **File size**: RDPS pressure-level files are ~50 MB each. At 4 runs/day × ~10 forecast hours × ~5 variables × ~10 pressure levels, total download ≈ 2 GB/day. Not a problem on a home lab with skippy.w5isp.com cache, but worth caching aggressively. +- **Data latency**: RDPS publishes ~90 minutes after synoptic; the cron is already aligned. +- **Fallback when RDPS is unavailable**: `scores_at/3` should gracefully degrade to HRRR-only if `rdps_profiles` is empty for the requested `valid_time`. diff --git a/lib/microwaveprop/weather/uwyo_sounding_client.ex b/lib/microwaveprop/weather/uwyo_sounding_client.ex new file mode 100644 index 00000000..0072d3f6 --- /dev/null +++ b/lib/microwaveprop/weather/uwyo_sounding_client.ex @@ -0,0 +1,158 @@ +defmodule Microwaveprop.Weather.UwyoSoundingClient do + @moduledoc """ + Client for University of Wyoming atmospheric sounding archive + (`weather.uwyo.edu`), which serves near-real-time radiosonde data for + the global WMO network — including Canadian stations that the IEM RAOB + endpoint no longer keeps current. + + Produces the same shape as `Microwaveprop.Weather.IemClient.parse_raob_json/1`: + `{:ok, [%{observed_at: DateTime.t(), profile: [map()]}]}`, where each + profile level is `%{"pres", "hght", "tmpc", "dwpc", "drct", "sknt"}` with + string keys so `Microwaveprop.Weather.SoundingParams.derive/1` can consume + it directly. + + Station codes are UWYO's 3-letter form — for Canadian ICAO codes (CWSE, + CYYR, ...) the caller must strip the leading `C`. + """ + + @base "https://weather.uwyo.edu/cgi-bin/sounding" + + @type profile_level :: %{String.t() => float() | nil} + @type sounding :: %{observed_at: DateTime.t(), profile: [profile_level()]} + + @spec sounding_url(String.t(), DateTime.t()) :: String.t() + def sounding_url(station_code, %DateTime{} = dt) do + month = dt.month |> Integer.to_string() |> String.pad_leading(2, "0") + day = dt.day |> Integer.to_string() |> String.pad_leading(2, "0") + hour = dt.hour |> Integer.to_string() |> String.pad_leading(2, "0") + dayhour = day <> hour + + "#{@base}?region=naconf&TYPE=TEXT%3ALIST" <> + "&YEAR=#{dt.year}&MONTH=#{month}&FROM=#{dayhour}&TO=#{dayhour}&STNM=#{station_code}" + end + + @spec fetch_sounding(String.t(), DateTime.t()) :: {:ok, [sounding()]} | {:error, term()} + def fetch_sounding(station_code, %DateTime{} = dt) do + url = sounding_url(station_code, dt) + + case Req.get(url, req_options()) do + {:ok, %{status: 200, body: body}} -> {:ok, parse_sounding_html(body)} + {:ok, %{status: status}} -> {:error, "UWYO sounding HTTP #{status}"} + {:error, reason} -> {:error, reason} + end + end + + @spec parse_sounding_html(String.t()) :: [sounding()] + def parse_sounding_html(html) when is_binary(html) do + with {:ok, observed_at} <- parse_h2_header(html), + {:ok, pre_text} <- extract_pre_block(html) do + profile = + pre_text + |> String.split("\n") + |> Enum.flat_map(&parse_profile_row/1) + + [%{observed_at: observed_at, profile: profile}] + else + _ -> [] + end + end + + # ---------- Private ---------- + + defp req_options do + defaults = [retry: &retry?/2, max_retries: 3, retry_delay: &retry_delay/1] + overrides = Application.get_env(:microwaveprop, :uwyo_req_options, []) + Keyword.merge(defaults, overrides) + end + + defp retry?(_request, response) do + case response do + %Req.Response{status: status} when status in [429, 500, 502, 503, 504] -> true + %{__exception__: true} -> true + _ -> false + end + end + + defp retry_delay(n) do + base = Integer.pow(2, n) * 1_000 + jitter = :rand.uniform(1_000) + base + jitter + end + + # Header line example: + #
(.*?)<\/PRE>/s, html) do
+ [_, body] -> {:ok, body}
+ _ -> :error
+ end
+ end
+
+ # UWYO rows are fixed-width, 7 chars per column. We care about the first 8
+ # columns: PRES, HGHT, TEMP, DWPT, RELH, MIXR, DRCT, SKNT. Any line without
+ # both a numeric pressure AND a numeric temperature is a header/separator
+ # line or an incomplete observation and is skipped.
+ defp parse_profile_row(line) when is_binary(line) do
+ if String.length(line) < 28 do
+ []
+ else
+ pres = parse_column(line, 0)
+ hght = parse_column(line, 7)
+ tmpc = parse_column(line, 14)
+ dwpc = parse_column(line, 21)
+ drct = parse_column(line, 42)
+ sknt = parse_column(line, 49)
+
+ if is_number(pres) and is_number(tmpc) do
+ [%{"pres" => pres, "hght" => hght, "tmpc" => tmpc, "dwpc" => dwpc, "drct" => drct, "sknt" => sknt}]
+ else
+ []
+ end
+ end
+ end
+
+ defp parse_column(line, offset) do
+ field = line |> String.slice(offset, 7) |> String.trim()
+
+ if field == "" do
+ nil
+ else
+ case Float.parse(field) do
+ {val, ""} -> val
+ _ -> nil
+ end
+ end
+ end
+
+ defp month_num("Jan"), do: {:ok, 1}
+ defp month_num("Feb"), do: {:ok, 2}
+ defp month_num("Mar"), do: {:ok, 3}
+ defp month_num("Apr"), do: {:ok, 4}
+ defp month_num("May"), do: {:ok, 5}
+ defp month_num("Jun"), do: {:ok, 6}
+ defp month_num("Jul"), do: {:ok, 7}
+ defp month_num("Aug"), do: {:ok, 8}
+ defp month_num("Sep"), do: {:ok, 9}
+ defp month_num("Oct"), do: {:ok, 10}
+ defp month_num("Nov"), do: {:ok, 11}
+ defp month_num("Dec"), do: {:ok, 12}
+ defp month_num(_), do: :error
+end
diff --git a/lib/microwaveprop/workers/canadian_sounding_fetch_worker.ex b/lib/microwaveprop/workers/canadian_sounding_fetch_worker.ex
new file mode 100644
index 00000000..a4740c3d
--- /dev/null
+++ b/lib/microwaveprop/workers/canadian_sounding_fetch_worker.ex
@@ -0,0 +1,129 @@
+defmodule Microwaveprop.Workers.CanadianSoundingFetchWorker do
+ @moduledoc """
+ Twice-daily batch fetcher for Canadian radiosonde soundings via the
+ University of Wyoming archive. IEM's RAOB endpoint has not been keeping
+ Canadian stations current (most stalled Sep 2024), so this worker fills
+ the gap — crucial because soundings drive the refractivity / ducting
+ calibration in `SoundingParams.derive/1`.
+
+ Finds every `weather_stations` row of type `sounding` whose `station_code`
+ starts with `C` (Canadian ICAO prefix), drops the leading `C` to get the
+ UWYO 3-letter code, fetches the latest available 00Z or 12Z sounding,
+ and upserts it into `soundings`.
+ """
+
+ use Oban.Worker,
+ queue: :weather,
+ max_attempts: 3,
+ unique: [period: 3600, states: [:available, :scheduled, :executing, :retryable]]
+
+ import Ecto.Query
+
+ alias Microwaveprop.Repo
+ alias Microwaveprop.Weather
+ alias Microwaveprop.Weather.SoundingParams
+ alias Microwaveprop.Weather.Station
+ alias Microwaveprop.Weather.UwyoSoundingClient
+
+ require Logger
+
+ # UWYO publishes ~90 minutes after the synoptic hour
+ @publish_delay_seconds 5_400
+
+ @impl Oban.Worker
+ def perform(%Oban.Job{args: args}) do
+ sounding_time =
+ case args do
+ %{"sounding_time" => iso} when is_binary(iso) ->
+ {:ok, dt, _} = DateTime.from_iso8601(iso)
+ DateTime.truncate(dt, :second)
+
+ _ ->
+ most_recent_sounding_time(DateTime.utc_now())
+ end
+
+ stations = canadian_sounding_stations()
+
+ Logger.info("CanadianSoundings: fetching #{length(stations)} stations for #{DateTime.to_iso8601(sounding_time)}")
+
+ Enum.each(stations, &fetch_station(&1, sounding_time))
+
+ :ok
+ end
+
+ @doc """
+ Return the most recently-publishable 00Z or 12Z sounding time, given
+ the current UTC instant. UWYO needs ~90 minutes after launch before the
+ decoded profile lands on the site, so we step back from synoptic hour
+ until we find one that's at least 90 minutes old.
+ """
+ @spec most_recent_sounding_time(DateTime.t()) :: DateTime.t()
+ def most_recent_sounding_time(%DateTime{} = now) do
+ cutoff = DateTime.add(now, -@publish_delay_seconds, :second)
+
+ cond do
+ cutoff.hour >= 12 ->
+ %{cutoff | hour: 12, minute: 0, second: 0, microsecond: {0, 0}}
+
+ cutoff.hour >= 0 and DateTime.compare(cutoff, %{cutoff | hour: 0, minute: 0, second: 0, microsecond: {0, 0}}) != :lt ->
+ %{cutoff | hour: 0, minute: 0, second: 0, microsecond: {0, 0}}
+ end
+ end
+
+ # ---------- Internal ----------
+
+ defp canadian_sounding_stations do
+ Repo.all(from(s in Station, where: s.station_type == "sounding" and ilike(s.station_code, "c%")))
+ end
+
+ defp fetch_station(%Station{station_code: code} = station, sounding_time) do
+ uwyo_code = String.slice(code, 1..-1//1)
+
+ if Weather.has_sounding?(station.id, sounding_time) do
+ Logger.debug("CanadianSoundings: #{code} already has sounding for #{sounding_time}")
+ :ok
+ else
+ do_fetch(station, code, uwyo_code, sounding_time)
+ end
+ end
+
+ defp do_fetch(station, code, uwyo_code, sounding_time) do
+ case UwyoSoundingClient.fetch_sounding(uwyo_code, sounding_time) do
+ {:ok, [%{profile: profile} | _]} when profile != [] ->
+ attrs = build_attrs(sounding_time, profile)
+ Weather.upsert_sounding(station, attrs)
+
+ Logger.info("CanadianSoundings: #{code} ingested sounding with #{length(profile)} levels")
+
+ {:ok, _} ->
+ Logger.info("CanadianSoundings: #{code} no data for #{sounding_time}")
+
+ {:error, reason} ->
+ Logger.warning("CanadianSoundings: #{code} failed: #{inspect(reason)}")
+ end
+ end
+
+ defp build_attrs(sounding_time, profile) do
+ base = %{observed_at: sounding_time, profile: profile, level_count: length(profile)}
+
+ case SoundingParams.derive(profile) do
+ nil ->
+ base
+
+ params ->
+ Map.merge(base, %{
+ surface_pressure_mb: params.surface_pressure_mb,
+ surface_temp_c: params.surface_temp_c,
+ surface_dewpoint_c: params.surface_dewpoint_c,
+ surface_refractivity: params.surface_refractivity,
+ min_refractivity_gradient: params.min_refractivity_gradient,
+ boundary_layer_depth_m: params.boundary_layer_depth_m,
+ precipitable_water_mm: params.precipitable_water_mm,
+ k_index: params.k_index,
+ lifted_index: params.lifted_index,
+ ducting_detected: params.ducting_detected,
+ duct_characteristics: params.duct_characteristics
+ })
+ end
+ end
+end
diff --git a/test/microwaveprop/weather/uwyo_sounding_client_test.exs b/test/microwaveprop/weather/uwyo_sounding_client_test.exs
new file mode 100644
index 00000000..a8e6626d
--- /dev/null
+++ b/test/microwaveprop/weather/uwyo_sounding_client_test.exs
@@ -0,0 +1,122 @@
+defmodule Microwaveprop.Weather.UwyoSoundingClientTest do
+ use ExUnit.Case, async: true
+
+ alias Microwaveprop.Weather.UwyoSoundingClient
+
+ @fixture File.read!("test/support/fixtures/uwyo_sounding_yyr.html")
+
+ describe "sounding_url/2" do
+ test "builds the correct URL for a 3-letter station code + datetime" do
+ url = UwyoSoundingClient.sounding_url("YYR", ~U[2026-04-13 00:00:00Z])
+
+ assert url ==
+ "https://weather.uwyo.edu/cgi-bin/sounding" <>
+ "?region=naconf&TYPE=TEXT%3ALIST" <>
+ "&YEAR=2026&MONTH=04&FROM=1300&TO=1300&STNM=YYR"
+ end
+
+ test "pads month and day-hour to two digits" do
+ url = UwyoSoundingClient.sounding_url("WSE", ~U[2026-01-05 12:00:00Z])
+
+ assert url =~ "MONTH=01"
+ assert url =~ "FROM=0512"
+ assert url =~ "TO=0512"
+ assert url =~ "STNM=WSE"
+ end
+ end
+
+ describe "parse_sounding_html/1" do
+ test "returns one sounding entry for a valid response" do
+ assert [sounding] = UwyoSoundingClient.parse_sounding_html(@fixture)
+ assert sounding.observed_at == ~U[2026-04-13 00:00:00Z]
+ assert is_list(sounding.profile)
+ assert length(sounding.profile) > 20
+ end
+
+ test "parses the surface level with all fields populated" do
+ [sounding] = UwyoSoundingClient.parse_sounding_html(@fixture)
+ surface = hd(sounding.profile)
+
+ assert surface["pres"] == 1016.0
+ assert surface["hght"] == 36.0
+ assert surface["tmpc"] == -3.5
+ assert surface["dwpc"] == -9.5
+ assert surface["drct"] == 145.0
+ assert surface["sknt"] == 1.0
+ end
+
+ test "each level has the same key set (pres, hght, tmpc, dwpc, drct, sknt)" do
+ [sounding] = UwyoSoundingClient.parse_sounding_html(@fixture)
+
+ Enum.each(sounding.profile, fn level ->
+ assert Map.has_key?(level, "pres")
+ assert Map.has_key?(level, "hght")
+ assert Map.has_key?(level, "tmpc")
+ assert Map.has_key?(level, "dwpc")
+ assert Map.has_key?(level, "drct")
+ assert Map.has_key?(level, "sknt")
+ end)
+ end
+
+ test "levels with missing temp/dewpoint are omitted" do
+ # Lines like ' 1000.0 92' (only PRES + HGHT, rest blank) must not
+ # appear in the parsed profile because SoundingParams.derive/1 needs temp.
+ html = """
+ 71119 WSE Edmonton Stony Plain Observations at 00Z 13 Apr 2026
+
+ -----------------------------------------------------------------------------
+ PRES HGHT TEMP DWPT RELH MIXR DRCT SKNT THTA THTE THTV
+ hPa m C C % g/kg deg knot K K K
+ -----------------------------------------------------------------------------
+ 1000.0 92
+ 920.0 766 2.2 -3.8 65 3.15 85 6 282.0 291.1 282.5
+
+ """
+
+ [sounding] = UwyoSoundingClient.parse_sounding_html(html)
+ assert length(sounding.profile) == 1
+ assert hd(sounding.profile)["pres"] == 920.0
+ end
+
+ test "returns an empty list when the response contains no PRE block" do
+ html = "Sorry, the server is too busy to process your request.\n"
+ assert UwyoSoundingClient.parse_sounding_html(html) == []
+ end
+
+ test "returns an empty list when the header is missing" do
+ html = " 1000.0 30 25.0 20.0 50 0.0 180 10 0 0 0
"
+ assert UwyoSoundingClient.parse_sounding_html(html) == []
+ end
+ end
+
+ describe "fetch_sounding/2" do
+ test "returns parsed sounding on a 200 response" do
+ Req.Test.stub(UwyoSoundingClient, fn conn ->
+ Req.Test.text(conn, @fixture)
+ end)
+
+ assert {:ok, [sounding]} =
+ UwyoSoundingClient.fetch_sounding("YYR", ~U[2026-04-13 00:00:00Z])
+
+ assert sounding.observed_at == ~U[2026-04-13 00:00:00Z]
+ refute sounding.profile == []
+ end
+
+ test "returns an empty list when the server is busy" do
+ Req.Test.stub(UwyoSoundingClient, fn conn ->
+ Req.Test.text(conn, "Sorry, the server is too busy to process your request.\n")
+ end)
+
+ assert {:ok, []} = UwyoSoundingClient.fetch_sounding("YYR", ~U[2026-04-13 00:00:00Z])
+ end
+
+ test "returns an error on a non-200 response" do
+ Req.Test.stub(UwyoSoundingClient, fn conn ->
+ Plug.Conn.send_resp(conn, 503, "Service Unavailable")
+ end)
+
+ assert {:error, "UWYO sounding HTTP 503"} =
+ UwyoSoundingClient.fetch_sounding("YYR", ~U[2026-04-13 00:00:00Z])
+ end
+ end
+end
diff --git a/test/microwaveprop/workers/canadian_sounding_fetch_worker_test.exs b/test/microwaveprop/workers/canadian_sounding_fetch_worker_test.exs
new file mode 100644
index 00000000..2a3630a7
--- /dev/null
+++ b/test/microwaveprop/workers/canadian_sounding_fetch_worker_test.exs
@@ -0,0 +1,130 @@
+defmodule Microwaveprop.Workers.CanadianSoundingFetchWorkerTest do
+ use Microwaveprop.DataCase, async: false
+
+ alias Microwaveprop.Repo
+ alias Microwaveprop.Weather
+ alias Microwaveprop.Weather.Sounding
+ alias Microwaveprop.Weather.Station
+ alias Microwaveprop.Weather.UwyoSoundingClient
+ alias Microwaveprop.Workers.CanadianSoundingFetchWorker
+
+ @fixture File.read!("test/support/fixtures/uwyo_sounding_yyr.html")
+
+ describe "most_recent_sounding_time/1" do
+ test "returns 00Z of the same day when called shortly after 02:00Z" do
+ # UWYO publishes 00Z soundings ~1h after launch; use 90 min buffer
+ assert CanadianSoundingFetchWorker.most_recent_sounding_time(~U[2026-04-13 02:00:00Z]) ==
+ ~U[2026-04-13 00:00:00Z]
+ end
+
+ test "returns 12Z of the same day after 14:00Z" do
+ assert CanadianSoundingFetchWorker.most_recent_sounding_time(~U[2026-04-13 14:00:00Z]) ==
+ ~U[2026-04-13 12:00:00Z]
+ end
+
+ test "returns prior day 12Z when called between midnight and 01:30Z" do
+ # Too early for today's 00Z run to be published
+ assert CanadianSoundingFetchWorker.most_recent_sounding_time(~U[2026-04-13 01:00:00Z]) ==
+ ~U[2026-04-12 12:00:00Z]
+ end
+
+ test "returns 00Z when called between 13:00Z and 13:30Z (before 12Z is ready)" do
+ assert CanadianSoundingFetchWorker.most_recent_sounding_time(~U[2026-04-13 13:00:00Z]) ==
+ ~U[2026-04-13 00:00:00Z]
+ end
+ end
+
+ describe "perform/1" do
+ setup do
+ cwse = insert_sounding_station("CWSE", "Edmonton Stony Plain")
+ cyyr = insert_sounding_station("CYYR", "Goose Bay")
+ us_sta = insert_sounding_station("KFWD", "Fort Worth")
+ %{cwse: cwse, cyyr: cyyr, us_sta: us_sta}
+ end
+
+ test "fetches only Canadian sounding stations (station_code starts with 'C')", %{cwse: cwse, us_sta: us_sta} do
+ stub_uwyo_with_fixture()
+
+ :ok =
+ CanadianSoundingFetchWorker.perform(%Oban.Job{
+ args: %{"sounding_time" => "2026-04-13T00:00:00Z"}
+ })
+
+ assert soundings_for_station(cwse.id) == 1
+ assert soundings_for_station(us_sta.id) == 0
+ end
+
+ test "stores a sounding with derived atmospheric parameters", %{cyyr: cyyr} do
+ stub_uwyo_with_fixture()
+
+ :ok =
+ CanadianSoundingFetchWorker.perform(%Oban.Job{
+ args: %{"sounding_time" => "2026-04-13T00:00:00Z"}
+ })
+
+ sounding = Repo.get_by!(Sounding, station_id: cyyr.id, observed_at: ~U[2026-04-13 00:00:00Z])
+
+ assert sounding.level_count > 0
+ assert is_float(sounding.surface_refractivity)
+ assert is_float(sounding.min_refractivity_gradient)
+ end
+
+ test "skips stations that already have a sounding for this observed_at", %{cwse: cwse, cyyr: cyyr} do
+ # Pre-existing stub sounding on BOTH Canadian stations so UWYO should
+ # never be called.
+ for sta <- [cwse, cyyr] do
+ {:ok, _} =
+ Weather.upsert_sounding(sta, %{
+ observed_at: ~U[2026-04-13 00:00:00Z],
+ profile: [],
+ level_count: 0
+ })
+ end
+
+ Req.Test.stub(UwyoSoundingClient, fn _conn ->
+ raise "UwyoSoundingClient was called for a station that already has a sounding"
+ end)
+
+ :ok =
+ CanadianSoundingFetchWorker.perform(%Oban.Job{
+ args: %{"sounding_time" => "2026-04-13T00:00:00Z"}
+ })
+
+ assert soundings_for_station(cwse.id) == 1
+ assert soundings_for_station(cyyr.id) == 1
+ end
+
+ test "uses most_recent_sounding_time when no sounding_time arg is given", %{cwse: cwse} do
+ stub_uwyo_with_fixture()
+
+ :ok = CanadianSoundingFetchWorker.perform(%Oban.Job{args: %{}})
+
+ # A sounding was created — use the function's own logic to figure out the observed_at
+ expected = CanadianSoundingFetchWorker.most_recent_sounding_time(DateTime.utc_now())
+
+ assert Repo.get_by(Sounding, station_id: cwse.id, observed_at: expected)
+ end
+ end
+
+ defp insert_sounding_station(code, name) do
+ %Station{}
+ |> Station.changeset(%{
+ station_code: code,
+ station_type: "sounding",
+ name: name,
+ lat: 50.0,
+ lon: -100.0
+ })
+ |> Repo.insert!()
+ end
+
+ defp soundings_for_station(station_id) do
+ Repo.aggregate(from(s in Sounding, where: s.station_id == ^station_id), :count, :id)
+ end
+
+ defp stub_uwyo_with_fixture do
+ Req.Test.stub(UwyoSoundingClient, fn conn ->
+ Req.Test.text(conn, @fixture)
+ end)
+ end
+end
diff --git a/test/support/fixtures/uwyo_sounding_yyr.html b/test/support/fixtures/uwyo_sounding_yyr.html
new file mode 100644
index 00000000..215c5927
--- /dev/null
+++ b/test/support/fixtures/uwyo_sounding_yyr.html
@@ -0,0 +1,141 @@
+
+University of Wyoming - Radiosonde Data
+
+71816 YYR Goose Bay Observations at 00Z 13 Apr 2026
+
+-----------------------------------------------------------------------------
+ PRES HGHT TEMP DWPT RELH MIXR DRCT SKNT THTA THTE THTV
+ hPa m C C % g/kg deg knot K K K
+-----------------------------------------------------------------------------
+ 1016.0 36 -3.5 -9.5 63 1.84 145 1 268.4 273.6 268.7
+ 1014.0 52 -0.1 -10.1 47 1.76 128 1 272.0 277.0 272.3
+ 1012.0 68 0.0 -11.0 43 1.64 111 1 272.2 276.9 272.5
+ 1007.0 109 0.2 -12.8 37 1.43 69 1 272.8 276.9 273.0
+ 1002.0 150 -0.3 -13.3 37 1.38 27 1 272.7 276.7 272.9
+ 1000.0 166 -0.3 -13.3 37 1.38 10 1 272.9 276.9 273.1
+ 982.7 305 -1.3 -15.8 32 1.14 20 5 273.2 276.6 273.4
+ 955.0 532 -2.9 -19.9 26 0.83 358 9 273.8 276.3 274.0
+ 945.6 610 -3.6 -20.3 26 0.81 350 11 273.9 276.4 274.1
+ 925.0 784 -5.1 -21.1 27 0.77 345 13 274.1 276.4 274.2
+ 909.6 914 -6.2 -21.0 30 0.79 335 14 274.3 276.7 274.4
+ 874.6 1219 -8.8 -20.8 37 0.84 310 17 274.6 277.2 274.8
+ 850.0 1441 -10.7 -20.7 44 0.87 300 18 274.9 277.6 275.1
+ 807.7 1829 -14.2 -20.5 59 0.93 310 20 275.3 278.1 275.4
+ 778.0 2114 -16.7 -20.4 73 0.98 319 22 275.5 278.4 275.7
+ 775.9 2134 -16.8 -20.5 73 0.97 320 22 275.6 278.5 275.8
+ 745.0 2438 -18.8 -21.8 77 0.90 330 23 276.7 279.4 276.9
+ 740.0 2488 -19.1 -22.0 78 0.89 328 23 276.9 279.6 277.0
+ 714.9 2743 -21.0 -23.7 79 0.80 320 23 277.6 280.0 277.7
+ 700.0 2899 -22.1 -24.7 79 0.74 310 22 278.0 280.3 278.1
+ 672.0 3199 -23.7 -26.4 78 0.66 303 19 279.4 281.5 279.6
+ 669.0 3231 -23.3 -30.3 53 0.46 303 18 280.3 281.7 280.3
+ 661.0 3320 -23.5 -33.5 39 0.34 301 17 281.0 282.1 281.1
+ 658.0 3353 -23.7 -35.2 34 0.29 300 17 281.1 282.1 281.2
+ 653.0 3409 -24.1 -38.1 26 0.22 301 18 281.3 282.0 281.3
+ 645.0 3499 -24.7 -33.7 43 0.35 302 19 281.6 282.8 281.7
+ 631.0 3658 -25.6 -40.6 23 0.18 305 22 282.3 282.9 282.4
+ 624.0 3739 -26.1 -44.1 17 0.12 306 23 282.7 283.1 282.7
+ 611.0 3891 -27.1 -36.1 42 0.29 307 24 283.2 284.2 283.3
+ 608.0 3927 -27.1 -42.1 23 0.16 307 24 283.6 284.2 283.7
+ 604.0 3974 -27.3 -40.3 28 0.19 308 25 283.9 284.6 284.0
+ 600.0 4022 -27.5 -45.5 16 0.11 308 25 284.2 284.6 284.3
+ 592.0 4119 -27.9 -44.9 18 0.12 309 26 284.9 285.3 284.9
+ 579.8 4267 -29.1 -44.4 21 0.13 310 27 285.2 285.7 285.2
+ 555.6 4572 -31.4 -43.4 30 0.15 310 26 285.9 286.4 285.9
+ 549.0 4657 -32.1 -43.1 33 0.16 309 25 286.1 286.6 286.1
+ 536.0 4826 -33.7 -46.7 26 0.11 306 23 286.1 286.5 286.2
+ 532.1 4877 -34.1 -45.1 32 0.13 305 22 286.3 286.7 286.3
+ 520.0 5038 -35.3 -40.3 60 0.22 299 23 286.7 287.5 286.8
+ 515.0 5105 -35.7 -46.7 31 0.11 297 24 287.0 287.4 287.0
+ 500.0 5310 -37.7 -42.7 59 0.18 290 25 287.0 287.6 287.1
+ 499.0 5324 -37.7 -43.7 53 0.16 290 25 287.2 287.7 287.2
+ 497.0 5351 -37.7 -51.7 22 0.07 289 25 287.5 287.8 287.5
+ 487.3 5486 -38.3 -55.7 14 0.04 285 25 288.4 288.5 288.4
+ 469.0 5748 -39.5 -63.5 6 0.02 287 25 290.1 290.1 290.1
+ 445.5 6096 -41.3 -61.6 9 0.02 290 24 292.1 292.2 292.1
+ 425.0 6415 -42.9 -59.9 14 0.03 294 29 294.0 294.1 294.0
+ 400.0 6820 -45.3 -60.3 17 0.03 300 36 296.0 296.1 296.0
+ 380.0 7161 -46.9 -60.9 19 0.03 306 41 298.3 298.4 298.3
+ 360.0 7518 -48.5 -65.5 12 0.02 313 46 300.8 300.9 300.8
+ 354.4 7620 -48.7 -66.0 12 0.01 315 48 301.8 301.9 301.8
+ 338.3 7925 -49.5 -67.7 10 0.01 320 53 304.9 304.9 304.9
+ 309.0 8518 -50.9 -70.9 8 0.01 312 55 310.9 310.9 310.9
+ 300.0 8710 -51.3 -70.3 9 0.01 310 56 312.9 313.0 312.9
+ 280.6 9144 -52.5 -71.9 8 0.01 305 59 317.3 317.4 317.3
+ 263.0 9563 -53.6 -73.5 7 0.01 310 65 321.6 321.6 321.6
+ 261.0 9613 -53.7 -73.7 7 0.01 315 65 322.1 322.1 322.1
+ 255.3 9754 -53.8 -75.3 5 0.01 310 65 324.0 324.0 324.0
+ 250.0 9890 -53.9 -76.9 4 0.00 305 50 325.8 325.8 325.8
+ 243.6 10058 -52.8 -77.9 3 0.00 310 50 329.9 329.9 329.9
+ 235.0 10290 -51.3 -79.3 2 0.00 310 51 335.6 335.6 335.6
+ 221.6 10668 -52.6 -80.6 2 0.00 310 53 339.2 339.2 339.2
+ 211.4 10973 -53.7 -81.7 2 0.00 310 46 342.1 342.1 342.1
+ 206.0 11140 -54.3 -82.3 2 0.00 308 48 343.7 343.7 343.7
+ 200.0 11330 -54.1 -82.1 2 0.00 305 50 346.9 346.9 346.9
+ 166.7 12497 -55.1 -83.1 2 0.00 300 53 363.8 363.8 363.8
+ 155.0 12960 -55.5 -83.5 2 0.00 297 46 370.8 370.8 370.8
+ 150.0 13170 -54.5 -83.5 2 0.00 295 43 376.0 376.0 376.0
+ 131.3 14021 -54.0 -83.8 1 0.00 290 46 391.4 391.4 391.4
+ 128.0 14185 -53.9 -83.9 1 0.00 285 42 394.5 394.5 394.5
+ 125.2 14326 -54.4 -84.1 1 0.00 280 39 396.1 396.1 396.1
+ 119.0 14650 -55.5 -84.5 2 0.00 282 40 399.8 399.9 399.8
+ 108.5 15240 -54.8 -84.3 1 0.00 285 42 411.9 412.0 411.9
+ 100.0 15760 -54.1 -84.1 1 0.00 285 36 422.9 422.9 422.9
+
Station information and sounding indices
+ Station identifier: YYR
+ Station number: 71816
+ Observation time: 260413/0000
+ Station latitude: 53.30
+ Station longitude: -60.36
+ Station elevation: 36.0
+ Showalter index: 8.12
+ Lifted index: 8.94
+ LIFT computed using virtual temperature: 8.95
+ SWEAT index: 61.01
+ K index: 3.70
+ Cross totals index: 17.00
+ Vertical totals index: 27.00
+ Totals totals index: 44.00
+ Convective Available Potential Energy: 0.00
+ CAPE using virtual temperature: 0.00
+ Convective Inhibition: 0.00
+ CINS using virtual temperature: 0.00
+ Bulk Richardson Number: 0.00
+ Bulk Richardson Number using CAPV: 0.00
+ Temp [K] of the Lifted Condensation Level: 255.05
+Pres [hPa] of the Lifted Condensation Level: 787.67
+ Equivalent potential temp [K] of the LCL: 276.50
+ Mean mixed layer potential temperature: 273.07
+ Mean mixed layer mixing ratio: 1.20
+ 1000 hPa to 500 hPa thickness: 5144.00
+Precipitable water [mm] for entire sounding: 3.59
+
+Description of the
+data columns
+or sounding indices.
+
+
+